blob: a82cfa1e2a4054ae9b09d679e13e71ce9617f802 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
Eddie Dong85f455f2007-07-06 12:20:49 +030018#include "irq.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080019#include "mmu.h"
Avi Kivitye4956062007-06-28 14:15:57 -040020
Avi Kivityedf88412007-12-16 11:02:48 +020021#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080022#include <linux/module.h>
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +020023#include <linux/kernel.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080024#include <linux/mm.h>
25#include <linux/highmem.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040026#include <linux/sched.h>
Avi Kivityc7addb92007-09-16 18:58:32 +020027#include <linux/moduleparam.h>
Marcelo Tosatti229456f2009-06-17 09:22:14 -030028#include <linux/ftrace_event.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090029#include <linux/slab.h>
Shane Wangcafd6652010-04-29 12:09:01 -040030#include <linux/tboot.h>
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030031#include "kvm_cache_regs.h"
Avi Kivity35920a32008-07-03 14:50:12 +030032#include "x86.h"
Avi Kivitye4956062007-06-28 14:15:57 -040033
Avi Kivity6aa8b732006-12-10 02:21:36 -080034#include <asm/io.h>
Anthony Liguori3b3be0d2006-12-13 00:33:43 -080035#include <asm/desc.h>
Eduardo Habkost13673a92008-11-17 19:03:13 -020036#include <asm/vmx.h>
Eduardo Habkost6210e372008-11-17 19:03:16 -020037#include <asm/virtext.h>
Andi Kleena0861c02009-06-08 17:37:09 +080038#include <asm/mce.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080039
Marcelo Tosatti229456f2009-06-17 09:22:14 -030040#include "trace.h"
41
Avi Kivity4ecac3f2008-05-13 13:23:38 +030042#define __ex(x) __kvm_handle_fault_on_reboot(x)
43
Avi Kivity6aa8b732006-12-10 02:21:36 -080044MODULE_AUTHOR("Qumranet");
45MODULE_LICENSE("GPL");
46
Avi Kivity4462d212009-03-23 17:53:37 +020047static int __read_mostly bypass_guest_pf = 1;
Avi Kivityc1f8bc02009-03-23 15:41:17 +020048module_param(bypass_guest_pf, bool, S_IRUGO);
Avi Kivityc7addb92007-09-16 18:58:32 +020049
Avi Kivity4462d212009-03-23 17:53:37 +020050static int __read_mostly enable_vpid = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020051module_param_named(vpid, enable_vpid, bool, 0444);
Sheng Yang2384d2b2008-01-17 15:14:33 +080052
Avi Kivity4462d212009-03-23 17:53:37 +020053static int __read_mostly flexpriority_enabled = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020054module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
Avi Kivity4c9fc8e2008-03-24 18:15:14 +020055
Avi Kivity4462d212009-03-23 17:53:37 +020056static int __read_mostly enable_ept = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020057module_param_named(ept, enable_ept, bool, S_IRUGO);
Sheng Yangd56f5462008-04-25 10:13:16 +080058
Nitin A Kamble3a624e22009-06-08 11:34:16 -070059static int __read_mostly enable_unrestricted_guest = 1;
60module_param_named(unrestricted_guest,
61 enable_unrestricted_guest, bool, S_IRUGO);
62
Avi Kivity4462d212009-03-23 17:53:37 +020063static int __read_mostly emulate_invalid_guest_state = 0;
Avi Kivityc1f8bc02009-03-23 15:41:17 +020064module_param(emulate_invalid_guest_state, bool, S_IRUGO);
Mohammed Gamal04fa4d32008-08-17 16:39:48 +030065
Avi Kivitycdc0e242009-12-06 17:21:14 +020066#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST \
67 (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
68#define KVM_GUEST_CR0_MASK \
69 (KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
70#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST \
Avi Kivity81231c62010-01-24 16:26:40 +020071 (X86_CR0_WP | X86_CR0_NE)
Avi Kivitycdc0e242009-12-06 17:21:14 +020072#define KVM_VM_CR0_ALWAYS_ON \
73 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
Avi Kivity4c386092009-12-07 12:26:18 +020074#define KVM_CR4_GUEST_OWNED_BITS \
75 (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
76 | X86_CR4_OSXMMEXCPT)
77
Avi Kivitycdc0e242009-12-06 17:21:14 +020078#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
79#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
80
Avi Kivity78ac8b42010-04-08 18:19:35 +030081#define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
82
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +080083/*
84 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
85 * ple_gap: upper bound on the amount of time between two successive
86 * executions of PAUSE in a loop. Also indicate if ple enabled.
87 * According to test, this time is usually small than 41 cycles.
88 * ple_window: upper bound on the amount of time a guest is allowed to execute
89 * in a PAUSE loop. Tests indicate that most spinlocks are held for
90 * less than 2^12 cycles
91 * Time is measured based on a counter that runs at the same rate as the TSC,
92 * refer SDM volume 3b section 21.6.13 & 22.1.3.
93 */
94#define KVM_VMX_DEFAULT_PLE_GAP 41
95#define KVM_VMX_DEFAULT_PLE_WINDOW 4096
96static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
97module_param(ple_gap, int, S_IRUGO);
98
99static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
100module_param(ple_window, int, S_IRUGO);
101
Avi Kivity61d2ef22010-04-28 16:40:38 +0300102#define NR_AUTOLOAD_MSRS 1
103
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400104struct vmcs {
105 u32 revision_id;
106 u32 abort;
107 char data[0];
108};
109
Avi Kivity26bb0982009-09-07 11:14:12 +0300110struct shared_msr_entry {
111 unsigned index;
112 u64 data;
Avi Kivityd5696722009-12-02 12:28:47 +0200113 u64 mask;
Avi Kivity26bb0982009-09-07 11:14:12 +0300114};
115
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400116struct vcpu_vmx {
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000117 struct kvm_vcpu vcpu;
Avi Kivity543e4242008-05-13 16:22:47 +0300118 struct list_head local_vcpus_link;
Avi Kivity313dbd42008-07-17 18:04:30 +0300119 unsigned long host_rsp;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400120 int launched;
Avi Kivity29bd8a72007-09-10 17:27:03 +0300121 u8 fail;
Avi Kivity1155f762007-11-22 11:30:47 +0200122 u32 idt_vectoring_info;
Avi Kivity26bb0982009-09-07 11:14:12 +0300123 struct shared_msr_entry *guest_msrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400124 int nmsrs;
125 int save_nmsrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400126#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300127 u64 msr_host_kernel_gs_base;
128 u64 msr_guest_kernel_gs_base;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400129#endif
130 struct vmcs *vmcs;
Avi Kivity61d2ef22010-04-28 16:40:38 +0300131 struct msr_autoload {
132 unsigned nr;
133 struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS];
134 struct vmx_msr_entry host[NR_AUTOLOAD_MSRS];
135 } msr_autoload;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400136 struct {
137 int loaded;
138 u16 fs_sel, gs_sel, ldt_sel;
Laurent Vivier152d3f22007-08-23 16:33:11 +0200139 int gs_ldt_reload_needed;
140 int fs_reload_needed;
Mike Dayd77c26f2007-10-08 09:02:08 -0400141 } host_state;
Avi Kivity9c8cba32007-11-22 11:42:59 +0200142 struct {
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300143 int vm86_active;
Avi Kivity78ac8b42010-04-08 18:19:35 +0300144 ulong save_rflags;
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300145 struct kvm_save_segment {
146 u16 selector;
147 unsigned long base;
148 u32 limit;
149 u32 ar;
150 } tr, es, ds, fs, gs;
Avi Kivity9c8cba32007-11-22 11:42:59 +0200151 struct {
152 bool pending;
153 u8 vector;
154 unsigned rip;
155 } irq;
156 } rmode;
Sheng Yang2384d2b2008-01-17 15:14:33 +0800157 int vpid;
Mohammed Gamal04fa4d32008-08-17 16:39:48 +0300158 bool emulation_required;
Jan Kiszka3b86cd92008-09-26 09:30:57 +0200159
160 /* Support for vnmi-less CPUs */
161 int soft_vnmi_blocked;
162 ktime_t entry_time;
163 s64 vnmi_blocked_time;
Andi Kleena0861c02009-06-08 17:37:09 +0800164 u32 exit_reason;
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800165
166 bool rdtscp_enabled;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400167};
168
169static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
170{
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000171 return container_of(vcpu, struct vcpu_vmx, vcpu);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400172}
173
Sheng Yangb7ebfb02008-04-25 21:44:52 +0800174static int init_rmode(struct kvm *kvm);
Sheng Yang4e1096d2008-07-06 19:16:51 +0800175static u64 construct_eptp(unsigned long root_hpa);
Avi Kivity75880a02007-06-20 11:20:04 +0300176
Avi Kivity6aa8b732006-12-10 02:21:36 -0800177static DEFINE_PER_CPU(struct vmcs *, vmxarea);
178static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
Avi Kivity543e4242008-05-13 16:22:47 +0300179static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800180
Avi Kivity3e7c73e2009-02-24 21:46:19 +0200181static unsigned long *vmx_io_bitmap_a;
182static unsigned long *vmx_io_bitmap_b;
Avi Kivity58972972009-02-24 22:26:47 +0200183static unsigned long *vmx_msr_bitmap_legacy;
184static unsigned long *vmx_msr_bitmap_longmode;
He, Qingfdef3ad2007-04-30 09:45:24 +0300185
Sheng Yang2384d2b2008-01-17 15:14:33 +0800186static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
187static DEFINE_SPINLOCK(vmx_vpid_lock);
188
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300189static struct vmcs_config {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800190 int size;
191 int order;
192 u32 revision_id;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300193 u32 pin_based_exec_ctrl;
194 u32 cpu_based_exec_ctrl;
Sheng Yangf78e0e22007-10-29 09:40:42 +0800195 u32 cpu_based_2nd_exec_ctrl;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300196 u32 vmexit_ctrl;
197 u32 vmentry_ctrl;
198} vmcs_config;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800199
Hannes Ederefff9e52008-11-28 17:02:06 +0100200static struct vmx_capability {
Sheng Yangd56f5462008-04-25 10:13:16 +0800201 u32 ept;
202 u32 vpid;
203} vmx_capability;
204
Avi Kivity6aa8b732006-12-10 02:21:36 -0800205#define VMX_SEGMENT_FIELD(seg) \
206 [VCPU_SREG_##seg] = { \
207 .selector = GUEST_##seg##_SELECTOR, \
208 .base = GUEST_##seg##_BASE, \
209 .limit = GUEST_##seg##_LIMIT, \
210 .ar_bytes = GUEST_##seg##_AR_BYTES, \
211 }
212
213static struct kvm_vmx_segment_field {
214 unsigned selector;
215 unsigned base;
216 unsigned limit;
217 unsigned ar_bytes;
218} kvm_vmx_segment_fields[] = {
219 VMX_SEGMENT_FIELD(CS),
220 VMX_SEGMENT_FIELD(DS),
221 VMX_SEGMENT_FIELD(ES),
222 VMX_SEGMENT_FIELD(FS),
223 VMX_SEGMENT_FIELD(GS),
224 VMX_SEGMENT_FIELD(SS),
225 VMX_SEGMENT_FIELD(TR),
226 VMX_SEGMENT_FIELD(LDTR),
227};
228
Avi Kivity26bb0982009-09-07 11:14:12 +0300229static u64 host_efer;
230
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300231static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
232
Avi Kivity4d56c8a2007-04-19 14:28:44 +0300233/*
234 * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
235 * away by decrementing the array size.
236 */
Avi Kivity6aa8b732006-12-10 02:21:36 -0800237static const u32 vmx_msr_index[] = {
Avi Kivity05b3e0c2006-12-13 00:33:45 -0800238#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300239 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
Avi Kivity6aa8b732006-12-10 02:21:36 -0800240#endif
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800241 MSR_EFER, MSR_TSC_AUX, MSR_K6_STAR,
Avi Kivity6aa8b732006-12-10 02:21:36 -0800242};
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +0200243#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800244
Gui Jianfeng31299942010-03-15 17:29:09 +0800245static inline bool is_page_fault(u32 intr_info)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800246{
247 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
248 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100249 (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800250}
251
Gui Jianfeng31299942010-03-15 17:29:09 +0800252static inline bool is_no_device(u32 intr_info)
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300253{
254 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
255 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100256 (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300257}
258
Gui Jianfeng31299942010-03-15 17:29:09 +0800259static inline bool is_invalid_opcode(u32 intr_info)
Anthony Liguori7aa81cc2007-09-17 14:57:50 -0500260{
261 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
262 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100263 (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -0500264}
265
Gui Jianfeng31299942010-03-15 17:29:09 +0800266static inline bool is_external_interrupt(u32 intr_info)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800267{
268 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
269 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
270}
271
Gui Jianfeng31299942010-03-15 17:29:09 +0800272static inline bool is_machine_check(u32 intr_info)
Andi Kleena0861c02009-06-08 17:37:09 +0800273{
274 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
275 INTR_INFO_VALID_MASK)) ==
276 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
277}
278
Gui Jianfeng31299942010-03-15 17:29:09 +0800279static inline bool cpu_has_vmx_msr_bitmap(void)
Sheng Yang25c5f222008-03-28 13:18:56 +0800280{
Sheng Yang04547152009-04-01 15:52:31 +0800281 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
Sheng Yang25c5f222008-03-28 13:18:56 +0800282}
283
Gui Jianfeng31299942010-03-15 17:29:09 +0800284static inline bool cpu_has_vmx_tpr_shadow(void)
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800285{
Sheng Yang04547152009-04-01 15:52:31 +0800286 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800287}
288
Gui Jianfeng31299942010-03-15 17:29:09 +0800289static inline bool vm_need_tpr_shadow(struct kvm *kvm)
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800290{
Sheng Yang04547152009-04-01 15:52:31 +0800291 return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800292}
293
Gui Jianfeng31299942010-03-15 17:29:09 +0800294static inline bool cpu_has_secondary_exec_ctrls(void)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800295{
Sheng Yang04547152009-04-01 15:52:31 +0800296 return vmcs_config.cpu_based_exec_ctrl &
297 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
Sheng Yangf78e0e22007-10-29 09:40:42 +0800298}
299
Avi Kivity774ead32007-12-26 13:57:04 +0200300static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800301{
Sheng Yang04547152009-04-01 15:52:31 +0800302 return vmcs_config.cpu_based_2nd_exec_ctrl &
303 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
304}
305
306static inline bool cpu_has_vmx_flexpriority(void)
307{
308 return cpu_has_vmx_tpr_shadow() &&
309 cpu_has_vmx_virtualize_apic_accesses();
Sheng Yangf78e0e22007-10-29 09:40:42 +0800310}
311
Marcelo Tosattie7997942009-06-11 12:07:40 -0300312static inline bool cpu_has_vmx_ept_execute_only(void)
313{
Gui Jianfeng31299942010-03-15 17:29:09 +0800314 return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
Marcelo Tosattie7997942009-06-11 12:07:40 -0300315}
316
317static inline bool cpu_has_vmx_eptp_uncacheable(void)
318{
Gui Jianfeng31299942010-03-15 17:29:09 +0800319 return vmx_capability.ept & VMX_EPTP_UC_BIT;
Marcelo Tosattie7997942009-06-11 12:07:40 -0300320}
321
322static inline bool cpu_has_vmx_eptp_writeback(void)
323{
Gui Jianfeng31299942010-03-15 17:29:09 +0800324 return vmx_capability.ept & VMX_EPTP_WB_BIT;
Marcelo Tosattie7997942009-06-11 12:07:40 -0300325}
326
327static inline bool cpu_has_vmx_ept_2m_page(void)
328{
Gui Jianfeng31299942010-03-15 17:29:09 +0800329 return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
Marcelo Tosattie7997942009-06-11 12:07:40 -0300330}
331
Sheng Yang878403b2010-01-05 19:02:29 +0800332static inline bool cpu_has_vmx_ept_1g_page(void)
333{
Gui Jianfeng31299942010-03-15 17:29:09 +0800334 return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
Sheng Yang878403b2010-01-05 19:02:29 +0800335}
336
Gui Jianfeng31299942010-03-15 17:29:09 +0800337static inline bool cpu_has_vmx_invept_individual_addr(void)
Sheng Yangd56f5462008-04-25 10:13:16 +0800338{
Gui Jianfeng31299942010-03-15 17:29:09 +0800339 return vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT;
Sheng Yangd56f5462008-04-25 10:13:16 +0800340}
341
Gui Jianfeng31299942010-03-15 17:29:09 +0800342static inline bool cpu_has_vmx_invept_context(void)
Sheng Yangd56f5462008-04-25 10:13:16 +0800343{
Gui Jianfeng31299942010-03-15 17:29:09 +0800344 return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
Sheng Yangd56f5462008-04-25 10:13:16 +0800345}
346
Gui Jianfeng31299942010-03-15 17:29:09 +0800347static inline bool cpu_has_vmx_invept_global(void)
Sheng Yangd56f5462008-04-25 10:13:16 +0800348{
Gui Jianfeng31299942010-03-15 17:29:09 +0800349 return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
Sheng Yangd56f5462008-04-25 10:13:16 +0800350}
351
Gui Jianfeng31299942010-03-15 17:29:09 +0800352static inline bool cpu_has_vmx_ept(void)
Sheng Yangd56f5462008-04-25 10:13:16 +0800353{
Sheng Yang04547152009-04-01 15:52:31 +0800354 return vmcs_config.cpu_based_2nd_exec_ctrl &
355 SECONDARY_EXEC_ENABLE_EPT;
Sheng Yangd56f5462008-04-25 10:13:16 +0800356}
357
Gui Jianfeng31299942010-03-15 17:29:09 +0800358static inline bool cpu_has_vmx_unrestricted_guest(void)
Nitin A Kamble3a624e22009-06-08 11:34:16 -0700359{
360 return vmcs_config.cpu_based_2nd_exec_ctrl &
361 SECONDARY_EXEC_UNRESTRICTED_GUEST;
362}
363
Gui Jianfeng31299942010-03-15 17:29:09 +0800364static inline bool cpu_has_vmx_ple(void)
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +0800365{
366 return vmcs_config.cpu_based_2nd_exec_ctrl &
367 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
368}
369
Gui Jianfeng31299942010-03-15 17:29:09 +0800370static inline bool vm_need_virtualize_apic_accesses(struct kvm *kvm)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800371{
Gui Jianfeng6d3e4352010-01-29 15:36:59 +0800372 return flexpriority_enabled && irqchip_in_kernel(kvm);
Sheng Yangf78e0e22007-10-29 09:40:42 +0800373}
374
Gui Jianfeng31299942010-03-15 17:29:09 +0800375static inline bool cpu_has_vmx_vpid(void)
Sheng Yang2384d2b2008-01-17 15:14:33 +0800376{
Sheng Yang04547152009-04-01 15:52:31 +0800377 return vmcs_config.cpu_based_2nd_exec_ctrl &
378 SECONDARY_EXEC_ENABLE_VPID;
Sheng Yang2384d2b2008-01-17 15:14:33 +0800379}
380
Gui Jianfeng31299942010-03-15 17:29:09 +0800381static inline bool cpu_has_vmx_rdtscp(void)
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800382{
383 return vmcs_config.cpu_based_2nd_exec_ctrl &
384 SECONDARY_EXEC_RDTSCP;
385}
386
Gui Jianfeng31299942010-03-15 17:29:09 +0800387static inline bool cpu_has_virtual_nmis(void)
Sheng Yangf08864b2008-05-15 18:23:25 +0800388{
389 return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
390}
391
Sheng Yang04547152009-04-01 15:52:31 +0800392static inline bool report_flexpriority(void)
393{
394 return flexpriority_enabled;
395}
396
Rusty Russell8b9cf982007-07-30 16:31:43 +1000397static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
Avi Kivity7725f0b2006-12-13 00:34:01 -0800398{
399 int i;
400
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400401 for (i = 0; i < vmx->nmsrs; ++i)
Avi Kivity26bb0982009-09-07 11:14:12 +0300402 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
Eddie Donga75beee2007-05-17 18:55:15 +0300403 return i;
404 return -1;
405}
406
Sheng Yang2384d2b2008-01-17 15:14:33 +0800407static inline void __invvpid(int ext, u16 vpid, gva_t gva)
408{
409 struct {
410 u64 vpid : 16;
411 u64 rsvd : 48;
412 u64 gva;
413 } operand = { vpid, 0, gva };
414
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300415 asm volatile (__ex(ASM_VMX_INVVPID)
Sheng Yang2384d2b2008-01-17 15:14:33 +0800416 /* CF==1 or ZF==1 --> rc = -1 */
417 "; ja 1f ; ud2 ; 1:"
418 : : "a"(&operand), "c"(ext) : "cc", "memory");
419}
420
Sheng Yang14394422008-04-28 12:24:45 +0800421static inline void __invept(int ext, u64 eptp, gpa_t gpa)
422{
423 struct {
424 u64 eptp, gpa;
425 } operand = {eptp, gpa};
426
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300427 asm volatile (__ex(ASM_VMX_INVEPT)
Sheng Yang14394422008-04-28 12:24:45 +0800428 /* CF==1 or ZF==1 --> rc = -1 */
429 "; ja 1f ; ud2 ; 1:\n"
430 : : "a" (&operand), "c" (ext) : "cc", "memory");
431}
432
Avi Kivity26bb0982009-09-07 11:14:12 +0300433static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
Eddie Donga75beee2007-05-17 18:55:15 +0300434{
435 int i;
436
Rusty Russell8b9cf982007-07-30 16:31:43 +1000437 i = __find_msr_index(vmx, msr);
Eddie Donga75beee2007-05-17 18:55:15 +0300438 if (i >= 0)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400439 return &vmx->guest_msrs[i];
Al Viro8b6d44c2007-02-09 16:38:40 +0000440 return NULL;
Avi Kivity7725f0b2006-12-13 00:34:01 -0800441}
442
Avi Kivity6aa8b732006-12-10 02:21:36 -0800443static void vmcs_clear(struct vmcs *vmcs)
444{
445 u64 phys_addr = __pa(vmcs);
446 u8 error;
447
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300448 asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
Avi Kivity6aa8b732006-12-10 02:21:36 -0800449 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
450 : "cc", "memory");
451 if (error)
452 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
453 vmcs, phys_addr);
454}
455
456static void __vcpu_clear(void *arg)
457{
Rusty Russell8b9cf982007-07-30 16:31:43 +1000458 struct vcpu_vmx *vmx = arg;
Ingo Molnard3b2c332007-01-05 16:36:23 -0800459 int cpu = raw_smp_processor_id();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800460
Rusty Russell8b9cf982007-07-30 16:31:43 +1000461 if (vmx->vcpu.cpu == cpu)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400462 vmcs_clear(vmx->vmcs);
463 if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800464 per_cpu(current_vmcs, cpu) = NULL;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800465 rdtscll(vmx->vcpu.arch.host_tsc);
Avi Kivity543e4242008-05-13 16:22:47 +0300466 list_del(&vmx->local_vcpus_link);
467 vmx->vcpu.cpu = -1;
468 vmx->launched = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800469}
470
Rusty Russell8b9cf982007-07-30 16:31:43 +1000471static void vcpu_clear(struct vcpu_vmx *vmx)
Avi Kivity8d0be2b2007-02-12 00:54:46 -0800472{
Avi Kivityeae5ecb2007-09-30 10:50:12 +0200473 if (vmx->vcpu.cpu == -1)
474 return;
Jens Axboe8691e5a2008-06-06 11:18:06 +0200475 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
Avi Kivity8d0be2b2007-02-12 00:54:46 -0800476}
477
Sheng Yang2384d2b2008-01-17 15:14:33 +0800478static inline void vpid_sync_vcpu_all(struct vcpu_vmx *vmx)
479{
480 if (vmx->vpid == 0)
481 return;
482
483 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
484}
485
Sheng Yang14394422008-04-28 12:24:45 +0800486static inline void ept_sync_global(void)
487{
488 if (cpu_has_vmx_invept_global())
489 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
490}
491
492static inline void ept_sync_context(u64 eptp)
493{
Avi Kivity089d0342009-03-23 18:26:32 +0200494 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +0800495 if (cpu_has_vmx_invept_context())
496 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
497 else
498 ept_sync_global();
499 }
500}
501
502static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
503{
Avi Kivity089d0342009-03-23 18:26:32 +0200504 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +0800505 if (cpu_has_vmx_invept_individual_addr())
506 __invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
507 eptp, gpa);
508 else
509 ept_sync_context(eptp);
510 }
511}
512
Avi Kivity6aa8b732006-12-10 02:21:36 -0800513static unsigned long vmcs_readl(unsigned long field)
514{
515 unsigned long value;
516
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300517 asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800518 : "=a"(value) : "d"(field) : "cc");
519 return value;
520}
521
522static u16 vmcs_read16(unsigned long field)
523{
524 return vmcs_readl(field);
525}
526
527static u32 vmcs_read32(unsigned long field)
528{
529 return vmcs_readl(field);
530}
531
532static u64 vmcs_read64(unsigned long field)
533{
Avi Kivity05b3e0c2006-12-13 00:33:45 -0800534#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -0800535 return vmcs_readl(field);
536#else
537 return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
538#endif
539}
540
Avi Kivitye52de1b2007-01-05 16:36:56 -0800541static noinline void vmwrite_error(unsigned long field, unsigned long value)
542{
543 printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
544 field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
545 dump_stack();
546}
547
Avi Kivity6aa8b732006-12-10 02:21:36 -0800548static void vmcs_writel(unsigned long field, unsigned long value)
549{
550 u8 error;
551
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300552 asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
Mike Dayd77c26f2007-10-08 09:02:08 -0400553 : "=q"(error) : "a"(value), "d"(field) : "cc");
Avi Kivitye52de1b2007-01-05 16:36:56 -0800554 if (unlikely(error))
555 vmwrite_error(field, value);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800556}
557
558static void vmcs_write16(unsigned long field, u16 value)
559{
560 vmcs_writel(field, value);
561}
562
563static void vmcs_write32(unsigned long field, u32 value)
564{
565 vmcs_writel(field, value);
566}
567
568static void vmcs_write64(unsigned long field, u64 value)
569{
Avi Kivity6aa8b732006-12-10 02:21:36 -0800570 vmcs_writel(field, value);
Avi Kivity7682f2d2008-05-12 19:25:43 +0300571#ifndef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -0800572 asm volatile ("");
573 vmcs_writel(field+1, value >> 32);
574#endif
575}
576
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300577static void vmcs_clear_bits(unsigned long field, u32 mask)
578{
579 vmcs_writel(field, vmcs_readl(field) & ~mask);
580}
581
582static void vmcs_set_bits(unsigned long field, u32 mask)
583{
584 vmcs_writel(field, vmcs_readl(field) | mask);
585}
586
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300587static void update_exception_bitmap(struct kvm_vcpu *vcpu)
588{
589 u32 eb;
590
Jan Kiszkafd7373c2010-01-20 18:20:20 +0100591 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
592 (1u << NM_VECTOR) | (1u << DB_VECTOR);
593 if ((vcpu->guest_debug &
594 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
595 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
596 eb |= 1u << BP_VECTOR;
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300597 if (to_vmx(vcpu)->rmode.vm86_active)
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300598 eb = ~0;
Avi Kivity089d0342009-03-23 18:26:32 +0200599 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +0800600 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
Avi Kivity02daab22009-12-30 12:40:26 +0200601 if (vcpu->fpu_active)
602 eb &= ~(1u << NM_VECTOR);
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300603 vmcs_write32(EXCEPTION_BITMAP, eb);
604}
605
Avi Kivity61d2ef22010-04-28 16:40:38 +0300606static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
607{
608 unsigned i;
609 struct msr_autoload *m = &vmx->msr_autoload;
610
611 for (i = 0; i < m->nr; ++i)
612 if (m->guest[i].index == msr)
613 break;
614
615 if (i == m->nr)
616 return;
617 --m->nr;
618 m->guest[i] = m->guest[m->nr];
619 m->host[i] = m->host[m->nr];
620 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
621 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
622}
623
624static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
625 u64 guest_val, u64 host_val)
626{
627 unsigned i;
628 struct msr_autoload *m = &vmx->msr_autoload;
629
630 for (i = 0; i < m->nr; ++i)
631 if (m->guest[i].index == msr)
632 break;
633
634 if (i == m->nr) {
635 ++m->nr;
636 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr);
637 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr);
638 }
639
640 m->guest[i].index = msr;
641 m->guest[i].value = guest_val;
642 m->host[i].index = msr;
643 m->host[i].value = host_val;
644}
645
Avi Kivity33ed6322007-05-02 16:54:03 +0300646static void reload_tss(void)
647{
Avi Kivity33ed6322007-05-02 16:54:03 +0300648 /*
649 * VT restores TR but not its size. Useless.
650 */
Gleb Natapov89a27f42010-02-16 10:51:48 +0200651 struct desc_ptr gdt;
Avi Kivitya5f61302008-02-20 17:57:21 +0200652 struct desc_struct *descs;
Avi Kivity33ed6322007-05-02 16:54:03 +0300653
Gleb Natapovd6ab1ed2010-02-25 12:43:07 +0200654 native_store_gdt(&gdt);
Gleb Natapov89a27f42010-02-16 10:51:48 +0200655 descs = (void *)gdt.address;
Avi Kivity33ed6322007-05-02 16:54:03 +0300656 descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
657 load_TR_desc();
Avi Kivity33ed6322007-05-02 16:54:03 +0300658}
659
Avi Kivity92c0d902009-10-29 11:00:16 +0200660static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
Eddie Dong2cc51562007-05-21 07:28:09 +0300661{
Roel Kluin3a34a882009-08-04 02:08:45 -0700662 u64 guest_efer;
Avi Kivity51c6cf62007-08-29 03:48:05 +0300663 u64 ignore_bits;
Eddie Dong2cc51562007-05-21 07:28:09 +0300664
Avi Kivityf6801df2010-01-21 15:31:50 +0200665 guest_efer = vmx->vcpu.arch.efer;
Roel Kluin3a34a882009-08-04 02:08:45 -0700666
Avi Kivity51c6cf62007-08-29 03:48:05 +0300667 /*
668 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
669 * outside long mode
670 */
671 ignore_bits = EFER_NX | EFER_SCE;
672#ifdef CONFIG_X86_64
673 ignore_bits |= EFER_LMA | EFER_LME;
674 /* SCE is meaningful only in long mode on Intel */
675 if (guest_efer & EFER_LMA)
676 ignore_bits &= ~(u64)EFER_SCE;
677#endif
Avi Kivity51c6cf62007-08-29 03:48:05 +0300678 guest_efer &= ~ignore_bits;
679 guest_efer |= host_efer & ignore_bits;
Avi Kivity26bb0982009-09-07 11:14:12 +0300680 vmx->guest_msrs[efer_offset].data = guest_efer;
Avi Kivityd5696722009-12-02 12:28:47 +0200681 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
Avi Kivity84ad33e2010-04-28 16:42:29 +0300682
683 clear_atomic_switch_msr(vmx, MSR_EFER);
684 /* On ept, can't emulate nx, and must switch nx atomically */
685 if (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX)) {
686 guest_efer = vmx->vcpu.arch.efer;
687 if (!(guest_efer & EFER_LMA))
688 guest_efer &= ~EFER_LME;
689 add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, host_efer);
690 return false;
691 }
692
Avi Kivity26bb0982009-09-07 11:14:12 +0300693 return true;
Avi Kivity51c6cf62007-08-29 03:48:05 +0300694}
695
Gleb Natapov2d49ec72010-02-25 12:43:09 +0200696static unsigned long segment_base(u16 selector)
697{
698 struct desc_ptr gdt;
699 struct desc_struct *d;
700 unsigned long table_base;
701 unsigned long v;
702
703 if (!(selector & ~3))
704 return 0;
705
706 native_store_gdt(&gdt);
707 table_base = gdt.address;
708
709 if (selector & 4) { /* from ldt */
710 u16 ldt_selector = kvm_read_ldt();
711
712 if (!(ldt_selector & ~3))
713 return 0;
714
715 table_base = segment_base(ldt_selector);
716 }
717 d = (struct desc_struct *)(table_base + (selector & ~7));
718 v = get_desc_base(d);
719#ifdef CONFIG_X86_64
720 if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
721 v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
722#endif
723 return v;
724}
725
726static inline unsigned long kvm_read_tr_base(void)
727{
728 u16 tr;
729 asm("str %0" : "=g"(tr));
730 return segment_base(tr);
731}
732
Avi Kivity04d2cc72007-09-10 18:10:54 +0300733static void vmx_save_host_state(struct kvm_vcpu *vcpu)
Avi Kivity33ed6322007-05-02 16:54:03 +0300734{
Avi Kivity04d2cc72007-09-10 18:10:54 +0300735 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +0300736 int i;
Avi Kivity04d2cc72007-09-10 18:10:54 +0300737
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400738 if (vmx->host_state.loaded)
Avi Kivity33ed6322007-05-02 16:54:03 +0300739 return;
740
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400741 vmx->host_state.loaded = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300742 /*
743 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
744 * allow segment selectors with cpl > 0 or ti == 1.
745 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300746 vmx->host_state.ldt_sel = kvm_read_ldt();
Laurent Vivier152d3f22007-08-23 16:33:11 +0200747 vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300748 vmx->host_state.fs_sel = kvm_read_fs();
Laurent Vivier152d3f22007-08-23 16:33:11 +0200749 if (!(vmx->host_state.fs_sel & 7)) {
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400750 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200751 vmx->host_state.fs_reload_needed = 0;
752 } else {
Avi Kivity33ed6322007-05-02 16:54:03 +0300753 vmcs_write16(HOST_FS_SELECTOR, 0);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200754 vmx->host_state.fs_reload_needed = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300755 }
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300756 vmx->host_state.gs_sel = kvm_read_gs();
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400757 if (!(vmx->host_state.gs_sel & 7))
758 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300759 else {
760 vmcs_write16(HOST_GS_SELECTOR, 0);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200761 vmx->host_state.gs_ldt_reload_needed = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300762 }
763
764#ifdef CONFIG_X86_64
765 vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
766 vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
767#else
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400768 vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
769 vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
Avi Kivity33ed6322007-05-02 16:54:03 +0300770#endif
Avi Kivity707c0872007-05-02 17:33:43 +0300771
772#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300773 if (is_long_mode(&vmx->vcpu)) {
774 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
775 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
776 }
Avi Kivity707c0872007-05-02 17:33:43 +0300777#endif
Avi Kivity26bb0982009-09-07 11:14:12 +0300778 for (i = 0; i < vmx->save_nmsrs; ++i)
779 kvm_set_shared_msr(vmx->guest_msrs[i].index,
Avi Kivityd5696722009-12-02 12:28:47 +0200780 vmx->guest_msrs[i].data,
781 vmx->guest_msrs[i].mask);
Avi Kivity33ed6322007-05-02 16:54:03 +0300782}
783
Avi Kivitya9b21b62008-06-24 11:48:49 +0300784static void __vmx_load_host_state(struct vcpu_vmx *vmx)
Avi Kivity33ed6322007-05-02 16:54:03 +0300785{
Avi Kivity15ad7142007-07-11 18:17:21 +0300786 unsigned long flags;
Avi Kivity33ed6322007-05-02 16:54:03 +0300787
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400788 if (!vmx->host_state.loaded)
Avi Kivity33ed6322007-05-02 16:54:03 +0300789 return;
790
Avi Kivitye1beb1d2007-11-18 13:50:24 +0200791 ++vmx->vcpu.stat.host_state_reload;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400792 vmx->host_state.loaded = 0;
Laurent Vivier152d3f22007-08-23 16:33:11 +0200793 if (vmx->host_state.fs_reload_needed)
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300794 kvm_load_fs(vmx->host_state.fs_sel);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200795 if (vmx->host_state.gs_ldt_reload_needed) {
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300796 kvm_load_ldt(vmx->host_state.ldt_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300797 /*
798 * If we have to reload gs, we must take care to
799 * preserve our gs base.
800 */
Avi Kivity15ad7142007-07-11 18:17:21 +0300801 local_irq_save(flags);
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300802 kvm_load_gs(vmx->host_state.gs_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300803#ifdef CONFIG_X86_64
804 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
805#endif
Avi Kivity15ad7142007-07-11 18:17:21 +0300806 local_irq_restore(flags);
Avi Kivity33ed6322007-05-02 16:54:03 +0300807 }
Laurent Vivier152d3f22007-08-23 16:33:11 +0200808 reload_tss();
Avi Kivity44ea2b12009-09-06 15:55:37 +0300809#ifdef CONFIG_X86_64
810 if (is_long_mode(&vmx->vcpu)) {
811 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
812 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
813 }
814#endif
Avi Kivity1c11e712010-05-03 16:05:44 +0300815 if (current_thread_info()->status & TS_USEDFPU)
816 clts();
Avi Kivity33ed6322007-05-02 16:54:03 +0300817}
818
Avi Kivitya9b21b62008-06-24 11:48:49 +0300819static void vmx_load_host_state(struct vcpu_vmx *vmx)
820{
821 preempt_disable();
822 __vmx_load_host_state(vmx);
823 preempt_enable();
824}
825
Avi Kivity6aa8b732006-12-10 02:21:36 -0800826/*
827 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
828 * vcpu mutex is already taken.
829 */
Avi Kivity15ad7142007-07-11 18:17:21 +0300830static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800831{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400832 struct vcpu_vmx *vmx = to_vmx(vcpu);
833 u64 phys_addr = __pa(vmx->vmcs);
Avi Kivity019960a2008-03-04 10:44:51 +0200834 u64 tsc_this, delta, new_offset;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800835
Eddie Donga3d7f852007-09-03 16:15:12 +0300836 if (vcpu->cpu != cpu) {
Rusty Russell8b9cf982007-07-30 16:31:43 +1000837 vcpu_clear(vmx);
Marcelo Tosatti2f599712008-05-27 12:10:20 -0300838 kvm_migrate_timers(vcpu);
Marcelo Tosattieb5109e2009-10-01 19:16:58 -0300839 set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests);
Avi Kivity543e4242008-05-13 16:22:47 +0300840 local_irq_disable();
841 list_add(&vmx->local_vcpus_link,
842 &per_cpu(vcpus_on_cpu, cpu));
843 local_irq_enable();
Eddie Donga3d7f852007-09-03 16:15:12 +0300844 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800845
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400846 if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800847 u8 error;
848
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400849 per_cpu(current_vmcs, cpu) = vmx->vmcs;
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300850 asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
Avi Kivity6aa8b732006-12-10 02:21:36 -0800851 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
852 : "cc");
853 if (error)
854 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400855 vmx->vmcs, phys_addr);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800856 }
857
858 if (vcpu->cpu != cpu) {
Gleb Natapov89a27f42010-02-16 10:51:48 +0200859 struct desc_ptr dt;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800860 unsigned long sysenter_esp;
861
862 vcpu->cpu = cpu;
863 /*
864 * Linux uses per-cpu TSS and GDT, so set these when switching
865 * processors.
866 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300867 vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
Gleb Natapovd6ab1ed2010-02-25 12:43:07 +0200868 native_store_gdt(&dt);
Gleb Natapov89a27f42010-02-16 10:51:48 +0200869 vmcs_writel(HOST_GDTR_BASE, dt.address); /* 22.2.4 */
Avi Kivity6aa8b732006-12-10 02:21:36 -0800870
871 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
872 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
Avi Kivity77002702007-06-13 19:55:28 +0300873
874 /*
875 * Make sure the time stamp counter is monotonous.
876 */
877 rdtscll(tsc_this);
Avi Kivity019960a2008-03-04 10:44:51 +0200878 if (tsc_this < vcpu->arch.host_tsc) {
879 delta = vcpu->arch.host_tsc - tsc_this;
880 new_offset = vmcs_read64(TSC_OFFSET) + delta;
881 vmcs_write64(TSC_OFFSET, new_offset);
882 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800883 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800884}
885
886static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
887{
Avi Kivitya9b21b62008-06-24 11:48:49 +0300888 __vmx_load_host_state(to_vmx(vcpu));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800889}
890
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300891static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
892{
Avi Kivity81231c62010-01-24 16:26:40 +0200893 ulong cr0;
894
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300895 if (vcpu->fpu_active)
896 return;
897 vcpu->fpu_active = 1;
Avi Kivity81231c62010-01-24 16:26:40 +0200898 cr0 = vmcs_readl(GUEST_CR0);
899 cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
900 cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
901 vmcs_writel(GUEST_CR0, cr0);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300902 update_exception_bitmap(vcpu);
Avi Kivityedcafe32009-12-30 18:07:40 +0200903 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
904 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300905}
906
Avi Kivityedcafe32009-12-30 18:07:40 +0200907static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
908
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300909static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
910{
Avi Kivityedcafe32009-12-30 18:07:40 +0200911 vmx_decache_cr0_guest_bits(vcpu);
Avi Kivity81231c62010-01-24 16:26:40 +0200912 vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300913 update_exception_bitmap(vcpu);
Avi Kivityedcafe32009-12-30 18:07:40 +0200914 vcpu->arch.cr0_guest_owned_bits = 0;
915 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
916 vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300917}
918
Avi Kivity6aa8b732006-12-10 02:21:36 -0800919static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
920{
Avi Kivity78ac8b42010-04-08 18:19:35 +0300921 unsigned long rflags, save_rflags;
Avi Kivity345dcaa2009-08-12 15:29:37 +0300922
923 rflags = vmcs_readl(GUEST_RFLAGS);
Avi Kivity78ac8b42010-04-08 18:19:35 +0300924 if (to_vmx(vcpu)->rmode.vm86_active) {
925 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
926 save_rflags = to_vmx(vcpu)->rmode.save_rflags;
927 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
928 }
Avi Kivity345dcaa2009-08-12 15:29:37 +0300929 return rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800930}
931
932static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
933{
Avi Kivity78ac8b42010-04-08 18:19:35 +0300934 if (to_vmx(vcpu)->rmode.vm86_active) {
935 to_vmx(vcpu)->rmode.save_rflags = rflags;
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100936 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
Avi Kivity78ac8b42010-04-08 18:19:35 +0300937 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800938 vmcs_writel(GUEST_RFLAGS, rflags);
939}
940
Glauber Costa2809f5d2009-05-12 16:21:05 -0400941static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
942{
943 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
944 int ret = 0;
945
946 if (interruptibility & GUEST_INTR_STATE_STI)
Jan Kiszka48005f62010-02-19 19:38:07 +0100947 ret |= KVM_X86_SHADOW_INT_STI;
Glauber Costa2809f5d2009-05-12 16:21:05 -0400948 if (interruptibility & GUEST_INTR_STATE_MOV_SS)
Jan Kiszka48005f62010-02-19 19:38:07 +0100949 ret |= KVM_X86_SHADOW_INT_MOV_SS;
Glauber Costa2809f5d2009-05-12 16:21:05 -0400950
951 return ret & mask;
952}
953
954static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
955{
956 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
957 u32 interruptibility = interruptibility_old;
958
959 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
960
Jan Kiszka48005f62010-02-19 19:38:07 +0100961 if (mask & KVM_X86_SHADOW_INT_MOV_SS)
Glauber Costa2809f5d2009-05-12 16:21:05 -0400962 interruptibility |= GUEST_INTR_STATE_MOV_SS;
Jan Kiszka48005f62010-02-19 19:38:07 +0100963 else if (mask & KVM_X86_SHADOW_INT_STI)
Glauber Costa2809f5d2009-05-12 16:21:05 -0400964 interruptibility |= GUEST_INTR_STATE_STI;
965
966 if ((interruptibility != interruptibility_old))
967 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
968}
969
Avi Kivity6aa8b732006-12-10 02:21:36 -0800970static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
971{
972 unsigned long rip;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800973
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300974 rip = kvm_rip_read(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800975 rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300976 kvm_rip_write(vcpu, rip);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800977
Glauber Costa2809f5d2009-05-12 16:21:05 -0400978 /* skipping an emulated instruction also counts */
979 vmx_set_interrupt_shadow(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800980}
981
Avi Kivity298101d2007-11-25 13:41:11 +0200982static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200983 bool has_error_code, u32 error_code,
984 bool reinject)
Avi Kivity298101d2007-11-25 13:41:11 +0200985{
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200986 struct vcpu_vmx *vmx = to_vmx(vcpu);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100987 u32 intr_info = nr | INTR_INFO_VALID_MASK;
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200988
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100989 if (has_error_code) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200990 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100991 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
992 }
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200993
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300994 if (vmx->rmode.vm86_active) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200995 vmx->rmode.irq.pending = true;
996 vmx->rmode.irq.vector = nr;
997 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
Gleb Natapovae0bb3e2009-05-19 11:07:10 +0300998 if (kvm_exception_is_soft(nr))
999 vmx->rmode.irq.rip +=
1000 vmx->vcpu.arch.event_exit_inst_len;
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +01001001 intr_info |= INTR_TYPE_SOFT_INTR;
1002 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
Jan Kiszka77ab6db2008-07-14 12:28:51 +02001003 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
1004 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
1005 return;
1006 }
1007
Gleb Natapov66fd3f72009-05-11 13:35:50 +03001008 if (kvm_exception_is_soft(nr)) {
1009 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1010 vmx->vcpu.arch.event_exit_inst_len);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +01001011 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1012 } else
1013 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1014
1015 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
Avi Kivity298101d2007-11-25 13:41:11 +02001016}
1017
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001018static bool vmx_rdtscp_supported(void)
1019{
1020 return cpu_has_vmx_rdtscp();
1021}
1022
Avi Kivity6aa8b732006-12-10 02:21:36 -08001023/*
Eddie Donga75beee2007-05-17 18:55:15 +03001024 * Swap MSR entry in host/guest MSR entry array.
1025 */
Rusty Russell8b9cf982007-07-30 16:31:43 +10001026static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
Eddie Donga75beee2007-05-17 18:55:15 +03001027{
Avi Kivity26bb0982009-09-07 11:14:12 +03001028 struct shared_msr_entry tmp;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001029
1030 tmp = vmx->guest_msrs[to];
1031 vmx->guest_msrs[to] = vmx->guest_msrs[from];
1032 vmx->guest_msrs[from] = tmp;
Eddie Donga75beee2007-05-17 18:55:15 +03001033}
1034
1035/*
Avi Kivitye38aea32007-04-19 13:22:48 +03001036 * Set up the vmcs to automatically save and restore system
1037 * msrs. Don't touch the 64-bit msrs if the guest is in legacy
1038 * mode, as fiddling with msrs is very expensive.
1039 */
Rusty Russell8b9cf982007-07-30 16:31:43 +10001040static void setup_msrs(struct vcpu_vmx *vmx)
Avi Kivitye38aea32007-04-19 13:22:48 +03001041{
Avi Kivity26bb0982009-09-07 11:14:12 +03001042 int save_nmsrs, index;
Avi Kivity58972972009-02-24 22:26:47 +02001043 unsigned long *msr_bitmap;
Avi Kivitye38aea32007-04-19 13:22:48 +03001044
Avi Kivity33f9c502008-02-27 16:06:57 +02001045 vmx_load_host_state(vmx);
Eddie Donga75beee2007-05-17 18:55:15 +03001046 save_nmsrs = 0;
Avi Kivity4d56c8a2007-04-19 14:28:44 +03001047#ifdef CONFIG_X86_64
Rusty Russell8b9cf982007-07-30 16:31:43 +10001048 if (is_long_mode(&vmx->vcpu)) {
Rusty Russell8b9cf982007-07-30 16:31:43 +10001049 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
Eddie Donga75beee2007-05-17 18:55:15 +03001050 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +10001051 move_msr_up(vmx, index, save_nmsrs++);
1052 index = __find_msr_index(vmx, MSR_LSTAR);
Eddie Donga75beee2007-05-17 18:55:15 +03001053 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +10001054 move_msr_up(vmx, index, save_nmsrs++);
1055 index = __find_msr_index(vmx, MSR_CSTAR);
Eddie Donga75beee2007-05-17 18:55:15 +03001056 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +10001057 move_msr_up(vmx, index, save_nmsrs++);
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001058 index = __find_msr_index(vmx, MSR_TSC_AUX);
1059 if (index >= 0 && vmx->rdtscp_enabled)
1060 move_msr_up(vmx, index, save_nmsrs++);
Eddie Donga75beee2007-05-17 18:55:15 +03001061 /*
1062 * MSR_K6_STAR is only needed on long mode guests, and only
1063 * if efer.sce is enabled.
1064 */
Rusty Russell8b9cf982007-07-30 16:31:43 +10001065 index = __find_msr_index(vmx, MSR_K6_STAR);
Avi Kivityf6801df2010-01-21 15:31:50 +02001066 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
Rusty Russell8b9cf982007-07-30 16:31:43 +10001067 move_msr_up(vmx, index, save_nmsrs++);
Avi Kivity4d56c8a2007-04-19 14:28:44 +03001068 }
Eddie Donga75beee2007-05-17 18:55:15 +03001069#endif
Avi Kivity92c0d902009-10-29 11:00:16 +02001070 index = __find_msr_index(vmx, MSR_EFER);
1071 if (index >= 0 && update_transition_efer(vmx, index))
Avi Kivity26bb0982009-09-07 11:14:12 +03001072 move_msr_up(vmx, index, save_nmsrs++);
Avi Kivity4d56c8a2007-04-19 14:28:44 +03001073
Avi Kivity26bb0982009-09-07 11:14:12 +03001074 vmx->save_nmsrs = save_nmsrs;
Avi Kivity58972972009-02-24 22:26:47 +02001075
1076 if (cpu_has_vmx_msr_bitmap()) {
1077 if (is_long_mode(&vmx->vcpu))
1078 msr_bitmap = vmx_msr_bitmap_longmode;
1079 else
1080 msr_bitmap = vmx_msr_bitmap_legacy;
1081
1082 vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
1083 }
Avi Kivitye38aea32007-04-19 13:22:48 +03001084}
1085
1086/*
Avi Kivity6aa8b732006-12-10 02:21:36 -08001087 * reads and returns guest's timestamp counter "register"
1088 * guest_tsc = host_tsc + tsc_offset -- 21.3
1089 */
1090static u64 guest_read_tsc(void)
1091{
1092 u64 host_tsc, tsc_offset;
1093
1094 rdtscll(host_tsc);
1095 tsc_offset = vmcs_read64(TSC_OFFSET);
1096 return host_tsc + tsc_offset;
1097}
1098
1099/*
1100 * writes 'guest_tsc' into guest's timestamp counter "register"
1101 * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
1102 */
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01001103static void guest_write_tsc(u64 guest_tsc, u64 host_tsc)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001104{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001105 vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
1106}
1107
Avi Kivity6aa8b732006-12-10 02:21:36 -08001108/*
1109 * Reads an msr value (of 'msr_index') into 'pdata'.
1110 * Returns 0 on success, non-0 otherwise.
1111 * Assumes vcpu_load() was already called.
1112 */
1113static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1114{
1115 u64 data;
Avi Kivity26bb0982009-09-07 11:14:12 +03001116 struct shared_msr_entry *msr;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001117
1118 if (!pdata) {
1119 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
1120 return -EINVAL;
1121 }
1122
1123 switch (msr_index) {
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001124#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001125 case MSR_FS_BASE:
1126 data = vmcs_readl(GUEST_FS_BASE);
1127 break;
1128 case MSR_GS_BASE:
1129 data = vmcs_readl(GUEST_GS_BASE);
1130 break;
Avi Kivity44ea2b12009-09-06 15:55:37 +03001131 case MSR_KERNEL_GS_BASE:
1132 vmx_load_host_state(to_vmx(vcpu));
1133 data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
1134 break;
Avi Kivity26bb0982009-09-07 11:14:12 +03001135#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -08001136 case MSR_EFER:
Avi Kivity3bab1f52006-12-29 16:49:48 -08001137 return kvm_get_msr_common(vcpu, msr_index, pdata);
Jaswinder Singh Rajputaf24a4e2009-05-15 18:42:05 +05301138 case MSR_IA32_TSC:
Avi Kivity6aa8b732006-12-10 02:21:36 -08001139 data = guest_read_tsc();
1140 break;
1141 case MSR_IA32_SYSENTER_CS:
1142 data = vmcs_read32(GUEST_SYSENTER_CS);
1143 break;
1144 case MSR_IA32_SYSENTER_EIP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001145 data = vmcs_readl(GUEST_SYSENTER_EIP);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001146 break;
1147 case MSR_IA32_SYSENTER_ESP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001148 data = vmcs_readl(GUEST_SYSENTER_ESP);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001149 break;
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001150 case MSR_TSC_AUX:
1151 if (!to_vmx(vcpu)->rdtscp_enabled)
1152 return 1;
1153 /* Otherwise falls through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001154 default:
Avi Kivity26bb0982009-09-07 11:14:12 +03001155 vmx_load_host_state(to_vmx(vcpu));
Rusty Russell8b9cf982007-07-30 16:31:43 +10001156 msr = find_msr_entry(to_vmx(vcpu), msr_index);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001157 if (msr) {
Gleb Natapov542423b2009-08-27 15:07:30 +03001158 vmx_load_host_state(to_vmx(vcpu));
Avi Kivity3bab1f52006-12-29 16:49:48 -08001159 data = msr->data;
1160 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001161 }
Avi Kivity3bab1f52006-12-29 16:49:48 -08001162 return kvm_get_msr_common(vcpu, msr_index, pdata);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001163 }
1164
1165 *pdata = data;
1166 return 0;
1167}
1168
1169/*
1170 * Writes msr value into into the appropriate "register".
1171 * Returns 0 on success, non-0 otherwise.
1172 * Assumes vcpu_load() was already called.
1173 */
1174static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1175{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001176 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +03001177 struct shared_msr_entry *msr;
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01001178 u64 host_tsc;
Eddie Dong2cc51562007-05-21 07:28:09 +03001179 int ret = 0;
1180
Avi Kivity6aa8b732006-12-10 02:21:36 -08001181 switch (msr_index) {
Avi Kivity3bab1f52006-12-29 16:49:48 -08001182 case MSR_EFER:
Avi Kivitya9b21b62008-06-24 11:48:49 +03001183 vmx_load_host_state(vmx);
Eddie Dong2cc51562007-05-21 07:28:09 +03001184 ret = kvm_set_msr_common(vcpu, msr_index, data);
Eddie Dong2cc51562007-05-21 07:28:09 +03001185 break;
Avi Kivity16175a72009-03-23 22:13:44 +02001186#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001187 case MSR_FS_BASE:
1188 vmcs_writel(GUEST_FS_BASE, data);
1189 break;
1190 case MSR_GS_BASE:
1191 vmcs_writel(GUEST_GS_BASE, data);
1192 break;
Avi Kivity44ea2b12009-09-06 15:55:37 +03001193 case MSR_KERNEL_GS_BASE:
1194 vmx_load_host_state(vmx);
1195 vmx->msr_guest_kernel_gs_base = data;
1196 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001197#endif
1198 case MSR_IA32_SYSENTER_CS:
1199 vmcs_write32(GUEST_SYSENTER_CS, data);
1200 break;
1201 case MSR_IA32_SYSENTER_EIP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001202 vmcs_writel(GUEST_SYSENTER_EIP, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001203 break;
1204 case MSR_IA32_SYSENTER_ESP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001205 vmcs_writel(GUEST_SYSENTER_ESP, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001206 break;
Jaswinder Singh Rajputaf24a4e2009-05-15 18:42:05 +05301207 case MSR_IA32_TSC:
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01001208 rdtscll(host_tsc);
1209 guest_write_tsc(data, host_tsc);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001210 break;
Sheng Yang468d4722008-10-09 16:01:55 +08001211 case MSR_IA32_CR_PAT:
1212 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
1213 vmcs_write64(GUEST_IA32_PAT, data);
1214 vcpu->arch.pat = data;
1215 break;
1216 }
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001217 ret = kvm_set_msr_common(vcpu, msr_index, data);
1218 break;
1219 case MSR_TSC_AUX:
1220 if (!vmx->rdtscp_enabled)
1221 return 1;
1222 /* Check reserved bit, higher 32 bits should be zero */
1223 if ((data >> 32) != 0)
1224 return 1;
1225 /* Otherwise falls through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001226 default:
Rusty Russell8b9cf982007-07-30 16:31:43 +10001227 msr = find_msr_entry(vmx, msr_index);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001228 if (msr) {
Gleb Natapov542423b2009-08-27 15:07:30 +03001229 vmx_load_host_state(vmx);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001230 msr->data = data;
1231 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001232 }
Eddie Dong2cc51562007-05-21 07:28:09 +03001233 ret = kvm_set_msr_common(vcpu, msr_index, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001234 }
1235
Eddie Dong2cc51562007-05-21 07:28:09 +03001236 return ret;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001237}
1238
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001239static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001240{
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001241 __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
1242 switch (reg) {
1243 case VCPU_REGS_RSP:
1244 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
1245 break;
1246 case VCPU_REGS_RIP:
1247 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
1248 break;
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001249 case VCPU_EXREG_PDPTR:
1250 if (enable_ept)
1251 ept_save_pdptrs(vcpu);
1252 break;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001253 default:
1254 break;
1255 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001256}
1257
Jan Kiszka355be0b2009-10-03 00:31:21 +02001258static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001259{
Jan Kiszkaae675ef2008-12-15 13:52:10 +01001260 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1261 vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
1262 else
1263 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
1264
Avi Kivityabd3f2d2007-05-02 17:57:40 +03001265 update_exception_bitmap(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001266}
1267
1268static __init int cpu_has_kvm_support(void)
1269{
Eduardo Habkost6210e372008-11-17 19:03:16 -02001270 return cpu_has_vmx();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001271}
1272
1273static __init int vmx_disabled_by_bios(void)
1274{
1275 u64 msr;
1276
1277 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
Shane Wangcafd6652010-04-29 12:09:01 -04001278 if (msr & FEATURE_CONTROL_LOCKED) {
1279 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
1280 && tboot_enabled())
1281 return 1;
1282 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
1283 && !tboot_enabled())
1284 return 1;
1285 }
1286
1287 return 0;
Yang, Sheng62b3ffb2007-07-25 12:17:06 +03001288 /* locked but not enabled */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001289}
1290
Alexander Graf10474ae2009-09-15 11:37:46 +02001291static int hardware_enable(void *garbage)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001292{
1293 int cpu = raw_smp_processor_id();
1294 u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
Shane Wangcafd6652010-04-29 12:09:01 -04001295 u64 old, test_bits;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001296
Alexander Graf10474ae2009-09-15 11:37:46 +02001297 if (read_cr4() & X86_CR4_VMXE)
1298 return -EBUSY;
1299
Avi Kivity543e4242008-05-13 16:22:47 +03001300 INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001301 rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
Shane Wangcafd6652010-04-29 12:09:01 -04001302
1303 test_bits = FEATURE_CONTROL_LOCKED;
1304 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
1305 if (tboot_enabled())
1306 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
1307
1308 if ((old & test_bits) != test_bits) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001309 /* enable and lock */
Shane Wangcafd6652010-04-29 12:09:01 -04001310 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
1311 }
Rusty Russell66aee912007-07-17 23:34:16 +10001312 write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001313 asm volatile (ASM_VMX_VMXON_RAX
1314 : : "a"(&phys_addr), "m"(phys_addr)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001315 : "memory", "cc");
Alexander Graf10474ae2009-09-15 11:37:46 +02001316
1317 ept_sync_global();
1318
1319 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001320}
1321
Avi Kivity543e4242008-05-13 16:22:47 +03001322static void vmclear_local_vcpus(void)
1323{
1324 int cpu = raw_smp_processor_id();
1325 struct vcpu_vmx *vmx, *n;
1326
1327 list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
1328 local_vcpus_link)
1329 __vcpu_clear(vmx);
1330}
1331
Eduardo Habkost710ff4a2008-11-17 19:03:18 -02001332
1333/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
1334 * tricks.
1335 */
1336static void kvm_cpu_vmxoff(void)
1337{
1338 asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
1339 write_cr4(read_cr4() & ~X86_CR4_VMXE);
1340}
1341
Avi Kivity6aa8b732006-12-10 02:21:36 -08001342static void hardware_disable(void *garbage)
1343{
Avi Kivity543e4242008-05-13 16:22:47 +03001344 vmclear_local_vcpus();
Eduardo Habkost710ff4a2008-11-17 19:03:18 -02001345 kvm_cpu_vmxoff();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001346}
1347
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001348static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
Mike Dayd77c26f2007-10-08 09:02:08 -04001349 u32 msr, u32 *result)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001350{
1351 u32 vmx_msr_low, vmx_msr_high;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001352 u32 ctl = ctl_min | ctl_opt;
1353
1354 rdmsr(msr, vmx_msr_low, vmx_msr_high);
1355
1356 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
1357 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
1358
1359 /* Ensure minimum (required) set of control bits are supported. */
1360 if (ctl_min & ~ctl)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001361 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001362
1363 *result = ctl;
1364 return 0;
1365}
1366
Yang, Sheng002c7f72007-07-31 14:23:01 +03001367static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001368{
1369 u32 vmx_msr_low, vmx_msr_high;
Sheng Yangd56f5462008-04-25 10:13:16 +08001370 u32 min, opt, min2, opt2;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001371 u32 _pin_based_exec_control = 0;
1372 u32 _cpu_based_exec_control = 0;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001373 u32 _cpu_based_2nd_exec_control = 0;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001374 u32 _vmexit_control = 0;
1375 u32 _vmentry_control = 0;
1376
1377 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
Sheng Yangf08864b2008-05-15 18:23:25 +08001378 opt = PIN_BASED_VIRTUAL_NMIS;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001379 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
1380 &_pin_based_exec_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001381 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001382
1383 min = CPU_BASED_HLT_EXITING |
1384#ifdef CONFIG_X86_64
1385 CPU_BASED_CR8_LOAD_EXITING |
1386 CPU_BASED_CR8_STORE_EXITING |
1387#endif
Sheng Yangd56f5462008-04-25 10:13:16 +08001388 CPU_BASED_CR3_LOAD_EXITING |
1389 CPU_BASED_CR3_STORE_EXITING |
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001390 CPU_BASED_USE_IO_BITMAPS |
1391 CPU_BASED_MOV_DR_EXITING |
Marcelo Tosattia7052892008-09-23 13:18:35 -03001392 CPU_BASED_USE_TSC_OFFSETING |
Sheng Yang59708672009-12-15 13:29:54 +08001393 CPU_BASED_MWAIT_EXITING |
1394 CPU_BASED_MONITOR_EXITING |
Marcelo Tosattia7052892008-09-23 13:18:35 -03001395 CPU_BASED_INVLPG_EXITING;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001396 opt = CPU_BASED_TPR_SHADOW |
Sheng Yang25c5f222008-03-28 13:18:56 +08001397 CPU_BASED_USE_MSR_BITMAPS |
Sheng Yangf78e0e22007-10-29 09:40:42 +08001398 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001399 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
1400 &_cpu_based_exec_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001401 return -EIO;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08001402#ifdef CONFIG_X86_64
1403 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
1404 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
1405 ~CPU_BASED_CR8_STORE_EXITING;
1406#endif
Sheng Yangf78e0e22007-10-29 09:40:42 +08001407 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
Sheng Yangd56f5462008-04-25 10:13:16 +08001408 min2 = 0;
1409 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
Sheng Yang2384d2b2008-01-17 15:14:33 +08001410 SECONDARY_EXEC_WBINVD_EXITING |
Sheng Yangd56f5462008-04-25 10:13:16 +08001411 SECONDARY_EXEC_ENABLE_VPID |
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001412 SECONDARY_EXEC_ENABLE_EPT |
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08001413 SECONDARY_EXEC_UNRESTRICTED_GUEST |
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001414 SECONDARY_EXEC_PAUSE_LOOP_EXITING |
1415 SECONDARY_EXEC_RDTSCP;
Sheng Yangd56f5462008-04-25 10:13:16 +08001416 if (adjust_vmx_controls(min2, opt2,
1417 MSR_IA32_VMX_PROCBASED_CTLS2,
Sheng Yangf78e0e22007-10-29 09:40:42 +08001418 &_cpu_based_2nd_exec_control) < 0)
1419 return -EIO;
1420 }
1421#ifndef CONFIG_X86_64
1422 if (!(_cpu_based_2nd_exec_control &
1423 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
1424 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
1425#endif
Sheng Yangd56f5462008-04-25 10:13:16 +08001426 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
Marcelo Tosattia7052892008-09-23 13:18:35 -03001427 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
1428 enabled */
Gleb Natapov5fff7d22009-08-27 18:41:30 +03001429 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
1430 CPU_BASED_CR3_STORE_EXITING |
1431 CPU_BASED_INVLPG_EXITING);
Sheng Yangd56f5462008-04-25 10:13:16 +08001432 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
1433 vmx_capability.ept, vmx_capability.vpid);
1434 }
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001435
1436 min = 0;
1437#ifdef CONFIG_X86_64
1438 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
1439#endif
Sheng Yang468d4722008-10-09 16:01:55 +08001440 opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001441 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
1442 &_vmexit_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001443 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001444
Sheng Yang468d4722008-10-09 16:01:55 +08001445 min = 0;
1446 opt = VM_ENTRY_LOAD_IA32_PAT;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001447 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
1448 &_vmentry_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001449 return -EIO;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001450
Nguyen Anh Quynhc68876f2006-12-29 16:49:54 -08001451 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001452
1453 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
1454 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001455 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001456
1457#ifdef CONFIG_X86_64
1458 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
1459 if (vmx_msr_high & (1u<<16))
Yang, Sheng002c7f72007-07-31 14:23:01 +03001460 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001461#endif
1462
1463 /* Require Write-Back (WB) memory type for VMCS accesses. */
1464 if (((vmx_msr_high >> 18) & 15) != 6)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001465 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001466
Yang, Sheng002c7f72007-07-31 14:23:01 +03001467 vmcs_conf->size = vmx_msr_high & 0x1fff;
1468 vmcs_conf->order = get_order(vmcs_config.size);
1469 vmcs_conf->revision_id = vmx_msr_low;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001470
Yang, Sheng002c7f72007-07-31 14:23:01 +03001471 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
1472 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001473 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Yang, Sheng002c7f72007-07-31 14:23:01 +03001474 vmcs_conf->vmexit_ctrl = _vmexit_control;
1475 vmcs_conf->vmentry_ctrl = _vmentry_control;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001476
1477 return 0;
Nguyen Anh Quynhc68876f2006-12-29 16:49:54 -08001478}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001479
1480static struct vmcs *alloc_vmcs_cpu(int cpu)
1481{
1482 int node = cpu_to_node(cpu);
1483 struct page *pages;
1484 struct vmcs *vmcs;
1485
Mel Gorman6484eb32009-06-16 15:31:54 -07001486 pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001487 if (!pages)
1488 return NULL;
1489 vmcs = page_address(pages);
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001490 memset(vmcs, 0, vmcs_config.size);
1491 vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001492 return vmcs;
1493}
1494
1495static struct vmcs *alloc_vmcs(void)
1496{
Ingo Molnard3b2c332007-01-05 16:36:23 -08001497 return alloc_vmcs_cpu(raw_smp_processor_id());
Avi Kivity6aa8b732006-12-10 02:21:36 -08001498}
1499
1500static void free_vmcs(struct vmcs *vmcs)
1501{
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001502 free_pages((unsigned long)vmcs, vmcs_config.order);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001503}
1504
Sam Ravnborg39959582007-06-01 00:47:13 -07001505static void free_kvm_area(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001506{
1507 int cpu;
1508
Zachary Amsden3230bb42009-09-29 11:38:37 -10001509 for_each_possible_cpu(cpu) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001510 free_vmcs(per_cpu(vmxarea, cpu));
Zachary Amsden3230bb42009-09-29 11:38:37 -10001511 per_cpu(vmxarea, cpu) = NULL;
1512 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001513}
1514
Avi Kivity6aa8b732006-12-10 02:21:36 -08001515static __init int alloc_kvm_area(void)
1516{
1517 int cpu;
1518
Zachary Amsden3230bb42009-09-29 11:38:37 -10001519 for_each_possible_cpu(cpu) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001520 struct vmcs *vmcs;
1521
1522 vmcs = alloc_vmcs_cpu(cpu);
1523 if (!vmcs) {
1524 free_kvm_area();
1525 return -ENOMEM;
1526 }
1527
1528 per_cpu(vmxarea, cpu) = vmcs;
1529 }
1530 return 0;
1531}
1532
1533static __init int hardware_setup(void)
1534{
Yang, Sheng002c7f72007-07-31 14:23:01 +03001535 if (setup_vmcs_config(&vmcs_config) < 0)
1536 return -EIO;
Joerg Roedel50a37eb2008-01-31 14:57:38 +01001537
1538 if (boot_cpu_has(X86_FEATURE_NX))
1539 kvm_enable_efer_bits(EFER_NX);
1540
Sheng Yang93ba03c2009-04-01 15:52:32 +08001541 if (!cpu_has_vmx_vpid())
1542 enable_vpid = 0;
1543
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001544 if (!cpu_has_vmx_ept()) {
Sheng Yang93ba03c2009-04-01 15:52:32 +08001545 enable_ept = 0;
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001546 enable_unrestricted_guest = 0;
1547 }
1548
1549 if (!cpu_has_vmx_unrestricted_guest())
1550 enable_unrestricted_guest = 0;
Sheng Yang93ba03c2009-04-01 15:52:32 +08001551
1552 if (!cpu_has_vmx_flexpriority())
1553 flexpriority_enabled = 0;
1554
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001555 if (!cpu_has_vmx_tpr_shadow())
1556 kvm_x86_ops->update_cr8_intercept = NULL;
1557
Marcelo Tosatti54dee992009-06-11 12:07:44 -03001558 if (enable_ept && !cpu_has_vmx_ept_2m_page())
1559 kvm_disable_largepages();
1560
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08001561 if (!cpu_has_vmx_ple())
1562 ple_gap = 0;
1563
Avi Kivity6aa8b732006-12-10 02:21:36 -08001564 return alloc_kvm_area();
1565}
1566
1567static __exit void hardware_unsetup(void)
1568{
1569 free_kvm_area();
1570}
1571
Avi Kivity6aa8b732006-12-10 02:21:36 -08001572static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
1573{
1574 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1575
Avi Kivity6af11b92007-03-19 13:18:10 +02001576 if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001577 vmcs_write16(sf->selector, save->selector);
1578 vmcs_writel(sf->base, save->base);
1579 vmcs_write32(sf->limit, save->limit);
1580 vmcs_write32(sf->ar_bytes, save->ar);
1581 } else {
1582 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
1583 << AR_DPL_SHIFT;
1584 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
1585 }
1586}
1587
1588static void enter_pmode(struct kvm_vcpu *vcpu)
1589{
1590 unsigned long flags;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001591 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001592
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001593 vmx->emulation_required = 1;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001594 vmx->rmode.vm86_active = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001595
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001596 vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
1597 vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
1598 vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001599
1600 flags = vmcs_readl(GUEST_RFLAGS);
Avi Kivity78ac8b42010-04-08 18:19:35 +03001601 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
1602 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001603 vmcs_writel(GUEST_RFLAGS, flags);
1604
Rusty Russell66aee912007-07-17 23:34:16 +10001605 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
1606 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001607
1608 update_exception_bitmap(vcpu);
1609
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001610 if (emulate_invalid_guest_state)
1611 return;
1612
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001613 fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
1614 fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
1615 fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
1616 fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001617
1618 vmcs_write16(GUEST_SS_SELECTOR, 0);
1619 vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
1620
1621 vmcs_write16(GUEST_CS_SELECTOR,
1622 vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
1623 vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1624}
1625
Mike Dayd77c26f2007-10-08 09:02:08 -04001626static gva_t rmode_tss_base(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001627{
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08001628 if (!kvm->arch.tss_addr) {
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001629 struct kvm_memslots *slots;
1630 gfn_t base_gfn;
1631
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08001632 slots = kvm_memslots(kvm);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001633 base_gfn = kvm->memslots->memslots[0].base_gfn +
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -02001634 kvm->memslots->memslots[0].npages - 3;
Izik Eiduscbc94022007-10-25 00:29:55 +02001635 return base_gfn << PAGE_SHIFT;
1636 }
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08001637 return kvm->arch.tss_addr;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001638}
1639
1640static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
1641{
1642 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1643
1644 save->selector = vmcs_read16(sf->selector);
1645 save->base = vmcs_readl(sf->base);
1646 save->limit = vmcs_read32(sf->limit);
1647 save->ar = vmcs_read32(sf->ar_bytes);
Jan Kiszka15b00f32007-11-19 10:21:45 +01001648 vmcs_write16(sf->selector, save->base >> 4);
1649 vmcs_write32(sf->base, save->base & 0xfffff);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001650 vmcs_write32(sf->limit, 0xffff);
1651 vmcs_write32(sf->ar_bytes, 0xf3);
1652}
1653
1654static void enter_rmode(struct kvm_vcpu *vcpu)
1655{
1656 unsigned long flags;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001657 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001658
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001659 if (enable_unrestricted_guest)
1660 return;
1661
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001662 vmx->emulation_required = 1;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001663 vmx->rmode.vm86_active = 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001664
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001665 vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001666 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
1667
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001668 vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001669 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
1670
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001671 vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001672 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1673
1674 flags = vmcs_readl(GUEST_RFLAGS);
Avi Kivity78ac8b42010-04-08 18:19:35 +03001675 vmx->rmode.save_rflags = flags;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001676
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +01001677 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001678
1679 vmcs_writel(GUEST_RFLAGS, flags);
Rusty Russell66aee912007-07-17 23:34:16 +10001680 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001681 update_exception_bitmap(vcpu);
1682
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001683 if (emulate_invalid_guest_state)
1684 goto continue_rmode;
1685
Avi Kivity6aa8b732006-12-10 02:21:36 -08001686 vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
1687 vmcs_write32(GUEST_SS_LIMIT, 0xffff);
1688 vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
1689
1690 vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
Michael Riepeabacf8d2006-12-22 01:05:45 -08001691 vmcs_write32(GUEST_CS_LIMIT, 0xffff);
Avi Kivity8cb5b032007-03-20 18:40:40 +02001692 if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
1693 vmcs_writel(GUEST_CS_BASE, 0xf0000);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001694 vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
1695
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001696 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
1697 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
1698 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
1699 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
Avi Kivity75880a02007-06-20 11:20:04 +03001700
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001701continue_rmode:
Eddie Dong8668a3c2007-10-10 14:26:45 +08001702 kvm_mmu_reset_context(vcpu);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08001703 init_rmode(vcpu->kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001704}
1705
Amit Shah401d10d2009-02-20 22:53:37 +05301706static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1707{
1708 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +03001709 struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
1710
1711 if (!msr)
1712 return;
Amit Shah401d10d2009-02-20 22:53:37 +05301713
Avi Kivity44ea2b12009-09-06 15:55:37 +03001714 /*
1715 * Force kernel_gs_base reloading before EFER changes, as control
1716 * of this msr depends on is_long_mode().
1717 */
1718 vmx_load_host_state(to_vmx(vcpu));
Avi Kivityf6801df2010-01-21 15:31:50 +02001719 vcpu->arch.efer = efer;
Amit Shah401d10d2009-02-20 22:53:37 +05301720 if (efer & EFER_LMA) {
1721 vmcs_write32(VM_ENTRY_CONTROLS,
1722 vmcs_read32(VM_ENTRY_CONTROLS) |
1723 VM_ENTRY_IA32E_MODE);
1724 msr->data = efer;
1725 } else {
1726 vmcs_write32(VM_ENTRY_CONTROLS,
1727 vmcs_read32(VM_ENTRY_CONTROLS) &
1728 ~VM_ENTRY_IA32E_MODE);
1729
1730 msr->data = efer & ~EFER_LME;
1731 }
1732 setup_msrs(vmx);
1733}
1734
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001735#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001736
1737static void enter_lmode(struct kvm_vcpu *vcpu)
1738{
1739 u32 guest_tr_ar;
1740
1741 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
1742 if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
1743 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001744 __func__);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001745 vmcs_write32(GUEST_TR_AR_BYTES,
1746 (guest_tr_ar & ~AR_TYPE_MASK)
1747 | AR_TYPE_BUSY_64_TSS);
1748 }
Avi Kivityda38f432010-07-06 11:30:49 +03001749 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001750}
1751
1752static void exit_lmode(struct kvm_vcpu *vcpu)
1753{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001754 vmcs_write32(VM_ENTRY_CONTROLS,
1755 vmcs_read32(VM_ENTRY_CONTROLS)
Li, Xin B1e4e6e02007-08-01 21:49:10 +03001756 & ~VM_ENTRY_IA32E_MODE);
Avi Kivityda38f432010-07-06 11:30:49 +03001757 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001758}
1759
1760#endif
1761
Sheng Yang2384d2b2008-01-17 15:14:33 +08001762static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
1763{
1764 vpid_sync_vcpu_all(to_vmx(vcpu));
Avi Kivity089d0342009-03-23 18:26:32 +02001765 if (enable_ept)
Sheng Yang4e1096d2008-07-06 19:16:51 +08001766 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
Sheng Yang2384d2b2008-01-17 15:14:33 +08001767}
1768
Avi Kivitye8467fd2009-12-29 18:43:06 +02001769static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1770{
1771 ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
1772
1773 vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
1774 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
1775}
1776
Anthony Liguori25c4c272007-04-27 09:29:21 +03001777static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
Avi Kivity399badf2007-01-05 16:36:38 -08001778{
Avi Kivityfc78f512009-12-07 12:16:48 +02001779 ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
1780
1781 vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
1782 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
Avi Kivity399badf2007-01-05 16:36:38 -08001783}
1784
Sheng Yang14394422008-04-28 12:24:45 +08001785static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
1786{
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001787 if (!test_bit(VCPU_EXREG_PDPTR,
1788 (unsigned long *)&vcpu->arch.regs_dirty))
1789 return;
1790
Sheng Yang14394422008-04-28 12:24:45 +08001791 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
Sheng Yang14394422008-04-28 12:24:45 +08001792 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]);
1793 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]);
1794 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]);
1795 vmcs_write64(GUEST_PDPTR3, vcpu->arch.pdptrs[3]);
1796 }
1797}
1798
Avi Kivity8f5d5492009-05-31 18:41:29 +03001799static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
1800{
1801 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1802 vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
1803 vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
1804 vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
1805 vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1806 }
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001807
1808 __set_bit(VCPU_EXREG_PDPTR,
1809 (unsigned long *)&vcpu->arch.regs_avail);
1810 __set_bit(VCPU_EXREG_PDPTR,
1811 (unsigned long *)&vcpu->arch.regs_dirty);
Avi Kivity8f5d5492009-05-31 18:41:29 +03001812}
1813
Sheng Yang14394422008-04-28 12:24:45 +08001814static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1815
1816static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
1817 unsigned long cr0,
1818 struct kvm_vcpu *vcpu)
1819{
1820 if (!(cr0 & X86_CR0_PG)) {
1821 /* From paging/starting to nonpaging */
1822 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
Sheng Yang65267ea2008-06-18 14:43:38 +08001823 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
Sheng Yang14394422008-04-28 12:24:45 +08001824 (CPU_BASED_CR3_LOAD_EXITING |
1825 CPU_BASED_CR3_STORE_EXITING));
1826 vcpu->arch.cr0 = cr0;
Avi Kivityfc78f512009-12-07 12:16:48 +02001827 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
Sheng Yang14394422008-04-28 12:24:45 +08001828 } else if (!is_paging(vcpu)) {
1829 /* From nonpaging to paging */
1830 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
Sheng Yang65267ea2008-06-18 14:43:38 +08001831 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
Sheng Yang14394422008-04-28 12:24:45 +08001832 ~(CPU_BASED_CR3_LOAD_EXITING |
1833 CPU_BASED_CR3_STORE_EXITING));
1834 vcpu->arch.cr0 = cr0;
Avi Kivityfc78f512009-12-07 12:16:48 +02001835 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
Sheng Yang14394422008-04-28 12:24:45 +08001836 }
Sheng Yang95eb84a2009-08-19 09:52:18 +08001837
1838 if (!(cr0 & X86_CR0_WP))
1839 *hw_cr0 &= ~X86_CR0_WP;
Sheng Yang14394422008-04-28 12:24:45 +08001840}
1841
Avi Kivity6aa8b732006-12-10 02:21:36 -08001842static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1843{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001844 struct vcpu_vmx *vmx = to_vmx(vcpu);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001845 unsigned long hw_cr0;
1846
1847 if (enable_unrestricted_guest)
1848 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
1849 | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
1850 else
1851 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
Sheng Yang14394422008-04-28 12:24:45 +08001852
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001853 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001854 enter_pmode(vcpu);
1855
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001856 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001857 enter_rmode(vcpu);
1858
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001859#ifdef CONFIG_X86_64
Avi Kivityf6801df2010-01-21 15:31:50 +02001860 if (vcpu->arch.efer & EFER_LME) {
Rusty Russell707d92f2007-07-17 23:19:08 +10001861 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001862 enter_lmode(vcpu);
Rusty Russell707d92f2007-07-17 23:19:08 +10001863 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001864 exit_lmode(vcpu);
1865 }
1866#endif
1867
Avi Kivity089d0342009-03-23 18:26:32 +02001868 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +08001869 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
1870
Avi Kivity02daab22009-12-30 12:40:26 +02001871 if (!vcpu->fpu_active)
Avi Kivity81231c62010-01-24 16:26:40 +02001872 hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
Avi Kivity02daab22009-12-30 12:40:26 +02001873
Avi Kivity6aa8b732006-12-10 02:21:36 -08001874 vmcs_writel(CR0_READ_SHADOW, cr0);
Sheng Yang14394422008-04-28 12:24:45 +08001875 vmcs_writel(GUEST_CR0, hw_cr0);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001876 vcpu->arch.cr0 = cr0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001877}
1878
Sheng Yang14394422008-04-28 12:24:45 +08001879static u64 construct_eptp(unsigned long root_hpa)
1880{
1881 u64 eptp;
1882
1883 /* TODO write the value reading from MSR */
1884 eptp = VMX_EPT_DEFAULT_MT |
1885 VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
1886 eptp |= (root_hpa & PAGE_MASK);
1887
1888 return eptp;
1889}
1890
Avi Kivity6aa8b732006-12-10 02:21:36 -08001891static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1892{
Sheng Yang14394422008-04-28 12:24:45 +08001893 unsigned long guest_cr3;
1894 u64 eptp;
1895
1896 guest_cr3 = cr3;
Avi Kivity089d0342009-03-23 18:26:32 +02001897 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +08001898 eptp = construct_eptp(cr3);
1899 vmcs_write64(EPT_POINTER, eptp);
Sheng Yang14394422008-04-28 12:24:45 +08001900 guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
Sheng Yangb927a3c2009-07-21 10:42:48 +08001901 vcpu->kvm->arch.ept_identity_map_addr;
Marcelo Tosatti7c93be42009-10-26 16:48:33 -02001902 ept_load_pdptrs(vcpu);
Sheng Yang14394422008-04-28 12:24:45 +08001903 }
1904
Sheng Yang2384d2b2008-01-17 15:14:33 +08001905 vmx_flush_tlb(vcpu);
Sheng Yang14394422008-04-28 12:24:45 +08001906 vmcs_writel(GUEST_CR3, guest_cr3);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001907}
1908
1909static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1910{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001911 unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
Sheng Yang14394422008-04-28 12:24:45 +08001912 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
1913
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001914 vcpu->arch.cr4 = cr4;
Avi Kivitybc230082009-12-08 12:14:42 +02001915 if (enable_ept) {
1916 if (!is_paging(vcpu)) {
1917 hw_cr4 &= ~X86_CR4_PAE;
1918 hw_cr4 |= X86_CR4_PSE;
1919 } else if (!(cr4 & X86_CR4_PAE)) {
1920 hw_cr4 &= ~X86_CR4_PAE;
1921 }
1922 }
Sheng Yang14394422008-04-28 12:24:45 +08001923
1924 vmcs_writel(CR4_READ_SHADOW, cr4);
1925 vmcs_writel(GUEST_CR4, hw_cr4);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001926}
1927
Avi Kivity6aa8b732006-12-10 02:21:36 -08001928static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1929{
1930 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1931
1932 return vmcs_readl(sf->base);
1933}
1934
1935static void vmx_get_segment(struct kvm_vcpu *vcpu,
1936 struct kvm_segment *var, int seg)
1937{
1938 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1939 u32 ar;
1940
1941 var->base = vmcs_readl(sf->base);
1942 var->limit = vmcs_read32(sf->limit);
1943 var->selector = vmcs_read16(sf->selector);
1944 ar = vmcs_read32(sf->ar_bytes);
Avi Kivity9fd4a3b2009-01-04 23:43:42 +02001945 if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001946 ar = 0;
1947 var->type = ar & 15;
1948 var->s = (ar >> 4) & 1;
1949 var->dpl = (ar >> 5) & 3;
1950 var->present = (ar >> 7) & 1;
1951 var->avl = (ar >> 12) & 1;
1952 var->l = (ar >> 13) & 1;
1953 var->db = (ar >> 14) & 1;
1954 var->g = (ar >> 15) & 1;
1955 var->unusable = (ar >> 16) & 1;
1956}
1957
Izik Eidus2e4d2652008-03-24 19:38:34 +02001958static int vmx_get_cpl(struct kvm_vcpu *vcpu)
1959{
Avi Kivity3eeb3282010-01-21 15:31:48 +02001960 if (!is_protmode(vcpu))
Izik Eidus2e4d2652008-03-24 19:38:34 +02001961 return 0;
1962
1963 if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
1964 return 3;
1965
Avi Kivityeab4b8a2009-08-04 15:02:54 +03001966 return vmcs_read16(GUEST_CS_SELECTOR) & 3;
Izik Eidus2e4d2652008-03-24 19:38:34 +02001967}
1968
Avi Kivity653e3102007-05-07 10:55:37 +03001969static u32 vmx_segment_access_rights(struct kvm_segment *var)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001970{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001971 u32 ar;
1972
Avi Kivity653e3102007-05-07 10:55:37 +03001973 if (var->unusable)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001974 ar = 1 << 16;
1975 else {
1976 ar = var->type & 15;
1977 ar |= (var->s & 1) << 4;
1978 ar |= (var->dpl & 3) << 5;
1979 ar |= (var->present & 1) << 7;
1980 ar |= (var->avl & 1) << 12;
1981 ar |= (var->l & 1) << 13;
1982 ar |= (var->db & 1) << 14;
1983 ar |= (var->g & 1) << 15;
1984 }
Uri Lublinf7fbf1f2006-12-13 00:34:00 -08001985 if (ar == 0) /* a 0 value means unusable */
1986 ar = AR_UNUSABLE_MASK;
Avi Kivity653e3102007-05-07 10:55:37 +03001987
1988 return ar;
1989}
1990
1991static void vmx_set_segment(struct kvm_vcpu *vcpu,
1992 struct kvm_segment *var, int seg)
1993{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001994 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity653e3102007-05-07 10:55:37 +03001995 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1996 u32 ar;
1997
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001998 if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
1999 vmx->rmode.tr.selector = var->selector;
2000 vmx->rmode.tr.base = var->base;
2001 vmx->rmode.tr.limit = var->limit;
2002 vmx->rmode.tr.ar = vmx_segment_access_rights(var);
Avi Kivity653e3102007-05-07 10:55:37 +03002003 return;
2004 }
2005 vmcs_writel(sf->base, var->base);
2006 vmcs_write32(sf->limit, var->limit);
2007 vmcs_write16(sf->selector, var->selector);
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002008 if (vmx->rmode.vm86_active && var->s) {
Avi Kivity653e3102007-05-07 10:55:37 +03002009 /*
2010 * Hack real-mode segments into vm86 compatibility.
2011 */
2012 if (var->base == 0xffff0000 && var->selector == 0xf000)
2013 vmcs_writel(sf->base, 0xf0000);
2014 ar = 0xf3;
2015 } else
2016 ar = vmx_segment_access_rights(var);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002017
2018 /*
2019 * Fix the "Accessed" bit in AR field of segment registers for older
2020 * qemu binaries.
2021 * IA32 arch specifies that at the time of processor reset the
2022 * "Accessed" bit in the AR field of segment registers is 1. And qemu
2023 * is setting it to 0 in the usedland code. This causes invalid guest
2024 * state vmexit when "unrestricted guest" mode is turned on.
2025 * Fix for this setup issue in cpu_reset is being pushed in the qemu
2026 * tree. Newer qemu binaries with that qemu fix would not need this
2027 * kvm hack.
2028 */
2029 if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
2030 ar |= 0x1; /* Accessed */
2031
Avi Kivity6aa8b732006-12-10 02:21:36 -08002032 vmcs_write32(sf->ar_bytes, ar);
2033}
2034
Avi Kivity6aa8b732006-12-10 02:21:36 -08002035static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
2036{
2037 u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
2038
2039 *db = (ar >> 14) & 1;
2040 *l = (ar >> 13) & 1;
2041}
2042
Gleb Natapov89a27f42010-02-16 10:51:48 +02002043static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002044{
Gleb Natapov89a27f42010-02-16 10:51:48 +02002045 dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
2046 dt->address = vmcs_readl(GUEST_IDTR_BASE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002047}
2048
Gleb Natapov89a27f42010-02-16 10:51:48 +02002049static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002050{
Gleb Natapov89a27f42010-02-16 10:51:48 +02002051 vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
2052 vmcs_writel(GUEST_IDTR_BASE, dt->address);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002053}
2054
Gleb Natapov89a27f42010-02-16 10:51:48 +02002055static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002056{
Gleb Natapov89a27f42010-02-16 10:51:48 +02002057 dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
2058 dt->address = vmcs_readl(GUEST_GDTR_BASE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002059}
2060
Gleb Natapov89a27f42010-02-16 10:51:48 +02002061static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002062{
Gleb Natapov89a27f42010-02-16 10:51:48 +02002063 vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
2064 vmcs_writel(GUEST_GDTR_BASE, dt->address);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002065}
2066
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002067static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
2068{
2069 struct kvm_segment var;
2070 u32 ar;
2071
2072 vmx_get_segment(vcpu, &var, seg);
2073 ar = vmx_segment_access_rights(&var);
2074
2075 if (var.base != (var.selector << 4))
2076 return false;
2077 if (var.limit != 0xffff)
2078 return false;
2079 if (ar != 0xf3)
2080 return false;
2081
2082 return true;
2083}
2084
2085static bool code_segment_valid(struct kvm_vcpu *vcpu)
2086{
2087 struct kvm_segment cs;
2088 unsigned int cs_rpl;
2089
2090 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
2091 cs_rpl = cs.selector & SELECTOR_RPL_MASK;
2092
Avi Kivity1872a3f2009-01-04 23:26:52 +02002093 if (cs.unusable)
2094 return false;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002095 if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
2096 return false;
2097 if (!cs.s)
2098 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02002099 if (cs.type & AR_TYPE_WRITEABLE_MASK) {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002100 if (cs.dpl > cs_rpl)
2101 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02002102 } else {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002103 if (cs.dpl != cs_rpl)
2104 return false;
2105 }
2106 if (!cs.present)
2107 return false;
2108
2109 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
2110 return true;
2111}
2112
2113static bool stack_segment_valid(struct kvm_vcpu *vcpu)
2114{
2115 struct kvm_segment ss;
2116 unsigned int ss_rpl;
2117
2118 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2119 ss_rpl = ss.selector & SELECTOR_RPL_MASK;
2120
Avi Kivity1872a3f2009-01-04 23:26:52 +02002121 if (ss.unusable)
2122 return true;
2123 if (ss.type != 3 && ss.type != 7)
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002124 return false;
2125 if (!ss.s)
2126 return false;
2127 if (ss.dpl != ss_rpl) /* DPL != RPL */
2128 return false;
2129 if (!ss.present)
2130 return false;
2131
2132 return true;
2133}
2134
2135static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
2136{
2137 struct kvm_segment var;
2138 unsigned int rpl;
2139
2140 vmx_get_segment(vcpu, &var, seg);
2141 rpl = var.selector & SELECTOR_RPL_MASK;
2142
Avi Kivity1872a3f2009-01-04 23:26:52 +02002143 if (var.unusable)
2144 return true;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002145 if (!var.s)
2146 return false;
2147 if (!var.present)
2148 return false;
2149 if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
2150 if (var.dpl < rpl) /* DPL < RPL */
2151 return false;
2152 }
2153
2154 /* TODO: Add other members to kvm_segment_field to allow checking for other access
2155 * rights flags
2156 */
2157 return true;
2158}
2159
2160static bool tr_valid(struct kvm_vcpu *vcpu)
2161{
2162 struct kvm_segment tr;
2163
2164 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
2165
Avi Kivity1872a3f2009-01-04 23:26:52 +02002166 if (tr.unusable)
2167 return false;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002168 if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */
2169 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02002170 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002171 return false;
2172 if (!tr.present)
2173 return false;
2174
2175 return true;
2176}
2177
2178static bool ldtr_valid(struct kvm_vcpu *vcpu)
2179{
2180 struct kvm_segment ldtr;
2181
2182 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
2183
Avi Kivity1872a3f2009-01-04 23:26:52 +02002184 if (ldtr.unusable)
2185 return true;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002186 if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */
2187 return false;
2188 if (ldtr.type != 2)
2189 return false;
2190 if (!ldtr.present)
2191 return false;
2192
2193 return true;
2194}
2195
2196static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
2197{
2198 struct kvm_segment cs, ss;
2199
2200 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
2201 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2202
2203 return ((cs.selector & SELECTOR_RPL_MASK) ==
2204 (ss.selector & SELECTOR_RPL_MASK));
2205}
2206
2207/*
2208 * Check if guest state is valid. Returns true if valid, false if
2209 * not.
2210 * We assume that registers are always usable
2211 */
2212static bool guest_state_valid(struct kvm_vcpu *vcpu)
2213{
2214 /* real mode guest state checks */
Avi Kivity3eeb3282010-01-21 15:31:48 +02002215 if (!is_protmode(vcpu)) {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002216 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
2217 return false;
2218 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
2219 return false;
2220 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
2221 return false;
2222 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
2223 return false;
2224 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
2225 return false;
2226 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
2227 return false;
2228 } else {
2229 /* protected mode guest state checks */
2230 if (!cs_ss_rpl_check(vcpu))
2231 return false;
2232 if (!code_segment_valid(vcpu))
2233 return false;
2234 if (!stack_segment_valid(vcpu))
2235 return false;
2236 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
2237 return false;
2238 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
2239 return false;
2240 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
2241 return false;
2242 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
2243 return false;
2244 if (!tr_valid(vcpu))
2245 return false;
2246 if (!ldtr_valid(vcpu))
2247 return false;
2248 }
2249 /* TODO:
2250 * - Add checks on RIP
2251 * - Add checks on RFLAGS
2252 */
2253
2254 return true;
2255}
2256
Mike Dayd77c26f2007-10-08 09:02:08 -04002257static int init_rmode_tss(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002258{
Avi Kivity6aa8b732006-12-10 02:21:36 -08002259 gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
Izik Eidus195aefd2007-10-01 22:14:18 +02002260 u16 data = 0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002261 int ret = 0;
Izik Eidus195aefd2007-10-01 22:14:18 +02002262 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002263
Izik Eidus195aefd2007-10-01 22:14:18 +02002264 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2265 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002266 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002267 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
Sheng Yang464d17c2008-08-13 14:10:33 +08002268 r = kvm_write_guest_page(kvm, fn++, &data,
2269 TSS_IOPB_BASE_OFFSET, sizeof(u16));
Izik Eidus195aefd2007-10-01 22:14:18 +02002270 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002271 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002272 r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
2273 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002274 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002275 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2276 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002277 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002278 data = ~0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002279 r = kvm_write_guest_page(kvm, fn, &data,
2280 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
2281 sizeof(u8));
Izik Eidus195aefd2007-10-01 22:14:18 +02002282 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002283 goto out;
2284
2285 ret = 1;
2286out:
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002287 return ret;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002288}
2289
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002290static int init_rmode_identity_map(struct kvm *kvm)
2291{
2292 int i, r, ret;
2293 pfn_t identity_map_pfn;
2294 u32 tmp;
2295
Avi Kivity089d0342009-03-23 18:26:32 +02002296 if (!enable_ept)
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002297 return 1;
2298 if (unlikely(!kvm->arch.ept_identity_pagetable)) {
2299 printk(KERN_ERR "EPT: identity-mapping pagetable "
2300 "haven't been allocated!\n");
2301 return 0;
2302 }
2303 if (likely(kvm->arch.ept_identity_pagetable_done))
2304 return 1;
2305 ret = 0;
Sheng Yangb927a3c2009-07-21 10:42:48 +08002306 identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002307 r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
2308 if (r < 0)
2309 goto out;
2310 /* Set up identity-mapping pagetable for EPT in real mode */
2311 for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
2312 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
2313 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
2314 r = kvm_write_guest_page(kvm, identity_map_pfn,
2315 &tmp, i * sizeof(tmp), sizeof(tmp));
2316 if (r < 0)
2317 goto out;
2318 }
2319 kvm->arch.ept_identity_pagetable_done = true;
2320 ret = 1;
2321out:
2322 return ret;
2323}
2324
Avi Kivity6aa8b732006-12-10 02:21:36 -08002325static void seg_setup(int seg)
2326{
2327 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002328 unsigned int ar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002329
2330 vmcs_write16(sf->selector, 0);
2331 vmcs_writel(sf->base, 0);
2332 vmcs_write32(sf->limit, 0xffff);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002333 if (enable_unrestricted_guest) {
2334 ar = 0x93;
2335 if (seg == VCPU_SREG_CS)
2336 ar |= 0x08; /* code segment */
2337 } else
2338 ar = 0xf3;
2339
2340 vmcs_write32(sf->ar_bytes, ar);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002341}
2342
Sheng Yangf78e0e22007-10-29 09:40:42 +08002343static int alloc_apic_access_page(struct kvm *kvm)
2344{
2345 struct kvm_userspace_memory_region kvm_userspace_mem;
2346 int r = 0;
2347
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002348 mutex_lock(&kvm->slots_lock);
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002349 if (kvm->arch.apic_access_page)
Sheng Yangf78e0e22007-10-29 09:40:42 +08002350 goto out;
2351 kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
2352 kvm_userspace_mem.flags = 0;
2353 kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
2354 kvm_userspace_mem.memory_size = PAGE_SIZE;
2355 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2356 if (r)
2357 goto out;
Izik Eidus72dc67a2008-02-10 18:04:15 +02002358
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002359 kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002360out:
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002361 mutex_unlock(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002362 return r;
2363}
2364
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002365static int alloc_identity_pagetable(struct kvm *kvm)
2366{
2367 struct kvm_userspace_memory_region kvm_userspace_mem;
2368 int r = 0;
2369
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002370 mutex_lock(&kvm->slots_lock);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002371 if (kvm->arch.ept_identity_pagetable)
2372 goto out;
2373 kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
2374 kvm_userspace_mem.flags = 0;
Sheng Yangb927a3c2009-07-21 10:42:48 +08002375 kvm_userspace_mem.guest_phys_addr =
2376 kvm->arch.ept_identity_map_addr;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002377 kvm_userspace_mem.memory_size = PAGE_SIZE;
2378 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2379 if (r)
2380 goto out;
2381
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002382 kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
Sheng Yangb927a3c2009-07-21 10:42:48 +08002383 kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002384out:
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002385 mutex_unlock(&kvm->slots_lock);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002386 return r;
2387}
2388
Sheng Yang2384d2b2008-01-17 15:14:33 +08002389static void allocate_vpid(struct vcpu_vmx *vmx)
2390{
2391 int vpid;
2392
2393 vmx->vpid = 0;
Avi Kivity919818a2009-03-23 18:01:29 +02002394 if (!enable_vpid)
Sheng Yang2384d2b2008-01-17 15:14:33 +08002395 return;
2396 spin_lock(&vmx_vpid_lock);
2397 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
2398 if (vpid < VMX_NR_VPIDS) {
2399 vmx->vpid = vpid;
2400 __set_bit(vpid, vmx_vpid_bitmap);
2401 }
2402 spin_unlock(&vmx_vpid_lock);
2403}
2404
Lai Jiangshancdbecfc2010-04-17 16:41:47 +08002405static void free_vpid(struct vcpu_vmx *vmx)
2406{
2407 if (!enable_vpid)
2408 return;
2409 spin_lock(&vmx_vpid_lock);
2410 if (vmx->vpid != 0)
2411 __clear_bit(vmx->vpid, vmx_vpid_bitmap);
2412 spin_unlock(&vmx_vpid_lock);
2413}
2414
Avi Kivity58972972009-02-24 22:26:47 +02002415static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
Sheng Yang25c5f222008-03-28 13:18:56 +08002416{
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002417 int f = sizeof(unsigned long);
Sheng Yang25c5f222008-03-28 13:18:56 +08002418
2419 if (!cpu_has_vmx_msr_bitmap())
2420 return;
2421
2422 /*
2423 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
2424 * have the write-low and read-high bitmap offsets the wrong way round.
2425 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
2426 */
Sheng Yang25c5f222008-03-28 13:18:56 +08002427 if (msr <= 0x1fff) {
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002428 __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
2429 __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
Sheng Yang25c5f222008-03-28 13:18:56 +08002430 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2431 msr &= 0x1fff;
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002432 __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
2433 __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
Sheng Yang25c5f222008-03-28 13:18:56 +08002434 }
Sheng Yang25c5f222008-03-28 13:18:56 +08002435}
2436
Avi Kivity58972972009-02-24 22:26:47 +02002437static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
2438{
2439 if (!longmode_only)
2440 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
2441 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
2442}
2443
Avi Kivity6aa8b732006-12-10 02:21:36 -08002444/*
2445 * Sets up the vmcs for emulated real mode.
2446 */
Rusty Russell8b9cf982007-07-30 16:31:43 +10002447static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002448{
Sheng Yang468d4722008-10-09 16:01:55 +08002449 u32 host_sysenter_cs, msr_low, msr_high;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002450 u32 junk;
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01002451 u64 host_pat, tsc_this, tsc_base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002452 unsigned long a;
Gleb Natapov89a27f42010-02-16 10:51:48 +02002453 struct desc_ptr dt;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002454 int i;
Avi Kivitycd2276a2007-05-14 20:41:13 +03002455 unsigned long kvm_vmx_return;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002456 u32 exec_control;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002457
Avi Kivity6aa8b732006-12-10 02:21:36 -08002458 /* I/O */
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002459 vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
2460 vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002461
Sheng Yang25c5f222008-03-28 13:18:56 +08002462 if (cpu_has_vmx_msr_bitmap())
Avi Kivity58972972009-02-24 22:26:47 +02002463 vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
Sheng Yang25c5f222008-03-28 13:18:56 +08002464
Avi Kivity6aa8b732006-12-10 02:21:36 -08002465 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
2466
Avi Kivity6aa8b732006-12-10 02:21:36 -08002467 /* Control */
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002468 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
2469 vmcs_config.pin_based_exec_ctrl);
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002470
2471 exec_control = vmcs_config.cpu_based_exec_ctrl;
2472 if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
2473 exec_control &= ~CPU_BASED_TPR_SHADOW;
2474#ifdef CONFIG_X86_64
2475 exec_control |= CPU_BASED_CR8_STORE_EXITING |
2476 CPU_BASED_CR8_LOAD_EXITING;
2477#endif
2478 }
Avi Kivity089d0342009-03-23 18:26:32 +02002479 if (!enable_ept)
Sheng Yangd56f5462008-04-25 10:13:16 +08002480 exec_control |= CPU_BASED_CR3_STORE_EXITING |
Marcelo Tosatti83dbc832008-10-07 17:01:27 -03002481 CPU_BASED_CR3_LOAD_EXITING |
2482 CPU_BASED_INVLPG_EXITING;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002483 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002484
Sheng Yang83ff3b92007-11-21 14:33:25 +08002485 if (cpu_has_secondary_exec_ctrls()) {
2486 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
2487 if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2488 exec_control &=
2489 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
Sheng Yang2384d2b2008-01-17 15:14:33 +08002490 if (vmx->vpid == 0)
2491 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
Sheng Yang046d8712009-11-27 16:46:26 +08002492 if (!enable_ept) {
Sheng Yangd56f5462008-04-25 10:13:16 +08002493 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
Sheng Yang046d8712009-11-27 16:46:26 +08002494 enable_unrestricted_guest = 0;
2495 }
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002496 if (!enable_unrestricted_guest)
2497 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08002498 if (!ple_gap)
2499 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
Sheng Yang83ff3b92007-11-21 14:33:25 +08002500 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
2501 }
Sheng Yangf78e0e22007-10-29 09:40:42 +08002502
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08002503 if (ple_gap) {
2504 vmcs_write32(PLE_GAP, ple_gap);
2505 vmcs_write32(PLE_WINDOW, ple_window);
2506 }
2507
Avi Kivityc7addb92007-09-16 18:58:32 +02002508 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
2509 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002510 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
2511
Avi Kivity1c11e712010-05-03 16:05:44 +03002512 vmcs_writel(HOST_CR0, read_cr0() | X86_CR0_TS); /* 22.2.3 */
Avi Kivity6aa8b732006-12-10 02:21:36 -08002513 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
2514 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
2515
2516 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
2517 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
2518 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +03002519 vmcs_write16(HOST_FS_SELECTOR, kvm_read_fs()); /* 22.2.4 */
2520 vmcs_write16(HOST_GS_SELECTOR, kvm_read_gs()); /* 22.2.4 */
Avi Kivity6aa8b732006-12-10 02:21:36 -08002521 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
Avi Kivity05b3e0c2006-12-13 00:33:45 -08002522#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08002523 rdmsrl(MSR_FS_BASE, a);
2524 vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
2525 rdmsrl(MSR_GS_BASE, a);
2526 vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
2527#else
2528 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
2529 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
2530#endif
2531
2532 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
2533
Wei Yongjunec687982010-03-05 12:11:48 +08002534 native_store_idt(&dt);
Gleb Natapov89a27f42010-02-16 10:51:48 +02002535 vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */
Avi Kivity6aa8b732006-12-10 02:21:36 -08002536
Mike Dayd77c26f2007-10-08 09:02:08 -04002537 asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
Avi Kivitycd2276a2007-05-14 20:41:13 +03002538 vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
Eddie Dong2cc51562007-05-21 07:28:09 +03002539 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
2540 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
Avi Kivity61d2ef22010-04-28 16:40:38 +03002541 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
Eddie Dong2cc51562007-05-21 07:28:09 +03002542 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
Avi Kivity61d2ef22010-04-28 16:40:38 +03002543 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002544
2545 rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
2546 vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
2547 rdmsrl(MSR_IA32_SYSENTER_ESP, a);
2548 vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */
2549 rdmsrl(MSR_IA32_SYSENTER_EIP, a);
2550 vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */
2551
Sheng Yang468d4722008-10-09 16:01:55 +08002552 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
2553 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2554 host_pat = msr_low | ((u64) msr_high << 32);
2555 vmcs_write64(HOST_IA32_PAT, host_pat);
2556 }
2557 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2558 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2559 host_pat = msr_low | ((u64) msr_high << 32);
2560 /* Write the default value follow host pat */
2561 vmcs_write64(GUEST_IA32_PAT, host_pat);
2562 /* Keep arch.pat sync with GUEST_IA32_PAT */
2563 vmx->vcpu.arch.pat = host_pat;
2564 }
2565
Avi Kivity6aa8b732006-12-10 02:21:36 -08002566 for (i = 0; i < NR_VMX_MSR; ++i) {
2567 u32 index = vmx_msr_index[i];
2568 u32 data_low, data_high;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002569 int j = vmx->nmsrs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002570
2571 if (rdmsr_safe(index, &data_low, &data_high) < 0)
2572 continue;
Avi Kivity432bd6c2007-01-31 23:48:13 -08002573 if (wrmsr_safe(index, data_low, data_high) < 0)
2574 continue;
Avi Kivity26bb0982009-09-07 11:14:12 +03002575 vmx->guest_msrs[j].index = i;
2576 vmx->guest_msrs[j].data = 0;
Avi Kivityd5696722009-12-02 12:28:47 +02002577 vmx->guest_msrs[j].mask = -1ull;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002578 ++vmx->nmsrs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002579 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002580
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002581 vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002582
2583 /* 22.2.1, 20.8.1 */
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002584 vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
2585
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002586 vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
Avi Kivity4c386092009-12-07 12:26:18 +02002587 vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
Avi Kivityce03e4f2009-12-07 12:29:14 +02002588 if (enable_ept)
2589 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
Avi Kivity4c386092009-12-07 12:26:18 +02002590 vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002591
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01002592 tsc_base = vmx->vcpu.kvm->arch.vm_init_tsc;
2593 rdtscll(tsc_this);
2594 if (tsc_this < vmx->vcpu.kvm->arch.vm_init_tsc)
2595 tsc_base = tsc_this;
2596
2597 guest_write_tsc(0, tsc_base);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002598
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002599 return 0;
2600}
2601
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002602static int init_rmode(struct kvm *kvm)
2603{
2604 if (!init_rmode_tss(kvm))
2605 return 0;
2606 if (!init_rmode_identity_map(kvm))
2607 return 0;
2608 return 1;
2609}
2610
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002611static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
2612{
2613 struct vcpu_vmx *vmx = to_vmx(vcpu);
2614 u64 msr;
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002615 int ret, idx;
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002616
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002617 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002618 idx = srcu_read_lock(&vcpu->kvm->srcu);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002619 if (!init_rmode(vmx->vcpu.kvm)) {
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002620 ret = -ENOMEM;
2621 goto out;
2622 }
2623
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002624 vmx->rmode.vm86_active = 0;
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002625
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002626 vmx->soft_vnmi_blocked = 0;
2627
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002628 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02002629 kvm_set_cr8(&vmx->vcpu, 0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002630 msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002631 if (kvm_vcpu_is_bsp(&vmx->vcpu))
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002632 msr |= MSR_IA32_APICBASE_BSP;
2633 kvm_set_apic_base(&vmx->vcpu, msr);
2634
2635 fx_init(&vmx->vcpu);
2636
Avi Kivity5706be02008-08-20 15:07:31 +03002637 seg_setup(VCPU_SREG_CS);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002638 /*
2639 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
2640 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh.
2641 */
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002642 if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002643 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
2644 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
2645 } else {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002646 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
2647 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002648 }
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002649
2650 seg_setup(VCPU_SREG_DS);
2651 seg_setup(VCPU_SREG_ES);
2652 seg_setup(VCPU_SREG_FS);
2653 seg_setup(VCPU_SREG_GS);
2654 seg_setup(VCPU_SREG_SS);
2655
2656 vmcs_write16(GUEST_TR_SELECTOR, 0);
2657 vmcs_writel(GUEST_TR_BASE, 0);
2658 vmcs_write32(GUEST_TR_LIMIT, 0xffff);
2659 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2660
2661 vmcs_write16(GUEST_LDTR_SELECTOR, 0);
2662 vmcs_writel(GUEST_LDTR_BASE, 0);
2663 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
2664 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
2665
2666 vmcs_write32(GUEST_SYSENTER_CS, 0);
2667 vmcs_writel(GUEST_SYSENTER_ESP, 0);
2668 vmcs_writel(GUEST_SYSENTER_EIP, 0);
2669
2670 vmcs_writel(GUEST_RFLAGS, 0x02);
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002671 if (kvm_vcpu_is_bsp(&vmx->vcpu))
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002672 kvm_rip_write(vcpu, 0xfff0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002673 else
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002674 kvm_rip_write(vcpu, 0);
2675 kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002676
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002677 vmcs_writel(GUEST_DR7, 0x400);
2678
2679 vmcs_writel(GUEST_GDTR_BASE, 0);
2680 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
2681
2682 vmcs_writel(GUEST_IDTR_BASE, 0);
2683 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
2684
2685 vmcs_write32(GUEST_ACTIVITY_STATE, 0);
2686 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
2687 vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
2688
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002689 /* Special registers */
2690 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
2691
2692 setup_msrs(vmx);
2693
Avi Kivity6aa8b732006-12-10 02:21:36 -08002694 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
2695
Sheng Yangf78e0e22007-10-29 09:40:42 +08002696 if (cpu_has_vmx_tpr_shadow()) {
2697 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
2698 if (vm_need_tpr_shadow(vmx->vcpu.kvm))
2699 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002700 page_to_phys(vmx->vcpu.arch.apic->regs_page));
Sheng Yangf78e0e22007-10-29 09:40:42 +08002701 vmcs_write32(TPR_THRESHOLD, 0);
2702 }
2703
2704 if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2705 vmcs_write64(APIC_ACCESS_ADDR,
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002706 page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002707
Sheng Yang2384d2b2008-01-17 15:14:33 +08002708 if (vmx->vpid != 0)
2709 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2710
Eduardo Habkostfa400522009-10-24 02:49:58 -02002711 vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
Avi Kivity4d4ec082009-12-29 18:07:30 +02002712 vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
Rusty Russell8b9cf982007-07-30 16:31:43 +10002713 vmx_set_cr4(&vmx->vcpu, 0);
Rusty Russell8b9cf982007-07-30 16:31:43 +10002714 vmx_set_efer(&vmx->vcpu, 0);
Rusty Russell8b9cf982007-07-30 16:31:43 +10002715 vmx_fpu_activate(&vmx->vcpu);
2716 update_exception_bitmap(&vmx->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002717
Sheng Yang2384d2b2008-01-17 15:14:33 +08002718 vpid_sync_vcpu_all(vmx);
2719
Marcelo Tosatti3200f402008-03-29 20:17:59 -03002720 ret = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002721
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03002722 /* HACK: Don't enable emulation on guest boot/reset */
2723 vmx->emulation_required = 0;
2724
Avi Kivity6aa8b732006-12-10 02:21:36 -08002725out:
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002726 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002727 return ret;
2728}
2729
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002730static void enable_irq_window(struct kvm_vcpu *vcpu)
2731{
2732 u32 cpu_based_vm_exec_control;
2733
2734 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2735 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
2736 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2737}
2738
2739static void enable_nmi_window(struct kvm_vcpu *vcpu)
2740{
2741 u32 cpu_based_vm_exec_control;
2742
2743 if (!cpu_has_virtual_nmis()) {
2744 enable_irq_window(vcpu);
2745 return;
2746 }
2747
2748 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2749 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
2750 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2751}
2752
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002753static void vmx_inject_irq(struct kvm_vcpu *vcpu)
Eddie Dong85f455f2007-07-06 12:20:49 +03002754{
Avi Kivity9c8cba32007-11-22 11:42:59 +02002755 struct vcpu_vmx *vmx = to_vmx(vcpu);
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002756 uint32_t intr;
2757 int irq = vcpu->arch.interrupt.nr;
Avi Kivity9c8cba32007-11-22 11:42:59 +02002758
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002759 trace_kvm_inj_virq(irq);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04002760
Avi Kivityfa89a812008-09-01 15:57:51 +03002761 ++vcpu->stat.irq_injections;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002762 if (vmx->rmode.vm86_active) {
Avi Kivity9c8cba32007-11-22 11:42:59 +02002763 vmx->rmode.irq.pending = true;
2764 vmx->rmode.irq.vector = irq;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002765 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
Gleb Natapovae0bb3e2009-05-19 11:07:10 +03002766 if (vcpu->arch.interrupt.soft)
2767 vmx->rmode.irq.rip +=
2768 vmx->vcpu.arch.event_exit_inst_len;
Avi Kivity9c5623e2007-11-08 18:19:20 +02002769 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2770 irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);
2771 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002772 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
Eddie Dong85f455f2007-07-06 12:20:49 +03002773 return;
2774 }
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002775 intr = irq | INTR_INFO_VALID_MASK;
2776 if (vcpu->arch.interrupt.soft) {
2777 intr |= INTR_TYPE_SOFT_INTR;
2778 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2779 vmx->vcpu.arch.event_exit_inst_len);
2780 } else
2781 intr |= INTR_TYPE_EXT_INTR;
2782 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
Eddie Dong85f455f2007-07-06 12:20:49 +03002783}
2784
Sheng Yangf08864b2008-05-15 18:23:25 +08002785static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
2786{
Jan Kiszka66a5a342008-09-26 09:30:51 +02002787 struct vcpu_vmx *vmx = to_vmx(vcpu);
2788
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002789 if (!cpu_has_virtual_nmis()) {
2790 /*
2791 * Tracking the NMI-blocked state in software is built upon
2792 * finding the next open IRQ window. This, in turn, depends on
2793 * well-behaving guests: They have to keep IRQs disabled at
2794 * least as long as the NMI handler runs. Otherwise we may
2795 * cause NMI nesting, maybe breaking the guest. But as this is
2796 * highly unlikely, we can live with the residual risk.
2797 */
2798 vmx->soft_vnmi_blocked = 1;
2799 vmx->vnmi_blocked_time = 0;
2800 }
2801
Jan Kiszka487b3912008-09-26 09:30:56 +02002802 ++vcpu->stat.nmi_injections;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002803 if (vmx->rmode.vm86_active) {
Jan Kiszka66a5a342008-09-26 09:30:51 +02002804 vmx->rmode.irq.pending = true;
2805 vmx->rmode.irq.vector = NMI_VECTOR;
2806 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
2807 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2808 NMI_VECTOR | INTR_TYPE_SOFT_INTR |
2809 INTR_INFO_VALID_MASK);
2810 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
2811 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
2812 return;
2813 }
Sheng Yangf08864b2008-05-15 18:23:25 +08002814 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2815 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
Sheng Yangf08864b2008-05-15 18:23:25 +08002816}
2817
Gleb Natapovc4282df2009-04-21 17:45:07 +03002818static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
Jan Kiszka33f089c2008-09-26 09:30:49 +02002819{
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002820 if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
Gleb Natapovc4282df2009-04-21 17:45:07 +03002821 return 0;
Jan Kiszka33f089c2008-09-26 09:30:49 +02002822
Gleb Natapovc4282df2009-04-21 17:45:07 +03002823 return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
Jan Kiszkaf8c5fae2010-05-11 15:16:46 +02002824 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_NMI));
Jan Kiszka33f089c2008-09-26 09:30:49 +02002825}
2826
Jan Kiszka3cfc3092009-11-12 01:04:25 +01002827static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
2828{
2829 if (!cpu_has_virtual_nmis())
2830 return to_vmx(vcpu)->soft_vnmi_blocked;
Avi Kivityc332c832010-05-04 12:24:12 +03002831 return vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01002832}
2833
2834static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
2835{
2836 struct vcpu_vmx *vmx = to_vmx(vcpu);
2837
2838 if (!cpu_has_virtual_nmis()) {
2839 if (vmx->soft_vnmi_blocked != masked) {
2840 vmx->soft_vnmi_blocked = masked;
2841 vmx->vnmi_blocked_time = 0;
2842 }
2843 } else {
2844 if (masked)
2845 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
2846 GUEST_INTR_STATE_NMI);
2847 else
2848 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
2849 GUEST_INTR_STATE_NMI);
2850 }
2851}
2852
Gleb Natapov78646122009-03-23 12:12:11 +02002853static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
2854{
Gleb Natapovc4282df2009-04-21 17:45:07 +03002855 return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
2856 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2857 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
Gleb Natapov78646122009-03-23 12:12:11 +02002858}
2859
Izik Eiduscbc94022007-10-25 00:29:55 +02002860static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
2861{
2862 int ret;
2863 struct kvm_userspace_memory_region tss_mem = {
Sheng Yang6fe63972008-10-16 17:30:58 +08002864 .slot = TSS_PRIVATE_MEMSLOT,
Izik Eiduscbc94022007-10-25 00:29:55 +02002865 .guest_phys_addr = addr,
2866 .memory_size = PAGE_SIZE * 3,
2867 .flags = 0,
2868 };
2869
2870 ret = kvm_set_memory_region(kvm, &tss_mem, 0);
2871 if (ret)
2872 return ret;
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002873 kvm->arch.tss_addr = addr;
Izik Eiduscbc94022007-10-25 00:29:55 +02002874 return 0;
2875}
2876
Avi Kivity6aa8b732006-12-10 02:21:36 -08002877static int handle_rmode_exception(struct kvm_vcpu *vcpu,
2878 int vec, u32 err_code)
2879{
Nitin A Kambleb3f37702007-05-17 15:50:34 +03002880 /*
2881 * Instruction with address size override prefix opcode 0x67
2882 * Cause the #SS fault with 0 error code in VM86 mode.
2883 */
2884 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
Avi Kivity851ba692009-08-24 11:10:17 +03002885 if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002886 return 1;
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002887 /*
2888 * Forward all other exceptions that are valid in real mode.
2889 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
2890 * the required debugging infrastructure rework.
2891 */
2892 switch (vec) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002893 case DB_VECTOR:
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002894 if (vcpu->guest_debug &
2895 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
2896 return 0;
2897 kvm_queue_exception(vcpu, vec);
2898 return 1;
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002899 case BP_VECTOR:
Jan Kiszkac573cd22010-02-23 17:47:53 +01002900 /*
2901 * Update instruction length as we may reinject the exception
2902 * from user space while in guest debugging mode.
2903 */
2904 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
2905 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002906 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2907 return 0;
2908 /* fall through */
2909 case DE_VECTOR:
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002910 case OF_VECTOR:
2911 case BR_VECTOR:
2912 case UD_VECTOR:
2913 case DF_VECTOR:
2914 case SS_VECTOR:
2915 case GP_VECTOR:
2916 case MF_VECTOR:
2917 kvm_queue_exception(vcpu, vec);
2918 return 1;
2919 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002920 return 0;
2921}
2922
Andi Kleena0861c02009-06-08 17:37:09 +08002923/*
2924 * Trigger machine check on the host. We assume all the MSRs are already set up
2925 * by the CPU and that we still run on the same CPU as the MCE occurred on.
2926 * We pass a fake environment to the machine check handler because we want
2927 * the guest to be always treated like user space, no matter what context
2928 * it used internally.
2929 */
2930static void kvm_machine_check(void)
2931{
2932#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
2933 struct pt_regs regs = {
2934 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
2935 .flags = X86_EFLAGS_IF,
2936 };
2937
2938 do_machine_check(&regs, 0);
2939#endif
2940}
2941
Avi Kivity851ba692009-08-24 11:10:17 +03002942static int handle_machine_check(struct kvm_vcpu *vcpu)
Andi Kleena0861c02009-06-08 17:37:09 +08002943{
2944 /* already handled by vcpu_run */
2945 return 1;
2946}
2947
Avi Kivity851ba692009-08-24 11:10:17 +03002948static int handle_exception(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002949{
Avi Kivity1155f762007-11-22 11:30:47 +02002950 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity851ba692009-08-24 11:10:17 +03002951 struct kvm_run *kvm_run = vcpu->run;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002952 u32 intr_info, ex_no, error_code;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002953 unsigned long cr2, rip, dr6;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002954 u32 vect_info;
2955 enum emulation_result er;
2956
Avi Kivity1155f762007-11-22 11:30:47 +02002957 vect_info = vmx->idt_vectoring_info;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002958 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
2959
Andi Kleena0861c02009-06-08 17:37:09 +08002960 if (is_machine_check(intr_info))
Avi Kivity851ba692009-08-24 11:10:17 +03002961 return handle_machine_check(vcpu);
Andi Kleena0861c02009-06-08 17:37:09 +08002962
Avi Kivity6aa8b732006-12-10 02:21:36 -08002963 if ((vect_info & VECTORING_INFO_VALID_MASK) &&
Avi Kivity65ac7262009-11-04 11:59:01 +02002964 !is_page_fault(intr_info)) {
2965 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2966 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
2967 vcpu->run->internal.ndata = 2;
2968 vcpu->run->internal.data[0] = vect_info;
2969 vcpu->run->internal.data[1] = intr_info;
2970 return 0;
2971 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002972
Jan Kiszkae4a41882008-09-26 09:30:46 +02002973 if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
Avi Kivity1b6269d2007-10-09 12:12:19 +02002974 return 1; /* already handled by vmx_vcpu_run() */
Anthony Liguori2ab455c2007-04-27 09:29:49 +03002975
2976 if (is_no_device(intr_info)) {
Avi Kivity5fd86fc2007-05-02 20:40:00 +03002977 vmx_fpu_activate(vcpu);
Anthony Liguori2ab455c2007-04-27 09:29:49 +03002978 return 1;
2979 }
2980
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002981 if (is_invalid_opcode(intr_info)) {
Avi Kivity851ba692009-08-24 11:10:17 +03002982 er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002983 if (er != EMULATE_DONE)
Avi Kivity7ee5d9402007-11-25 15:22:50 +02002984 kvm_queue_exception(vcpu, UD_VECTOR);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002985 return 1;
2986 }
2987
Avi Kivity6aa8b732006-12-10 02:21:36 -08002988 error_code = 0;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002989 rip = kvm_rip_read(vcpu);
Ryan Harper2e113842008-02-11 10:26:38 -06002990 if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002991 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
2992 if (is_page_fault(intr_info)) {
Sheng Yang14394422008-04-28 12:24:45 +08002993 /* EPT won't cause page fault directly */
Avi Kivity089d0342009-03-23 18:26:32 +02002994 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +08002995 BUG();
Avi Kivity6aa8b732006-12-10 02:21:36 -08002996 cr2 = vmcs_readl(EXIT_QUALIFICATION);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002997 trace_kvm_page_fault(cr2, error_code);
2998
Gleb Natapov3298b752009-05-11 13:35:46 +03002999 if (kvm_event_needs_reinjection(vcpu))
Avi Kivity577bdc42008-07-19 08:57:05 +03003000 kvm_mmu_unprotect_page_virt(vcpu, cr2);
Avi Kivity30677142007-10-28 18:48:59 +02003001 return kvm_mmu_page_fault(vcpu, cr2, error_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003002 }
3003
Avi Kivity7ffd92c2009-06-09 14:10:45 +03003004 if (vmx->rmode.vm86_active &&
Avi Kivity6aa8b732006-12-10 02:21:36 -08003005 handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
Avi Kivity72d6e5a2007-06-05 16:15:51 +03003006 error_code)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003007 if (vcpu->arch.halt_request) {
3008 vcpu->arch.halt_request = 0;
Avi Kivity72d6e5a2007-06-05 16:15:51 +03003009 return kvm_emulate_halt(vcpu);
3010 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003011 return 1;
Avi Kivity72d6e5a2007-06-05 16:15:51 +03003012 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003013
Jan Kiszkad0bfb942008-12-15 13:52:10 +01003014 ex_no = intr_info & INTR_INFO_VECTOR_MASK;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003015 switch (ex_no) {
3016 case DB_VECTOR:
3017 dr6 = vmcs_readl(EXIT_QUALIFICATION);
3018 if (!(vcpu->guest_debug &
3019 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
3020 vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
3021 kvm_queue_exception(vcpu, DB_VECTOR);
3022 return 1;
3023 }
3024 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
3025 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
3026 /* fall through */
3027 case BP_VECTOR:
Jan Kiszkac573cd22010-02-23 17:47:53 +01003028 /*
3029 * Update instruction length as we may reinject #BP from
3030 * user space while in guest debugging mode. Reading it for
3031 * #DB as well causes no harm, it is not used in that case.
3032 */
3033 vmx->vcpu.arch.event_exit_inst_len =
3034 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003035 kvm_run->exit_reason = KVM_EXIT_DEBUG;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01003036 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
3037 kvm_run->debug.arch.exception = ex_no;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003038 break;
3039 default:
Jan Kiszkad0bfb942008-12-15 13:52:10 +01003040 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
3041 kvm_run->ex.exception = ex_no;
3042 kvm_run->ex.error_code = error_code;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003043 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003044 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003045 return 0;
3046}
3047
Avi Kivity851ba692009-08-24 11:10:17 +03003048static int handle_external_interrupt(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003049{
Avi Kivity1165f5f2007-04-19 17:27:43 +03003050 ++vcpu->stat.irq_exits;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003051 return 1;
3052}
3053
Avi Kivity851ba692009-08-24 11:10:17 +03003054static int handle_triple_fault(struct kvm_vcpu *vcpu)
Avi Kivity988ad742007-02-12 00:54:36 -08003055{
Avi Kivity851ba692009-08-24 11:10:17 +03003056 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
Avi Kivity988ad742007-02-12 00:54:36 -08003057 return 0;
3058}
Avi Kivity6aa8b732006-12-10 02:21:36 -08003059
Avi Kivity851ba692009-08-24 11:10:17 +03003060static int handle_io(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003061{
He, Qingbfdaab02007-09-12 14:18:28 +08003062 unsigned long exit_qualification;
Jan Kiszka34c33d12009-02-08 13:28:15 +01003063 int size, in, string;
Avi Kivity039576c2007-03-20 12:46:50 +02003064 unsigned port;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003065
He, Qingbfdaab02007-09-12 14:18:28 +08003066 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Avi Kivity039576c2007-03-20 12:46:50 +02003067 string = (exit_qualification & 16) != 0;
Laurent Viviere70669a2007-08-05 10:36:40 +03003068 in = (exit_qualification & 8) != 0;
Laurent Viviere70669a2007-08-05 10:36:40 +03003069
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02003070 ++vcpu->stat.io_exits;
3071
3072 if (string || in)
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03003073 return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02003074
3075 port = exit_qualification >> 16;
3076 size = (exit_qualification & 7) + 1;
Guillaume Thouvenine93f36b2008-10-28 10:51:30 +01003077 skip_emulated_instruction(vcpu);
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02003078
3079 return kvm_fast_pio_out(vcpu, size, port);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003080}
3081
Ingo Molnar102d8322007-02-19 14:37:47 +02003082static void
3083vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3084{
3085 /*
3086 * Patch in the VMCALL instruction:
3087 */
3088 hypercall[0] = 0x0f;
3089 hypercall[1] = 0x01;
3090 hypercall[2] = 0xc1;
Ingo Molnar102d8322007-02-19 14:37:47 +02003091}
3092
Avi Kivity851ba692009-08-24 11:10:17 +03003093static int handle_cr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003094{
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003095 unsigned long exit_qualification, val;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003096 int cr;
3097 int reg;
3098
He, Qingbfdaab02007-09-12 14:18:28 +08003099 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003100 cr = exit_qualification & 15;
3101 reg = (exit_qualification >> 8) & 15;
3102 switch ((exit_qualification >> 4) & 3) {
3103 case 0: /* mov to cr */
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003104 val = kvm_register_read(vcpu, reg);
3105 trace_kvm_cr_write(cr, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003106 switch (cr) {
3107 case 0:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003108 kvm_set_cr0(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003109 skip_emulated_instruction(vcpu);
3110 return 1;
3111 case 3:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003112 kvm_set_cr3(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003113 skip_emulated_instruction(vcpu);
3114 return 1;
3115 case 4:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003116 kvm_set_cr4(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003117 skip_emulated_instruction(vcpu);
3118 return 1;
Gleb Natapov0a5fff192009-04-21 17:45:06 +03003119 case 8: {
3120 u8 cr8_prev = kvm_get_cr8(vcpu);
3121 u8 cr8 = kvm_register_read(vcpu, reg);
3122 kvm_set_cr8(vcpu, cr8);
3123 skip_emulated_instruction(vcpu);
3124 if (irqchip_in_kernel(vcpu->kvm))
3125 return 1;
3126 if (cr8_prev <= cr8)
3127 return 1;
Avi Kivity851ba692009-08-24 11:10:17 +03003128 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
Gleb Natapov0a5fff192009-04-21 17:45:06 +03003129 return 0;
3130 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003131 };
3132 break;
Anthony Liguori25c4c272007-04-27 09:29:21 +03003133 case 2: /* clts */
Avi Kivityedcafe32009-12-30 18:07:40 +02003134 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
Avi Kivity4d4ec082009-12-29 18:07:30 +02003135 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
Anthony Liguori25c4c272007-04-27 09:29:21 +03003136 skip_emulated_instruction(vcpu);
Avi Kivity6b52d182010-01-21 15:31:47 +02003137 vmx_fpu_activate(vcpu);
Anthony Liguori25c4c272007-04-27 09:29:21 +03003138 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003139 case 1: /*mov from cr*/
3140 switch (cr) {
3141 case 3:
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003142 kvm_register_write(vcpu, reg, vcpu->arch.cr3);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003143 trace_kvm_cr_read(cr, vcpu->arch.cr3);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003144 skip_emulated_instruction(vcpu);
3145 return 1;
3146 case 8:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003147 val = kvm_get_cr8(vcpu);
3148 kvm_register_write(vcpu, reg, val);
3149 trace_kvm_cr_read(cr, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003150 skip_emulated_instruction(vcpu);
3151 return 1;
3152 }
3153 break;
3154 case 3: /* lmsw */
Avi Kivitya1f83a72009-12-29 17:33:58 +02003155 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
Avi Kivity4d4ec082009-12-29 18:07:30 +02003156 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
Avi Kivitya1f83a72009-12-29 17:33:58 +02003157 kvm_lmsw(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003158
3159 skip_emulated_instruction(vcpu);
3160 return 1;
3161 default:
3162 break;
3163 }
Avi Kivity851ba692009-08-24 11:10:17 +03003164 vcpu->run->exit_reason = 0;
Rusty Russellf0242472007-08-01 10:48:02 +10003165 pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
Avi Kivity6aa8b732006-12-10 02:21:36 -08003166 (int)(exit_qualification >> 4) & 3, cr);
3167 return 0;
3168}
3169
Avi Kivity851ba692009-08-24 11:10:17 +03003170static int handle_dr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003171{
He, Qingbfdaab02007-09-12 14:18:28 +08003172 unsigned long exit_qualification;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003173 int dr, reg;
3174
Jan Kiszkaf2483412010-01-20 18:20:20 +01003175 /* Do not handle if the CPL > 0, will trigger GP on re-entry */
Avi Kivity0a79b002009-09-01 12:03:25 +03003176 if (!kvm_require_cpl(vcpu, 0))
3177 return 1;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003178 dr = vmcs_readl(GUEST_DR7);
3179 if (dr & DR7_GD) {
3180 /*
3181 * As the vm-exit takes precedence over the debug trap, we
3182 * need to emulate the latter, either for the host or the
3183 * guest debugging itself.
3184 */
3185 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
Avi Kivity851ba692009-08-24 11:10:17 +03003186 vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
3187 vcpu->run->debug.arch.dr7 = dr;
3188 vcpu->run->debug.arch.pc =
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003189 vmcs_readl(GUEST_CS_BASE) +
3190 vmcs_readl(GUEST_RIP);
Avi Kivity851ba692009-08-24 11:10:17 +03003191 vcpu->run->debug.arch.exception = DB_VECTOR;
3192 vcpu->run->exit_reason = KVM_EXIT_DEBUG;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003193 return 0;
3194 } else {
3195 vcpu->arch.dr7 &= ~DR7_GD;
3196 vcpu->arch.dr6 |= DR6_BD;
3197 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
3198 kvm_queue_exception(vcpu, DB_VECTOR);
3199 return 1;
3200 }
3201 }
3202
He, Qingbfdaab02007-09-12 14:18:28 +08003203 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003204 dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
3205 reg = DEBUG_REG_ACCESS_REG(exit_qualification);
3206 if (exit_qualification & TYPE_MOV_FROM_DR) {
Gleb Natapov020df072010-04-13 10:05:23 +03003207 unsigned long val;
3208 if (!kvm_get_dr(vcpu, dr, &val))
3209 kvm_register_write(vcpu, reg, val);
3210 } else
3211 kvm_set_dr(vcpu, dr, vcpu->arch.regs[reg]);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003212 skip_emulated_instruction(vcpu);
3213 return 1;
3214}
3215
Gleb Natapov020df072010-04-13 10:05:23 +03003216static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
3217{
3218 vmcs_writel(GUEST_DR7, val);
3219}
3220
Avi Kivity851ba692009-08-24 11:10:17 +03003221static int handle_cpuid(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003222{
Avi Kivity06465c52007-02-28 20:46:53 +02003223 kvm_emulate_cpuid(vcpu);
3224 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003225}
3226
Avi Kivity851ba692009-08-24 11:10:17 +03003227static int handle_rdmsr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003228{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003229 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
Avi Kivity6aa8b732006-12-10 02:21:36 -08003230 u64 data;
3231
3232 if (vmx_get_msr(vcpu, ecx, &data)) {
Avi Kivity59200272010-01-25 19:47:02 +02003233 trace_kvm_msr_read_ex(ecx);
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02003234 kvm_inject_gp(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003235 return 1;
3236 }
3237
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003238 trace_kvm_msr_read(ecx, data);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003239
Avi Kivity6aa8b732006-12-10 02:21:36 -08003240 /* FIXME: handling of bits 32:63 of rax, rdx */
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003241 vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
3242 vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003243 skip_emulated_instruction(vcpu);
3244 return 1;
3245}
3246
Avi Kivity851ba692009-08-24 11:10:17 +03003247static int handle_wrmsr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003248{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003249 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
3250 u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
3251 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003252
3253 if (vmx_set_msr(vcpu, ecx, data) != 0) {
Avi Kivity59200272010-01-25 19:47:02 +02003254 trace_kvm_msr_write_ex(ecx, data);
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02003255 kvm_inject_gp(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003256 return 1;
3257 }
3258
Avi Kivity59200272010-01-25 19:47:02 +02003259 trace_kvm_msr_write(ecx, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003260 skip_emulated_instruction(vcpu);
3261 return 1;
3262}
3263
Avi Kivity851ba692009-08-24 11:10:17 +03003264static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003265{
3266 return 1;
3267}
3268
Avi Kivity851ba692009-08-24 11:10:17 +03003269static int handle_interrupt_window(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003270{
Eddie Dong85f455f2007-07-06 12:20:49 +03003271 u32 cpu_based_vm_exec_control;
3272
3273 /* clear pending irq */
3274 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3275 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
3276 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003277
Jan Kiszkaa26bf122008-09-26 09:30:45 +02003278 ++vcpu->stat.irq_window_exits;
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003279
Dor Laorc1150d82007-01-05 16:36:24 -08003280 /*
3281 * If the user space waits to inject interrupts, exit as soon as
3282 * possible
3283 */
Gleb Natapov80618232009-04-21 17:44:56 +03003284 if (!irqchip_in_kernel(vcpu->kvm) &&
Avi Kivity851ba692009-08-24 11:10:17 +03003285 vcpu->run->request_interrupt_window &&
Gleb Natapov80618232009-04-21 17:44:56 +03003286 !kvm_cpu_has_interrupt(vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003287 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
Dor Laorc1150d82007-01-05 16:36:24 -08003288 return 0;
3289 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003290 return 1;
3291}
3292
Avi Kivity851ba692009-08-24 11:10:17 +03003293static int handle_halt(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003294{
3295 skip_emulated_instruction(vcpu);
Avi Kivityd3bef152007-06-05 15:53:05 +03003296 return kvm_emulate_halt(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003297}
3298
Avi Kivity851ba692009-08-24 11:10:17 +03003299static int handle_vmcall(struct kvm_vcpu *vcpu)
Ingo Molnarc21415e2007-02-19 14:37:47 +02003300{
Dor Laor510043d2007-02-19 18:25:43 +02003301 skip_emulated_instruction(vcpu);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05003302 kvm_emulate_hypercall(vcpu);
3303 return 1;
Ingo Molnarc21415e2007-02-19 14:37:47 +02003304}
3305
Avi Kivity851ba692009-08-24 11:10:17 +03003306static int handle_vmx_insn(struct kvm_vcpu *vcpu)
Avi Kivitye3c7cb62009-06-16 14:19:52 +03003307{
3308 kvm_queue_exception(vcpu, UD_VECTOR);
3309 return 1;
3310}
3311
Avi Kivity851ba692009-08-24 11:10:17 +03003312static int handle_invlpg(struct kvm_vcpu *vcpu)
Marcelo Tosattia7052892008-09-23 13:18:35 -03003313{
Sheng Yangf9c617f2009-03-25 10:08:52 +08003314 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Marcelo Tosattia7052892008-09-23 13:18:35 -03003315
3316 kvm_mmu_invlpg(vcpu, exit_qualification);
3317 skip_emulated_instruction(vcpu);
3318 return 1;
3319}
3320
Avi Kivity851ba692009-08-24 11:10:17 +03003321static int handle_wbinvd(struct kvm_vcpu *vcpu)
Eddie Donge5edaa02007-11-11 12:28:35 +02003322{
3323 skip_emulated_instruction(vcpu);
3324 /* TODO: Add support for VT-d/pass-through device */
3325 return 1;
3326}
3327
Avi Kivity851ba692009-08-24 11:10:17 +03003328static int handle_apic_access(struct kvm_vcpu *vcpu)
Sheng Yangf78e0e22007-10-29 09:40:42 +08003329{
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03003330 return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
Sheng Yangf78e0e22007-10-29 09:40:42 +08003331}
3332
Avi Kivity851ba692009-08-24 11:10:17 +03003333static int handle_task_switch(struct kvm_vcpu *vcpu)
Izik Eidus37817f22008-03-24 23:14:53 +02003334{
Jan Kiszka60637aa2008-09-26 09:30:47 +02003335 struct vcpu_vmx *vmx = to_vmx(vcpu);
Izik Eidus37817f22008-03-24 23:14:53 +02003336 unsigned long exit_qualification;
Jan Kiszkae269fb22010-04-14 15:51:09 +02003337 bool has_error_code = false;
3338 u32 error_code = 0;
Izik Eidus37817f22008-03-24 23:14:53 +02003339 u16 tss_selector;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003340 int reason, type, idt_v;
3341
3342 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
3343 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
Izik Eidus37817f22008-03-24 23:14:53 +02003344
3345 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3346
3347 reason = (u32)exit_qualification >> 30;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003348 if (reason == TASK_SWITCH_GATE && idt_v) {
3349 switch (type) {
3350 case INTR_TYPE_NMI_INTR:
3351 vcpu->arch.nmi_injected = false;
3352 if (cpu_has_virtual_nmis())
3353 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3354 GUEST_INTR_STATE_NMI);
3355 break;
3356 case INTR_TYPE_EXT_INTR:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003357 case INTR_TYPE_SOFT_INTR:
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003358 kvm_clear_interrupt_queue(vcpu);
3359 break;
3360 case INTR_TYPE_HARD_EXCEPTION:
Jan Kiszkae269fb22010-04-14 15:51:09 +02003361 if (vmx->idt_vectoring_info &
3362 VECTORING_INFO_DELIVER_CODE_MASK) {
3363 has_error_code = true;
3364 error_code =
3365 vmcs_read32(IDT_VECTORING_ERROR_CODE);
3366 }
3367 /* fall through */
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003368 case INTR_TYPE_SOFT_EXCEPTION:
3369 kvm_clear_exception_queue(vcpu);
3370 break;
3371 default:
3372 break;
3373 }
Jan Kiszka60637aa2008-09-26 09:30:47 +02003374 }
Izik Eidus37817f22008-03-24 23:14:53 +02003375 tss_selector = exit_qualification;
3376
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003377 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
3378 type != INTR_TYPE_EXT_INTR &&
3379 type != INTR_TYPE_NMI_INTR))
3380 skip_emulated_instruction(vcpu);
3381
Gleb Natapovacb54512010-04-15 21:03:50 +03003382 if (kvm_task_switch(vcpu, tss_selector, reason,
3383 has_error_code, error_code) == EMULATE_FAIL) {
3384 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3385 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3386 vcpu->run->internal.ndata = 0;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003387 return 0;
Gleb Natapovacb54512010-04-15 21:03:50 +03003388 }
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003389
3390 /* clear all local breakpoint enable flags */
3391 vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);
3392
3393 /*
3394 * TODO: What about debug traps on tss switch?
3395 * Are we supposed to inject them and update dr6?
3396 */
3397
3398 return 1;
Izik Eidus37817f22008-03-24 23:14:53 +02003399}
3400
Avi Kivity851ba692009-08-24 11:10:17 +03003401static int handle_ept_violation(struct kvm_vcpu *vcpu)
Sheng Yang14394422008-04-28 12:24:45 +08003402{
Sheng Yangf9c617f2009-03-25 10:08:52 +08003403 unsigned long exit_qualification;
Sheng Yang14394422008-04-28 12:24:45 +08003404 gpa_t gpa;
Sheng Yang14394422008-04-28 12:24:45 +08003405 int gla_validity;
Sheng Yang14394422008-04-28 12:24:45 +08003406
Sheng Yangf9c617f2009-03-25 10:08:52 +08003407 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Sheng Yang14394422008-04-28 12:24:45 +08003408
3409 if (exit_qualification & (1 << 6)) {
3410 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
Jan Kiszka7f582ab2009-07-22 23:53:01 +02003411 return -EINVAL;
Sheng Yang14394422008-04-28 12:24:45 +08003412 }
3413
3414 gla_validity = (exit_qualification >> 7) & 0x3;
3415 if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
3416 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
3417 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
3418 (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
Sheng Yangf9c617f2009-03-25 10:08:52 +08003419 vmcs_readl(GUEST_LINEAR_ADDRESS));
Sheng Yang14394422008-04-28 12:24:45 +08003420 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
3421 (long unsigned int)exit_qualification);
Avi Kivity851ba692009-08-24 11:10:17 +03003422 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3423 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
Avi Kivity596ae892009-06-03 14:12:10 +03003424 return 0;
Sheng Yang14394422008-04-28 12:24:45 +08003425 }
3426
3427 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003428 trace_kvm_page_fault(gpa, exit_qualification);
Sheng Yang49cd7d22009-02-11 13:50:40 +08003429 return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
Sheng Yang14394422008-04-28 12:24:45 +08003430}
3431
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003432static u64 ept_rsvd_mask(u64 spte, int level)
3433{
3434 int i;
3435 u64 mask = 0;
3436
3437 for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
3438 mask |= (1ULL << i);
3439
3440 if (level > 2)
3441 /* bits 7:3 reserved */
3442 mask |= 0xf8;
3443 else if (level == 2) {
3444 if (spte & (1ULL << 7))
3445 /* 2MB ref, bits 20:12 reserved */
3446 mask |= 0x1ff000;
3447 else
3448 /* bits 6:3 reserved */
3449 mask |= 0x78;
3450 }
3451
3452 return mask;
3453}
3454
3455static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
3456 int level)
3457{
3458 printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);
3459
3460 /* 010b (write-only) */
3461 WARN_ON((spte & 0x7) == 0x2);
3462
3463 /* 110b (write/execute) */
3464 WARN_ON((spte & 0x7) == 0x6);
3465
3466 /* 100b (execute-only) and value not supported by logical processor */
3467 if (!cpu_has_vmx_ept_execute_only())
3468 WARN_ON((spte & 0x7) == 0x4);
3469
3470 /* not 000b */
3471 if ((spte & 0x7)) {
3472 u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);
3473
3474 if (rsvd_bits != 0) {
3475 printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
3476 __func__, rsvd_bits);
3477 WARN_ON(1);
3478 }
3479
3480 if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
3481 u64 ept_mem_type = (spte & 0x38) >> 3;
3482
3483 if (ept_mem_type == 2 || ept_mem_type == 3 ||
3484 ept_mem_type == 7) {
3485 printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
3486 __func__, ept_mem_type);
3487 WARN_ON(1);
3488 }
3489 }
3490 }
3491}
3492
Avi Kivity851ba692009-08-24 11:10:17 +03003493static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003494{
3495 u64 sptes[4];
3496 int nr_sptes, i;
3497 gpa_t gpa;
3498
3499 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3500
3501 printk(KERN_ERR "EPT: Misconfiguration.\n");
3502 printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);
3503
3504 nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);
3505
3506 for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
3507 ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);
3508
Avi Kivity851ba692009-08-24 11:10:17 +03003509 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3510 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003511
3512 return 0;
3513}
3514
Avi Kivity851ba692009-08-24 11:10:17 +03003515static int handle_nmi_window(struct kvm_vcpu *vcpu)
Sheng Yangf08864b2008-05-15 18:23:25 +08003516{
3517 u32 cpu_based_vm_exec_control;
3518
3519 /* clear pending NMI */
3520 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3521 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
3522 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
3523 ++vcpu->stat.nmi_window_exits;
3524
3525 return 1;
3526}
3527
Mohammed Gamal80ced182009-09-01 12:48:18 +02003528static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003529{
Avi Kivity8b3079a2009-01-05 12:10:54 +02003530 struct vcpu_vmx *vmx = to_vmx(vcpu);
3531 enum emulation_result err = EMULATE_DONE;
Mohammed Gamal80ced182009-09-01 12:48:18 +02003532 int ret = 1;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003533
3534 while (!guest_state_valid(vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003535 err = emulate_instruction(vcpu, 0, 0, 0);
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003536
Mohammed Gamal80ced182009-09-01 12:48:18 +02003537 if (err == EMULATE_DO_MMIO) {
3538 ret = 0;
3539 goto out;
3540 }
Guillaume Thouvenin1d5a4d92008-10-29 09:39:42 +01003541
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03003542 if (err != EMULATE_DONE)
3543 return 0;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003544
3545 if (signal_pending(current))
Mohammed Gamal80ced182009-09-01 12:48:18 +02003546 goto out;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003547 if (need_resched())
3548 schedule();
3549 }
3550
Mohammed Gamal80ced182009-09-01 12:48:18 +02003551 vmx->emulation_required = 0;
3552out:
3553 return ret;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003554}
3555
Avi Kivity6aa8b732006-12-10 02:21:36 -08003556/*
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003557 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
3558 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
3559 */
Marcelo Tosatti9fb41ba2009-10-12 19:37:31 -03003560static int handle_pause(struct kvm_vcpu *vcpu)
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003561{
3562 skip_emulated_instruction(vcpu);
3563 kvm_vcpu_on_spin(vcpu);
3564
3565 return 1;
3566}
3567
Sheng Yang59708672009-12-15 13:29:54 +08003568static int handle_invalid_op(struct kvm_vcpu *vcpu)
3569{
3570 kvm_queue_exception(vcpu, UD_VECTOR);
3571 return 1;
3572}
3573
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003574/*
Avi Kivity6aa8b732006-12-10 02:21:36 -08003575 * The exit handlers return 1 if the exit was handled fully and guest execution
3576 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
3577 * to be done to userspace and return 0.
3578 */
Avi Kivity851ba692009-08-24 11:10:17 +03003579static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08003580 [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
3581 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
Avi Kivity988ad742007-02-12 00:54:36 -08003582 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
Sheng Yangf08864b2008-05-15 18:23:25 +08003583 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003584 [EXIT_REASON_IO_INSTRUCTION] = handle_io,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003585 [EXIT_REASON_CR_ACCESS] = handle_cr,
3586 [EXIT_REASON_DR_ACCESS] = handle_dr,
3587 [EXIT_REASON_CPUID] = handle_cpuid,
3588 [EXIT_REASON_MSR_READ] = handle_rdmsr,
3589 [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
3590 [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
3591 [EXIT_REASON_HLT] = handle_halt,
Marcelo Tosattia7052892008-09-23 13:18:35 -03003592 [EXIT_REASON_INVLPG] = handle_invlpg,
Ingo Molnarc21415e2007-02-19 14:37:47 +02003593 [EXIT_REASON_VMCALL] = handle_vmcall,
Avi Kivitye3c7cb62009-06-16 14:19:52 +03003594 [EXIT_REASON_VMCLEAR] = handle_vmx_insn,
3595 [EXIT_REASON_VMLAUNCH] = handle_vmx_insn,
3596 [EXIT_REASON_VMPTRLD] = handle_vmx_insn,
3597 [EXIT_REASON_VMPTRST] = handle_vmx_insn,
3598 [EXIT_REASON_VMREAD] = handle_vmx_insn,
3599 [EXIT_REASON_VMRESUME] = handle_vmx_insn,
3600 [EXIT_REASON_VMWRITE] = handle_vmx_insn,
3601 [EXIT_REASON_VMOFF] = handle_vmx_insn,
3602 [EXIT_REASON_VMON] = handle_vmx_insn,
Sheng Yangf78e0e22007-10-29 09:40:42 +08003603 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
3604 [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
Eddie Donge5edaa02007-11-11 12:28:35 +02003605 [EXIT_REASON_WBINVD] = handle_wbinvd,
Izik Eidus37817f22008-03-24 23:14:53 +02003606 [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
Andi Kleena0861c02009-06-08 17:37:09 +08003607 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003608 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
3609 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig,
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003610 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause,
Sheng Yang59708672009-12-15 13:29:54 +08003611 [EXIT_REASON_MWAIT_INSTRUCTION] = handle_invalid_op,
3612 [EXIT_REASON_MONITOR_INSTRUCTION] = handle_invalid_op,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003613};
3614
3615static const int kvm_vmx_max_exit_handlers =
Robert P. J. Day50a34852007-06-03 13:35:29 -04003616 ARRAY_SIZE(kvm_vmx_exit_handlers);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003617
3618/*
3619 * The guest has exited. See if we can fix it or if we need userspace
3620 * assistance.
3621 */
Avi Kivity851ba692009-08-24 11:10:17 +03003622static int vmx_handle_exit(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003623{
Avi Kivity29bd8a72007-09-10 17:27:03 +03003624 struct vcpu_vmx *vmx = to_vmx(vcpu);
Andi Kleena0861c02009-06-08 17:37:09 +08003625 u32 exit_reason = vmx->exit_reason;
Avi Kivity1155f762007-11-22 11:30:47 +02003626 u32 vectoring_info = vmx->idt_vectoring_info;
Avi Kivity29bd8a72007-09-10 17:27:03 +03003627
Avi Kivity5bfd8b52010-03-11 10:50:44 +02003628 trace_kvm_exit(exit_reason, vcpu);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003629
Mohammed Gamal80ced182009-09-01 12:48:18 +02003630 /* If guest state is invalid, start emulating */
3631 if (vmx->emulation_required && emulate_invalid_guest_state)
3632 return handle_invalid_guest_state(vcpu);
Guillaume Thouvenin1d5a4d92008-10-29 09:39:42 +01003633
Sheng Yang14394422008-04-28 12:24:45 +08003634 /* Access CR3 don't cause VMExit in paging mode, so we need
3635 * to sync with guest real CR3. */
Avi Kivity6de4f3a2009-05-31 22:58:47 +03003636 if (enable_ept && is_paging(vcpu))
Sheng Yang14394422008-04-28 12:24:45 +08003637 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
Sheng Yang14394422008-04-28 12:24:45 +08003638
Avi Kivity29bd8a72007-09-10 17:27:03 +03003639 if (unlikely(vmx->fail)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003640 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3641 vcpu->run->fail_entry.hardware_entry_failure_reason
Avi Kivity29bd8a72007-09-10 17:27:03 +03003642 = vmcs_read32(VM_INSTRUCTION_ERROR);
3643 return 0;
3644 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003645
Mike Dayd77c26f2007-10-08 09:02:08 -04003646 if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
Sheng Yang14394422008-04-28 12:24:45 +08003647 (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
Jan Kiszka60637aa2008-09-26 09:30:47 +02003648 exit_reason != EXIT_REASON_EPT_VIOLATION &&
3649 exit_reason != EXIT_REASON_TASK_SWITCH))
3650 printk(KERN_WARNING "%s: unexpected, valid vectoring info "
3651 "(0x%x) and exit reason is 0x%x\n",
3652 __func__, vectoring_info, exit_reason);
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003653
3654 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
Gleb Natapovc4282df2009-04-21 17:45:07 +03003655 if (vmx_interrupt_allowed(vcpu)) {
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003656 vmx->soft_vnmi_blocked = 0;
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003657 } else if (vmx->vnmi_blocked_time > 1000000000LL &&
Jan Kiszka45312202008-12-11 16:54:54 +01003658 vcpu->arch.nmi_pending) {
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003659 /*
3660 * This CPU don't support us in finding the end of an
3661 * NMI-blocked window if the guest runs with IRQs
3662 * disabled. So we pull the trigger after 1 s of
3663 * futile waiting, but inform the user about this.
3664 */
3665 printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
3666 "state on VCPU %d after 1 s timeout\n",
3667 __func__, vcpu->vcpu_id);
3668 vmx->soft_vnmi_blocked = 0;
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003669 }
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003670 }
3671
Avi Kivity6aa8b732006-12-10 02:21:36 -08003672 if (exit_reason < kvm_vmx_max_exit_handlers
3673 && kvm_vmx_exit_handlers[exit_reason])
Avi Kivity851ba692009-08-24 11:10:17 +03003674 return kvm_vmx_exit_handlers[exit_reason](vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003675 else {
Avi Kivity851ba692009-08-24 11:10:17 +03003676 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3677 vcpu->run->hw.hardware_exit_reason = exit_reason;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003678 }
3679 return 0;
3680}
3681
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003682static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003683{
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003684 if (irr == -1 || tpr < irr) {
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003685 vmcs_write32(TPR_THRESHOLD, 0);
3686 return;
3687 }
3688
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003689 vmcs_write32(TPR_THRESHOLD, irr);
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003690}
3691
Avi Kivitycf393f72008-07-01 16:20:21 +03003692static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
3693{
3694 u32 exit_intr_info;
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003695 u32 idt_vectoring_info = vmx->idt_vectoring_info;
Avi Kivitycf393f72008-07-01 16:20:21 +03003696 bool unblock_nmi;
3697 u8 vector;
Avi Kivity668f6122008-07-02 09:28:55 +03003698 int type;
3699 bool idtv_info_valid;
Avi Kivitycf393f72008-07-01 16:20:21 +03003700
3701 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
Gleb Natapov20f65982009-05-11 13:35:55 +03003702
Andi Kleena0861c02009-06-08 17:37:09 +08003703 vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
3704
3705 /* Handle machine checks before interrupts are enabled */
3706 if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
3707 || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
3708 && is_machine_check(exit_intr_info)))
3709 kvm_machine_check();
3710
Gleb Natapov20f65982009-05-11 13:35:55 +03003711 /* We need to handle NMIs before interrupts are enabled */
3712 if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
Zhang, Yanminff9d07a2010-04-19 13:32:45 +08003713 (exit_intr_info & INTR_INFO_VALID_MASK)) {
3714 kvm_before_handle_nmi(&vmx->vcpu);
Gleb Natapov20f65982009-05-11 13:35:55 +03003715 asm("int $2");
Zhang, Yanminff9d07a2010-04-19 13:32:45 +08003716 kvm_after_handle_nmi(&vmx->vcpu);
3717 }
Gleb Natapov20f65982009-05-11 13:35:55 +03003718
3719 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3720
Avi Kivitycf393f72008-07-01 16:20:21 +03003721 if (cpu_has_virtual_nmis()) {
3722 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
3723 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
3724 /*
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003725 * SDM 3: 27.7.1.2 (September 2008)
Avi Kivitycf393f72008-07-01 16:20:21 +03003726 * Re-set bit "block by NMI" before VM entry if vmexit caused by
3727 * a guest IRET fault.
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003728 * SDM 3: 23.2.2 (September 2008)
3729 * Bit 12 is undefined in any of the following cases:
3730 * If the VM exit sets the valid bit in the IDT-vectoring
3731 * information field.
3732 * If the VM exit is due to a double fault.
Avi Kivitycf393f72008-07-01 16:20:21 +03003733 */
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003734 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
3735 vector != DF_VECTOR && !idtv_info_valid)
Avi Kivitycf393f72008-07-01 16:20:21 +03003736 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3737 GUEST_INTR_STATE_NMI);
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003738 } else if (unlikely(vmx->soft_vnmi_blocked))
3739 vmx->vnmi_blocked_time +=
3740 ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
Avi Kivity668f6122008-07-02 09:28:55 +03003741
Gleb Natapov37b96e92009-03-30 16:03:13 +03003742 vmx->vcpu.arch.nmi_injected = false;
3743 kvm_clear_exception_queue(&vmx->vcpu);
3744 kvm_clear_interrupt_queue(&vmx->vcpu);
3745
3746 if (!idtv_info_valid)
3747 return;
3748
Avi Kivity668f6122008-07-02 09:28:55 +03003749 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
3750 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
Gleb Natapov37b96e92009-03-30 16:03:13 +03003751
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003752 switch (type) {
Gleb Natapov37b96e92009-03-30 16:03:13 +03003753 case INTR_TYPE_NMI_INTR:
3754 vmx->vcpu.arch.nmi_injected = true;
Avi Kivity668f6122008-07-02 09:28:55 +03003755 /*
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003756 * SDM 3: 27.7.1.2 (September 2008)
Gleb Natapov37b96e92009-03-30 16:03:13 +03003757 * Clear bit "block by NMI" before VM entry if a NMI
3758 * delivery faulted.
Avi Kivity668f6122008-07-02 09:28:55 +03003759 */
Gleb Natapov37b96e92009-03-30 16:03:13 +03003760 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
3761 GUEST_INTR_STATE_NMI);
3762 break;
Gleb Natapov37b96e92009-03-30 16:03:13 +03003763 case INTR_TYPE_SOFT_EXCEPTION:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003764 vmx->vcpu.arch.event_exit_inst_len =
3765 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3766 /* fall through */
3767 case INTR_TYPE_HARD_EXCEPTION:
Avi Kivity35920a32008-07-03 14:50:12 +03003768 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
Gleb Natapov37b96e92009-03-30 16:03:13 +03003769 u32 err = vmcs_read32(IDT_VECTORING_ERROR_CODE);
3770 kvm_queue_exception_e(&vmx->vcpu, vector, err);
Avi Kivity35920a32008-07-03 14:50:12 +03003771 } else
3772 kvm_queue_exception(&vmx->vcpu, vector);
Gleb Natapov37b96e92009-03-30 16:03:13 +03003773 break;
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003774 case INTR_TYPE_SOFT_INTR:
3775 vmx->vcpu.arch.event_exit_inst_len =
3776 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3777 /* fall through */
Gleb Natapov37b96e92009-03-30 16:03:13 +03003778 case INTR_TYPE_EXT_INTR:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003779 kvm_queue_interrupt(&vmx->vcpu, vector,
3780 type == INTR_TYPE_SOFT_INTR);
Gleb Natapov37b96e92009-03-30 16:03:13 +03003781 break;
3782 default:
3783 break;
Avi Kivityf7d92382008-07-03 16:14:28 +03003784 }
Avi Kivitycf393f72008-07-01 16:20:21 +03003785}
3786
Avi Kivity9c8cba32007-11-22 11:42:59 +02003787/*
3788 * Failure to inject an interrupt should give us the information
3789 * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs
3790 * when fetching the interrupt redirection bitmap in the real-mode
3791 * tss, this doesn't happen. So we do it ourselves.
3792 */
3793static void fixup_rmode_irq(struct vcpu_vmx *vmx)
3794{
3795 vmx->rmode.irq.pending = 0;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003796 if (kvm_rip_read(&vmx->vcpu) + 1 != vmx->rmode.irq.rip)
Avi Kivity9c8cba32007-11-22 11:42:59 +02003797 return;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003798 kvm_rip_write(&vmx->vcpu, vmx->rmode.irq.rip);
Avi Kivity9c8cba32007-11-22 11:42:59 +02003799 if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {
3800 vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK;
3801 vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR;
3802 return;
3803 }
3804 vmx->idt_vectoring_info =
3805 VECTORING_INFO_VALID_MASK
3806 | INTR_TYPE_EXT_INTR
3807 | vmx->rmode.irq.vector;
3808}
3809
Avi Kivityc8019492008-07-14 14:44:59 +03003810#ifdef CONFIG_X86_64
3811#define R "r"
3812#define Q "q"
3813#else
3814#define R "e"
3815#define Q "l"
3816#endif
3817
Avi Kivity851ba692009-08-24 11:10:17 +03003818static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003819{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003820 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivitye6adf282007-04-30 16:07:54 +03003821
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003822 /* Record the guest's net vcpu time for enforced NMI injections. */
3823 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
3824 vmx->entry_time = ktime_get();
3825
Mohammed Gamal80ced182009-09-01 12:48:18 +02003826 /* Don't enter VMX if guest state is invalid, let the exit handler
3827 start emulation until we arrive back to a valid state */
3828 if (vmx->emulation_required && emulate_invalid_guest_state)
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03003829 return;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03003830
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003831 if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
3832 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
3833 if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
3834 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
3835
Gleb Natapov787ff732009-05-18 11:44:06 +03003836 /* When single-stepping over STI and MOV SS, we must clear the
3837 * corresponding interruptibility bits in the guest state. Otherwise
3838 * vmentry fails as it then expects bit 14 (BS) in pending debug
3839 * exceptions being set, but that's not correct for the guest debugging
3840 * case. */
3841 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
3842 vmx_set_interrupt_shadow(vcpu, 0);
3843
Mike Dayd77c26f2007-10-08 09:02:08 -04003844 asm(
Avi Kivity6aa8b732006-12-10 02:21:36 -08003845 /* Store host registers */
Avi Kivityc8019492008-07-14 14:44:59 +03003846 "push %%"R"dx; push %%"R"bp;"
3847 "push %%"R"cx \n\t"
Avi Kivity313dbd42008-07-17 18:04:30 +03003848 "cmp %%"R"sp, %c[host_rsp](%0) \n\t"
3849 "je 1f \n\t"
3850 "mov %%"R"sp, %c[host_rsp](%0) \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003851 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
Avi Kivity313dbd42008-07-17 18:04:30 +03003852 "1: \n\t"
Avi Kivityd3edefc2009-06-16 12:33:56 +03003853 /* Reload cr2 if changed */
3854 "mov %c[cr2](%0), %%"R"ax \n\t"
3855 "mov %%cr2, %%"R"dx \n\t"
3856 "cmp %%"R"ax, %%"R"dx \n\t"
3857 "je 2f \n\t"
3858 "mov %%"R"ax, %%cr2 \n\t"
3859 "2: \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003860 /* Check if vmlaunch of vmresume is needed */
Avi Kivitye08aa782007-11-15 18:06:18 +02003861 "cmpl $0, %c[launched](%0) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003862 /* Load guest registers. Don't clobber flags. */
Avi Kivityc8019492008-07-14 14:44:59 +03003863 "mov %c[rax](%0), %%"R"ax \n\t"
3864 "mov %c[rbx](%0), %%"R"bx \n\t"
3865 "mov %c[rdx](%0), %%"R"dx \n\t"
3866 "mov %c[rsi](%0), %%"R"si \n\t"
3867 "mov %c[rdi](%0), %%"R"di \n\t"
3868 "mov %c[rbp](%0), %%"R"bp \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003869#ifdef CONFIG_X86_64
Avi Kivitye08aa782007-11-15 18:06:18 +02003870 "mov %c[r8](%0), %%r8 \n\t"
3871 "mov %c[r9](%0), %%r9 \n\t"
3872 "mov %c[r10](%0), %%r10 \n\t"
3873 "mov %c[r11](%0), %%r11 \n\t"
3874 "mov %c[r12](%0), %%r12 \n\t"
3875 "mov %c[r13](%0), %%r13 \n\t"
3876 "mov %c[r14](%0), %%r14 \n\t"
3877 "mov %c[r15](%0), %%r15 \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003878#endif
Avi Kivityc8019492008-07-14 14:44:59 +03003879 "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */
3880
Avi Kivity6aa8b732006-12-10 02:21:36 -08003881 /* Enter guest mode */
Avi Kivitycd2276a2007-05-14 20:41:13 +03003882 "jne .Llaunched \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003883 __ex(ASM_VMX_VMLAUNCH) "\n\t"
Avi Kivitycd2276a2007-05-14 20:41:13 +03003884 "jmp .Lkvm_vmx_return \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003885 ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
Avi Kivitycd2276a2007-05-14 20:41:13 +03003886 ".Lkvm_vmx_return: "
Avi Kivity6aa8b732006-12-10 02:21:36 -08003887 /* Save guest registers, load host registers, keep flags */
Avi Kivityc8019492008-07-14 14:44:59 +03003888 "xchg %0, (%%"R"sp) \n\t"
3889 "mov %%"R"ax, %c[rax](%0) \n\t"
3890 "mov %%"R"bx, %c[rbx](%0) \n\t"
3891 "push"Q" (%%"R"sp); pop"Q" %c[rcx](%0) \n\t"
3892 "mov %%"R"dx, %c[rdx](%0) \n\t"
3893 "mov %%"R"si, %c[rsi](%0) \n\t"
3894 "mov %%"R"di, %c[rdi](%0) \n\t"
3895 "mov %%"R"bp, %c[rbp](%0) \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003896#ifdef CONFIG_X86_64
Avi Kivitye08aa782007-11-15 18:06:18 +02003897 "mov %%r8, %c[r8](%0) \n\t"
3898 "mov %%r9, %c[r9](%0) \n\t"
3899 "mov %%r10, %c[r10](%0) \n\t"
3900 "mov %%r11, %c[r11](%0) \n\t"
3901 "mov %%r12, %c[r12](%0) \n\t"
3902 "mov %%r13, %c[r13](%0) \n\t"
3903 "mov %%r14, %c[r14](%0) \n\t"
3904 "mov %%r15, %c[r15](%0) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003905#endif
Avi Kivityc8019492008-07-14 14:44:59 +03003906 "mov %%cr2, %%"R"ax \n\t"
3907 "mov %%"R"ax, %c[cr2](%0) \n\t"
3908
3909 "pop %%"R"bp; pop %%"R"bp; pop %%"R"dx \n\t"
Avi Kivitye08aa782007-11-15 18:06:18 +02003910 "setbe %c[fail](%0) \n\t"
3911 : : "c"(vmx), "d"((unsigned long)HOST_RSP),
3912 [launched]"i"(offsetof(struct vcpu_vmx, launched)),
3913 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
Avi Kivity313dbd42008-07-17 18:04:30 +03003914 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003915 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
3916 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
3917 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
3918 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
3919 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
3920 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
3921 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003922#ifdef CONFIG_X86_64
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003923 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
3924 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
3925 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
3926 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
3927 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
3928 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
3929 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
3930 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
Avi Kivity6aa8b732006-12-10 02:21:36 -08003931#endif
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003932 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2))
Laurent Vivierc2036302007-10-25 14:18:52 +02003933 : "cc", "memory"
Avi Kivityc8019492008-07-14 14:44:59 +03003934 , R"bx", R"di", R"si"
Laurent Vivierc2036302007-10-25 14:18:52 +02003935#ifdef CONFIG_X86_64
Laurent Vivierc2036302007-10-25 14:18:52 +02003936 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
3937#endif
3938 );
Avi Kivity6aa8b732006-12-10 02:21:36 -08003939
Avi Kivity6de4f3a2009-05-31 22:58:47 +03003940 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
3941 | (1 << VCPU_EXREG_PDPTR));
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003942 vcpu->arch.regs_dirty = 0;
3943
Avi Kivity1155f762007-11-22 11:30:47 +02003944 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
Avi Kivity9c8cba32007-11-22 11:42:59 +02003945 if (vmx->rmode.irq.pending)
3946 fixup_rmode_irq(vmx);
Avi Kivity1155f762007-11-22 11:30:47 +02003947
Mike Dayd77c26f2007-10-08 09:02:08 -04003948 asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
Avi Kivity15ad7142007-07-11 18:17:21 +03003949 vmx->launched = 1;
Avi Kivity1b6269d2007-10-09 12:12:19 +02003950
Avi Kivitycf393f72008-07-01 16:20:21 +03003951 vmx_complete_interrupts(vmx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003952}
3953
Avi Kivityc8019492008-07-14 14:44:59 +03003954#undef R
3955#undef Q
3956
Avi Kivity6aa8b732006-12-10 02:21:36 -08003957static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
3958{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003959 struct vcpu_vmx *vmx = to_vmx(vcpu);
3960
3961 if (vmx->vmcs) {
Avi Kivity543e4242008-05-13 16:22:47 +03003962 vcpu_clear(vmx);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003963 free_vmcs(vmx->vmcs);
3964 vmx->vmcs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003965 }
3966}
3967
3968static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
3969{
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003970 struct vcpu_vmx *vmx = to_vmx(vcpu);
3971
Lai Jiangshancdbecfc2010-04-17 16:41:47 +08003972 free_vpid(vmx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003973 vmx_free_vmcs(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003974 kfree(vmx->guest_msrs);
3975 kvm_vcpu_uninit(vcpu);
Rusty Russella4770342007-08-01 14:46:11 +10003976 kmem_cache_free(kvm_vcpu_cache, vmx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003977}
3978
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003979static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003980{
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003981 int err;
Rusty Russellc16f8622007-07-30 21:12:19 +10003982 struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
Avi Kivity15ad7142007-07-11 18:17:21 +03003983 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003984
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003985 if (!vmx)
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003986 return ERR_PTR(-ENOMEM);
3987
Sheng Yang2384d2b2008-01-17 15:14:33 +08003988 allocate_vpid(vmx);
3989
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003990 err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
3991 if (err)
3992 goto free_vcpu;
Ingo Molnar965b58a2007-01-05 16:36:23 -08003993
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003994 vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003995 if (!vmx->guest_msrs) {
3996 err = -ENOMEM;
3997 goto uninit_vcpu;
3998 }
Ingo Molnar965b58a2007-01-05 16:36:23 -08003999
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004000 vmx->vmcs = alloc_vmcs();
4001 if (!vmx->vmcs)
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004002 goto free_msrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04004003
4004 vmcs_clear(vmx->vmcs);
4005
Avi Kivity15ad7142007-07-11 18:17:21 +03004006 cpu = get_cpu();
4007 vmx_vcpu_load(&vmx->vcpu, cpu);
Rusty Russell8b9cf982007-07-30 16:31:43 +10004008 err = vmx_vcpu_setup(vmx);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004009 vmx_vcpu_put(&vmx->vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03004010 put_cpu();
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004011 if (err)
4012 goto free_vmcs;
Marcelo Tosatti5e4a0b32008-02-14 21:21:43 -02004013 if (vm_need_virtualize_apic_accesses(kvm))
4014 if (alloc_apic_access_page(kvm) != 0)
4015 goto free_vmcs;
Ingo Molnar965b58a2007-01-05 16:36:23 -08004016
Sheng Yangb927a3c2009-07-21 10:42:48 +08004017 if (enable_ept) {
4018 if (!kvm->arch.ept_identity_map_addr)
4019 kvm->arch.ept_identity_map_addr =
4020 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08004021 if (alloc_identity_pagetable(kvm) != 0)
4022 goto free_vmcs;
Sheng Yangb927a3c2009-07-21 10:42:48 +08004023 }
Sheng Yangb7ebfb02008-04-25 21:44:52 +08004024
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004025 return &vmx->vcpu;
Ingo Molnar965b58a2007-01-05 16:36:23 -08004026
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004027free_vmcs:
4028 free_vmcs(vmx->vmcs);
4029free_msrs:
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004030 kfree(vmx->guest_msrs);
4031uninit_vcpu:
4032 kvm_vcpu_uninit(&vmx->vcpu);
4033free_vcpu:
Lai Jiangshancdbecfc2010-04-17 16:41:47 +08004034 free_vpid(vmx);
Rusty Russella4770342007-08-01 14:46:11 +10004035 kmem_cache_free(kvm_vcpu_cache, vmx);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10004036 return ERR_PTR(err);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004037}
4038
Yang, Sheng002c7f72007-07-31 14:23:01 +03004039static void __init vmx_check_processor_compat(void *rtn)
4040{
4041 struct vmcs_config vmcs_conf;
4042
4043 *(int *)rtn = 0;
4044 if (setup_vmcs_config(&vmcs_conf) < 0)
4045 *(int *)rtn = -EIO;
4046 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
4047 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
4048 smp_processor_id());
4049 *(int *)rtn = -EIO;
4050 }
4051}
4052
Sheng Yang67253af2008-04-25 10:20:22 +08004053static int get_ept_level(void)
4054{
4055 return VMX_EPT_DEFAULT_GAW + 1;
4056}
4057
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004058static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
Sheng Yang64d4d522008-10-09 16:01:57 +08004059{
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004060 u64 ret;
4061
Sheng Yang522c68c2009-04-27 20:35:43 +08004062 /* For VT-d and EPT combination
4063 * 1. MMIO: always map as UC
4064 * 2. EPT with VT-d:
4065 * a. VT-d without snooping control feature: can't guarantee the
4066 * result, try to trust guest.
4067 * b. VT-d with snooping control feature: snooping control feature of
4068 * VT-d engine can guarantee the cache correctness. Just set it
4069 * to WB to keep consistent with host. So the same as item 3.
Sheng Yanga19a6d12010-02-09 16:41:53 +08004070 * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
Sheng Yang522c68c2009-04-27 20:35:43 +08004071 * consistent with host MTRR
4072 */
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004073 if (is_mmio)
4074 ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
Sheng Yang522c68c2009-04-27 20:35:43 +08004075 else if (vcpu->kvm->arch.iommu_domain &&
4076 !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
4077 ret = kvm_get_guest_memory_type(vcpu, gfn) <<
4078 VMX_EPT_MT_EPTE_SHIFT;
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004079 else
Sheng Yang522c68c2009-04-27 20:35:43 +08004080 ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
Sheng Yanga19a6d12010-02-09 16:41:53 +08004081 | VMX_EPT_IPAT_BIT;
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004082
4083 return ret;
Sheng Yang64d4d522008-10-09 16:01:57 +08004084}
4085
Avi Kivityf4c9e872009-12-28 16:06:35 +02004086#define _ER(x) { EXIT_REASON_##x, #x }
4087
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004088static const struct trace_print_flags vmx_exit_reasons_str[] = {
Avi Kivityf4c9e872009-12-28 16:06:35 +02004089 _ER(EXCEPTION_NMI),
4090 _ER(EXTERNAL_INTERRUPT),
4091 _ER(TRIPLE_FAULT),
4092 _ER(PENDING_INTERRUPT),
4093 _ER(NMI_WINDOW),
4094 _ER(TASK_SWITCH),
4095 _ER(CPUID),
4096 _ER(HLT),
4097 _ER(INVLPG),
4098 _ER(RDPMC),
4099 _ER(RDTSC),
4100 _ER(VMCALL),
4101 _ER(VMCLEAR),
4102 _ER(VMLAUNCH),
4103 _ER(VMPTRLD),
4104 _ER(VMPTRST),
4105 _ER(VMREAD),
4106 _ER(VMRESUME),
4107 _ER(VMWRITE),
4108 _ER(VMOFF),
4109 _ER(VMON),
4110 _ER(CR_ACCESS),
4111 _ER(DR_ACCESS),
4112 _ER(IO_INSTRUCTION),
4113 _ER(MSR_READ),
4114 _ER(MSR_WRITE),
4115 _ER(MWAIT_INSTRUCTION),
4116 _ER(MONITOR_INSTRUCTION),
4117 _ER(PAUSE_INSTRUCTION),
4118 _ER(MCE_DURING_VMENTRY),
4119 _ER(TPR_BELOW_THRESHOLD),
4120 _ER(APIC_ACCESS),
4121 _ER(EPT_VIOLATION),
4122 _ER(EPT_MISCONFIG),
4123 _ER(WBINVD),
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004124 { -1, NULL }
4125};
4126
Avi Kivityf4c9e872009-12-28 16:06:35 +02004127#undef _ER
4128
Sheng Yang17cc3932010-01-05 19:02:27 +08004129static int vmx_get_lpage_level(void)
Joerg Roedel344f4142009-07-27 16:30:48 +02004130{
Sheng Yang878403b2010-01-05 19:02:29 +08004131 if (enable_ept && !cpu_has_vmx_ept_1g_page())
4132 return PT_DIRECTORY_LEVEL;
4133 else
4134 /* For shadow and EPT supported 1GB page */
4135 return PT_PDPE_LEVEL;
Joerg Roedel344f4142009-07-27 16:30:48 +02004136}
4137
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004138static inline u32 bit(int bitno)
4139{
4140 return 1 << (bitno & 31);
4141}
4142
Sheng Yang0e851882009-12-18 16:48:46 +08004143static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
4144{
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004145 struct kvm_cpuid_entry2 *best;
4146 struct vcpu_vmx *vmx = to_vmx(vcpu);
4147 u32 exec_control;
4148
4149 vmx->rdtscp_enabled = false;
4150 if (vmx_rdtscp_supported()) {
4151 exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
4152 if (exec_control & SECONDARY_EXEC_RDTSCP) {
4153 best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
4154 if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
4155 vmx->rdtscp_enabled = true;
4156 else {
4157 exec_control &= ~SECONDARY_EXEC_RDTSCP;
4158 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
4159 exec_control);
4160 }
4161 }
4162 }
Sheng Yang0e851882009-12-18 16:48:46 +08004163}
4164
Joerg Roedeld4330ef2010-04-22 12:33:11 +02004165static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4166{
4167}
4168
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03004169static struct kvm_x86_ops vmx_x86_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08004170 .cpu_has_kvm_support = cpu_has_kvm_support,
4171 .disabled_by_bios = vmx_disabled_by_bios,
4172 .hardware_setup = hardware_setup,
4173 .hardware_unsetup = hardware_unsetup,
Yang, Sheng002c7f72007-07-31 14:23:01 +03004174 .check_processor_compatibility = vmx_check_processor_compat,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004175 .hardware_enable = hardware_enable,
4176 .hardware_disable = hardware_disable,
Sheng Yang04547152009-04-01 15:52:31 +08004177 .cpu_has_accelerated_tpr = report_flexpriority,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004178
4179 .vcpu_create = vmx_create_vcpu,
4180 .vcpu_free = vmx_free_vcpu,
Avi Kivity04d2cc72007-09-10 18:10:54 +03004181 .vcpu_reset = vmx_vcpu_reset,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004182
Avi Kivity04d2cc72007-09-10 18:10:54 +03004183 .prepare_guest_switch = vmx_save_host_state,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004184 .vcpu_load = vmx_vcpu_load,
4185 .vcpu_put = vmx_vcpu_put,
4186
4187 .set_guest_debug = set_guest_debug,
4188 .get_msr = vmx_get_msr,
4189 .set_msr = vmx_set_msr,
4190 .get_segment_base = vmx_get_segment_base,
4191 .get_segment = vmx_get_segment,
4192 .set_segment = vmx_set_segment,
Izik Eidus2e4d2652008-03-24 19:38:34 +02004193 .get_cpl = vmx_get_cpl,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004194 .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
Avi Kivitye8467fd2009-12-29 18:43:06 +02004195 .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
Anthony Liguori25c4c272007-04-27 09:29:21 +03004196 .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004197 .set_cr0 = vmx_set_cr0,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004198 .set_cr3 = vmx_set_cr3,
4199 .set_cr4 = vmx_set_cr4,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004200 .set_efer = vmx_set_efer,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004201 .get_idt = vmx_get_idt,
4202 .set_idt = vmx_set_idt,
4203 .get_gdt = vmx_get_gdt,
4204 .set_gdt = vmx_set_gdt,
Gleb Natapov020df072010-04-13 10:05:23 +03004205 .set_dr7 = vmx_set_dr7,
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03004206 .cache_reg = vmx_cache_reg,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004207 .get_rflags = vmx_get_rflags,
4208 .set_rflags = vmx_set_rflags,
Avi Kivityebcbab42010-02-07 11:56:52 +02004209 .fpu_activate = vmx_fpu_activate,
Avi Kivity02daab22009-12-30 12:40:26 +02004210 .fpu_deactivate = vmx_fpu_deactivate,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004211
4212 .tlb_flush = vmx_flush_tlb,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004213
Avi Kivity6aa8b732006-12-10 02:21:36 -08004214 .run = vmx_vcpu_run,
Avi Kivity6062d012009-03-23 17:35:17 +02004215 .handle_exit = vmx_handle_exit,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004216 .skip_emulated_instruction = skip_emulated_instruction,
Glauber Costa2809f5d2009-05-12 16:21:05 -04004217 .set_interrupt_shadow = vmx_set_interrupt_shadow,
4218 .get_interrupt_shadow = vmx_get_interrupt_shadow,
Ingo Molnar102d8322007-02-19 14:37:47 +02004219 .patch_hypercall = vmx_patch_hypercall,
Eddie Dong2a8067f2007-08-06 16:29:07 +03004220 .set_irq = vmx_inject_irq,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004221 .set_nmi = vmx_inject_nmi,
Avi Kivity298101d2007-11-25 13:41:11 +02004222 .queue_exception = vmx_queue_exception,
Gleb Natapov78646122009-03-23 12:12:11 +02004223 .interrupt_allowed = vmx_interrupt_allowed,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004224 .nmi_allowed = vmx_nmi_allowed,
Jan Kiszka3cfc3092009-11-12 01:04:25 +01004225 .get_nmi_mask = vmx_get_nmi_mask,
4226 .set_nmi_mask = vmx_set_nmi_mask,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004227 .enable_nmi_window = enable_nmi_window,
4228 .enable_irq_window = enable_irq_window,
4229 .update_cr8_intercept = update_cr8_intercept,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004230
Izik Eiduscbc94022007-10-25 00:29:55 +02004231 .set_tss_addr = vmx_set_tss_addr,
Sheng Yang67253af2008-04-25 10:20:22 +08004232 .get_tdp_level = get_ept_level,
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004233 .get_mt_mask = vmx_get_mt_mask,
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004234
4235 .exit_reasons_str = vmx_exit_reasons_str,
Sheng Yang17cc3932010-01-05 19:02:27 +08004236 .get_lpage_level = vmx_get_lpage_level,
Sheng Yang0e851882009-12-18 16:48:46 +08004237
4238 .cpuid_update = vmx_cpuid_update,
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004239
4240 .rdtscp_supported = vmx_rdtscp_supported,
Joerg Roedeld4330ef2010-04-22 12:33:11 +02004241
4242 .set_supported_cpuid = vmx_set_supported_cpuid,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004243};
4244
4245static int __init vmx_init(void)
4246{
Avi Kivity26bb0982009-09-07 11:14:12 +03004247 int r, i;
4248
4249 rdmsrl_safe(MSR_EFER, &host_efer);
4250
4251 for (i = 0; i < NR_VMX_MSR; ++i)
4252 kvm_define_shared_msr(i, vmx_msr_index[i]);
He, Qingfdef3ad2007-04-30 09:45:24 +03004253
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004254 vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
He, Qingfdef3ad2007-04-30 09:45:24 +03004255 if (!vmx_io_bitmap_a)
4256 return -ENOMEM;
4257
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004258 vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
He, Qingfdef3ad2007-04-30 09:45:24 +03004259 if (!vmx_io_bitmap_b) {
4260 r = -ENOMEM;
4261 goto out;
4262 }
4263
Avi Kivity58972972009-02-24 22:26:47 +02004264 vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
4265 if (!vmx_msr_bitmap_legacy) {
Sheng Yang25c5f222008-03-28 13:18:56 +08004266 r = -ENOMEM;
4267 goto out1;
4268 }
4269
Avi Kivity58972972009-02-24 22:26:47 +02004270 vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
4271 if (!vmx_msr_bitmap_longmode) {
4272 r = -ENOMEM;
4273 goto out2;
4274 }
4275
He, Qingfdef3ad2007-04-30 09:45:24 +03004276 /*
4277 * Allow direct access to the PC debug port (it is often used for I/O
4278 * delays, but the vmexits simply slow things down).
4279 */
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004280 memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
4281 clear_bit(0x80, vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004282
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004283 memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
He, Qingfdef3ad2007-04-30 09:45:24 +03004284
Avi Kivity58972972009-02-24 22:26:47 +02004285 memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
4286 memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
Sheng Yang25c5f222008-03-28 13:18:56 +08004287
Sheng Yang2384d2b2008-01-17 15:14:33 +08004288 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
4289
Avi Kivity0ee75be2010-04-28 15:39:01 +03004290 r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
4291 __alignof__(struct vcpu_vmx), THIS_MODULE);
He, Qingfdef3ad2007-04-30 09:45:24 +03004292 if (r)
Avi Kivity58972972009-02-24 22:26:47 +02004293 goto out3;
Sheng Yang25c5f222008-03-28 13:18:56 +08004294
Avi Kivity58972972009-02-24 22:26:47 +02004295 vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
4296 vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
4297 vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
4298 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
4299 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
4300 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
He, Qingfdef3ad2007-04-30 09:45:24 +03004301
Avi Kivity089d0342009-03-23 18:26:32 +02004302 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +08004303 bypass_guest_pf = 0;
Sheng Yang5fdbcb92008-07-16 09:25:40 +08004304 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
Sheng Yang2aaf69d2009-01-21 16:52:16 +08004305 VMX_EPT_WRITABLE_MASK);
Sheng Yang534e38b2008-09-08 15:12:30 +08004306 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004307 VMX_EPT_EXECUTABLE_MASK);
Sheng Yang5fdbcb92008-07-16 09:25:40 +08004308 kvm_enable_tdp();
4309 } else
4310 kvm_disable_tdp();
Sheng Yang14394422008-04-28 12:24:45 +08004311
Avi Kivityc7addb92007-09-16 18:58:32 +02004312 if (bypass_guest_pf)
4313 kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);
4314
He, Qingfdef3ad2007-04-30 09:45:24 +03004315 return 0;
4316
Avi Kivity58972972009-02-24 22:26:47 +02004317out3:
4318 free_page((unsigned long)vmx_msr_bitmap_longmode);
Sheng Yang25c5f222008-03-28 13:18:56 +08004319out2:
Avi Kivity58972972009-02-24 22:26:47 +02004320 free_page((unsigned long)vmx_msr_bitmap_legacy);
He, Qingfdef3ad2007-04-30 09:45:24 +03004321out1:
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004322 free_page((unsigned long)vmx_io_bitmap_b);
He, Qingfdef3ad2007-04-30 09:45:24 +03004323out:
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004324 free_page((unsigned long)vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004325 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004326}
4327
4328static void __exit vmx_exit(void)
4329{
Avi Kivity58972972009-02-24 22:26:47 +02004330 free_page((unsigned long)vmx_msr_bitmap_legacy);
4331 free_page((unsigned long)vmx_msr_bitmap_longmode);
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004332 free_page((unsigned long)vmx_io_bitmap_b);
4333 free_page((unsigned long)vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004334
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004335 kvm_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004336}
4337
4338module_init(vmx_init)
4339module_exit(vmx_exit)