blob: 334b016267a9b780a2a9832720164636b26446b3 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
Eddie Dong85f455f2007-07-06 12:20:49 +030018#include "irq.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080019#include "mmu.h"
Avi Kivitye4956062007-06-28 14:15:57 -040020
Avi Kivityedf88412007-12-16 11:02:48 +020021#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080022#include <linux/module.h>
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +020023#include <linux/kernel.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080024#include <linux/mm.h>
25#include <linux/highmem.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040026#include <linux/sched.h>
Avi Kivityc7addb92007-09-16 18:58:32 +020027#include <linux/moduleparam.h>
Marcelo Tosatti229456f2009-06-17 09:22:14 -030028#include <linux/ftrace_event.h>
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030029#include "kvm_cache_regs.h"
Avi Kivity35920a32008-07-03 14:50:12 +030030#include "x86.h"
Avi Kivitye4956062007-06-28 14:15:57 -040031
Avi Kivity6aa8b732006-12-10 02:21:36 -080032#include <asm/io.h>
Anthony Liguori3b3be0d2006-12-13 00:33:43 -080033#include <asm/desc.h>
Eduardo Habkost13673a92008-11-17 19:03:13 -020034#include <asm/vmx.h>
Eduardo Habkost6210e372008-11-17 19:03:16 -020035#include <asm/virtext.h>
Andi Kleena0861c02009-06-08 17:37:09 +080036#include <asm/mce.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080037
Marcelo Tosatti229456f2009-06-17 09:22:14 -030038#include "trace.h"
39
Avi Kivity4ecac3f2008-05-13 13:23:38 +030040#define __ex(x) __kvm_handle_fault_on_reboot(x)
41
Avi Kivity6aa8b732006-12-10 02:21:36 -080042MODULE_AUTHOR("Qumranet");
43MODULE_LICENSE("GPL");
44
Avi Kivity4462d212009-03-23 17:53:37 +020045static int __read_mostly bypass_guest_pf = 1;
Avi Kivityc1f8bc02009-03-23 15:41:17 +020046module_param(bypass_guest_pf, bool, S_IRUGO);
Avi Kivityc7addb92007-09-16 18:58:32 +020047
Avi Kivity4462d212009-03-23 17:53:37 +020048static int __read_mostly enable_vpid = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020049module_param_named(vpid, enable_vpid, bool, 0444);
Sheng Yang2384d2b2008-01-17 15:14:33 +080050
Avi Kivity4462d212009-03-23 17:53:37 +020051static int __read_mostly flexpriority_enabled = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020052module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
Avi Kivity4c9fc8e2008-03-24 18:15:14 +020053
Avi Kivity4462d212009-03-23 17:53:37 +020054static int __read_mostly enable_ept = 1;
Avi Kivity736caef2009-03-23 17:39:48 +020055module_param_named(ept, enable_ept, bool, S_IRUGO);
Sheng Yangd56f5462008-04-25 10:13:16 +080056
Nitin A Kamble3a624e22009-06-08 11:34:16 -070057static int __read_mostly enable_unrestricted_guest = 1;
58module_param_named(unrestricted_guest,
59 enable_unrestricted_guest, bool, S_IRUGO);
60
Avi Kivity4462d212009-03-23 17:53:37 +020061static int __read_mostly emulate_invalid_guest_state = 0;
Avi Kivityc1f8bc02009-03-23 15:41:17 +020062module_param(emulate_invalid_guest_state, bool, S_IRUGO);
Mohammed Gamal04fa4d32008-08-17 16:39:48 +030063
Avi Kivitycdc0e242009-12-06 17:21:14 +020064#define KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST \
65 (X86_CR0_WP | X86_CR0_NE | X86_CR0_NW | X86_CR0_CD)
66#define KVM_GUEST_CR0_MASK \
67 (KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
68#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST \
Avi Kivity81231c62010-01-24 16:26:40 +020069 (X86_CR0_WP | X86_CR0_NE)
Avi Kivitycdc0e242009-12-06 17:21:14 +020070#define KVM_VM_CR0_ALWAYS_ON \
71 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE)
Avi Kivity4c386092009-12-07 12:26:18 +020072#define KVM_CR4_GUEST_OWNED_BITS \
73 (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \
74 | X86_CR4_OSXMMEXCPT)
75
Avi Kivitycdc0e242009-12-06 17:21:14 +020076#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
77#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
78
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +080079/*
80 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
81 * ple_gap: upper bound on the amount of time between two successive
82 * executions of PAUSE in a loop. Also indicate if ple enabled.
83 * According to test, this time is usually small than 41 cycles.
84 * ple_window: upper bound on the amount of time a guest is allowed to execute
85 * in a PAUSE loop. Tests indicate that most spinlocks are held for
86 * less than 2^12 cycles
87 * Time is measured based on a counter that runs at the same rate as the TSC,
88 * refer SDM volume 3b section 21.6.13 & 22.1.3.
89 */
90#define KVM_VMX_DEFAULT_PLE_GAP 41
91#define KVM_VMX_DEFAULT_PLE_WINDOW 4096
92static int ple_gap = KVM_VMX_DEFAULT_PLE_GAP;
93module_param(ple_gap, int, S_IRUGO);
94
95static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
96module_param(ple_window, int, S_IRUGO);
97
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -040098struct vmcs {
99 u32 revision_id;
100 u32 abort;
101 char data[0];
102};
103
Avi Kivity26bb0982009-09-07 11:14:12 +0300104struct shared_msr_entry {
105 unsigned index;
106 u64 data;
Avi Kivityd5696722009-12-02 12:28:47 +0200107 u64 mask;
Avi Kivity26bb0982009-09-07 11:14:12 +0300108};
109
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400110struct vcpu_vmx {
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000111 struct kvm_vcpu vcpu;
Avi Kivity543e4242008-05-13 16:22:47 +0300112 struct list_head local_vcpus_link;
Avi Kivity313dbd42008-07-17 18:04:30 +0300113 unsigned long host_rsp;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400114 int launched;
Avi Kivity29bd8a72007-09-10 17:27:03 +0300115 u8 fail;
Avi Kivity1155f762007-11-22 11:30:47 +0200116 u32 idt_vectoring_info;
Avi Kivity26bb0982009-09-07 11:14:12 +0300117 struct shared_msr_entry *guest_msrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400118 int nmsrs;
119 int save_nmsrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400120#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300121 u64 msr_host_kernel_gs_base;
122 u64 msr_guest_kernel_gs_base;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400123#endif
124 struct vmcs *vmcs;
125 struct {
126 int loaded;
127 u16 fs_sel, gs_sel, ldt_sel;
Laurent Vivier152d3f22007-08-23 16:33:11 +0200128 int gs_ldt_reload_needed;
129 int fs_reload_needed;
Mike Dayd77c26f2007-10-08 09:02:08 -0400130 } host_state;
Avi Kivity9c8cba32007-11-22 11:42:59 +0200131 struct {
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300132 int vm86_active;
133 u8 save_iopl;
134 struct kvm_save_segment {
135 u16 selector;
136 unsigned long base;
137 u32 limit;
138 u32 ar;
139 } tr, es, ds, fs, gs;
Avi Kivity9c8cba32007-11-22 11:42:59 +0200140 struct {
141 bool pending;
142 u8 vector;
143 unsigned rip;
144 } irq;
145 } rmode;
Sheng Yang2384d2b2008-01-17 15:14:33 +0800146 int vpid;
Mohammed Gamal04fa4d32008-08-17 16:39:48 +0300147 bool emulation_required;
Jan Kiszka3b86cd92008-09-26 09:30:57 +0200148
149 /* Support for vnmi-less CPUs */
150 int soft_vnmi_blocked;
151 ktime_t entry_time;
152 s64 vnmi_blocked_time;
Andi Kleena0861c02009-06-08 17:37:09 +0800153 u32 exit_reason;
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800154
155 bool rdtscp_enabled;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400156};
157
158static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
159{
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000160 return container_of(vcpu, struct vcpu_vmx, vcpu);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400161}
162
Sheng Yangb7ebfb02008-04-25 21:44:52 +0800163static int init_rmode(struct kvm *kvm);
Sheng Yang4e1096d2008-07-06 19:16:51 +0800164static u64 construct_eptp(unsigned long root_hpa);
Avi Kivity75880a02007-06-20 11:20:04 +0300165
Avi Kivity6aa8b732006-12-10 02:21:36 -0800166static DEFINE_PER_CPU(struct vmcs *, vmxarea);
167static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
Avi Kivity543e4242008-05-13 16:22:47 +0300168static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800169
Avi Kivity3e7c73e2009-02-24 21:46:19 +0200170static unsigned long *vmx_io_bitmap_a;
171static unsigned long *vmx_io_bitmap_b;
Avi Kivity58972972009-02-24 22:26:47 +0200172static unsigned long *vmx_msr_bitmap_legacy;
173static unsigned long *vmx_msr_bitmap_longmode;
He, Qingfdef3ad2007-04-30 09:45:24 +0300174
Sheng Yang2384d2b2008-01-17 15:14:33 +0800175static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
176static DEFINE_SPINLOCK(vmx_vpid_lock);
177
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300178static struct vmcs_config {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800179 int size;
180 int order;
181 u32 revision_id;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300182 u32 pin_based_exec_ctrl;
183 u32 cpu_based_exec_ctrl;
Sheng Yangf78e0e22007-10-29 09:40:42 +0800184 u32 cpu_based_2nd_exec_ctrl;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +0300185 u32 vmexit_ctrl;
186 u32 vmentry_ctrl;
187} vmcs_config;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800188
Hannes Ederefff9e52008-11-28 17:02:06 +0100189static struct vmx_capability {
Sheng Yangd56f5462008-04-25 10:13:16 +0800190 u32 ept;
191 u32 vpid;
192} vmx_capability;
193
Avi Kivity6aa8b732006-12-10 02:21:36 -0800194#define VMX_SEGMENT_FIELD(seg) \
195 [VCPU_SREG_##seg] = { \
196 .selector = GUEST_##seg##_SELECTOR, \
197 .base = GUEST_##seg##_BASE, \
198 .limit = GUEST_##seg##_LIMIT, \
199 .ar_bytes = GUEST_##seg##_AR_BYTES, \
200 }
201
202static struct kvm_vmx_segment_field {
203 unsigned selector;
204 unsigned base;
205 unsigned limit;
206 unsigned ar_bytes;
207} kvm_vmx_segment_fields[] = {
208 VMX_SEGMENT_FIELD(CS),
209 VMX_SEGMENT_FIELD(DS),
210 VMX_SEGMENT_FIELD(ES),
211 VMX_SEGMENT_FIELD(FS),
212 VMX_SEGMENT_FIELD(GS),
213 VMX_SEGMENT_FIELD(SS),
214 VMX_SEGMENT_FIELD(TR),
215 VMX_SEGMENT_FIELD(LDTR),
216};
217
Avi Kivity26bb0982009-09-07 11:14:12 +0300218static u64 host_efer;
219
Avi Kivity6de4f3a2009-05-31 22:58:47 +0300220static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
221
Avi Kivity4d56c8a2007-04-19 14:28:44 +0300222/*
223 * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
224 * away by decrementing the array size.
225 */
Avi Kivity6aa8b732006-12-10 02:21:36 -0800226static const u32 vmx_msr_index[] = {
Avi Kivity05b3e0c2006-12-13 00:33:45 -0800227#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300228 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
Avi Kivity6aa8b732006-12-10 02:21:36 -0800229#endif
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800230 MSR_EFER, MSR_TSC_AUX, MSR_K6_STAR,
Avi Kivity6aa8b732006-12-10 02:21:36 -0800231};
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +0200232#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800233
Avi Kivity6aa8b732006-12-10 02:21:36 -0800234static inline int is_page_fault(u32 intr_info)
235{
236 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
237 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100238 (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800239}
240
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300241static inline int is_no_device(u32 intr_info)
242{
243 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
244 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100245 (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300246}
247
Anthony Liguori7aa81cc2007-09-17 14:57:50 -0500248static inline int is_invalid_opcode(u32 intr_info)
249{
250 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
251 INTR_INFO_VALID_MASK)) ==
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100252 (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -0500253}
254
Avi Kivity6aa8b732006-12-10 02:21:36 -0800255static inline int is_external_interrupt(u32 intr_info)
256{
257 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
258 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
259}
260
Andi Kleena0861c02009-06-08 17:37:09 +0800261static inline int is_machine_check(u32 intr_info)
262{
263 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
264 INTR_INFO_VALID_MASK)) ==
265 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
266}
267
Sheng Yang25c5f222008-03-28 13:18:56 +0800268static inline int cpu_has_vmx_msr_bitmap(void)
269{
Sheng Yang04547152009-04-01 15:52:31 +0800270 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
Sheng Yang25c5f222008-03-28 13:18:56 +0800271}
272
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800273static inline int cpu_has_vmx_tpr_shadow(void)
274{
Sheng Yang04547152009-04-01 15:52:31 +0800275 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800276}
277
278static inline int vm_need_tpr_shadow(struct kvm *kvm)
279{
Sheng Yang04547152009-04-01 15:52:31 +0800280 return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
Yang, Sheng6e5d8652007-09-12 18:03:11 +0800281}
282
Sheng Yangf78e0e22007-10-29 09:40:42 +0800283static inline int cpu_has_secondary_exec_ctrls(void)
284{
Sheng Yang04547152009-04-01 15:52:31 +0800285 return vmcs_config.cpu_based_exec_ctrl &
286 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
Sheng Yangf78e0e22007-10-29 09:40:42 +0800287}
288
Avi Kivity774ead32007-12-26 13:57:04 +0200289static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800290{
Sheng Yang04547152009-04-01 15:52:31 +0800291 return vmcs_config.cpu_based_2nd_exec_ctrl &
292 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
293}
294
295static inline bool cpu_has_vmx_flexpriority(void)
296{
297 return cpu_has_vmx_tpr_shadow() &&
298 cpu_has_vmx_virtualize_apic_accesses();
Sheng Yangf78e0e22007-10-29 09:40:42 +0800299}
300
Marcelo Tosattie7997942009-06-11 12:07:40 -0300301static inline bool cpu_has_vmx_ept_execute_only(void)
302{
303 return !!(vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT);
304}
305
306static inline bool cpu_has_vmx_eptp_uncacheable(void)
307{
308 return !!(vmx_capability.ept & VMX_EPTP_UC_BIT);
309}
310
311static inline bool cpu_has_vmx_eptp_writeback(void)
312{
313 return !!(vmx_capability.ept & VMX_EPTP_WB_BIT);
314}
315
316static inline bool cpu_has_vmx_ept_2m_page(void)
317{
318 return !!(vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT);
319}
320
Sheng Yang878403b2010-01-05 19:02:29 +0800321static inline bool cpu_has_vmx_ept_1g_page(void)
322{
323 return !!(vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT);
324}
325
Sheng Yangd56f5462008-04-25 10:13:16 +0800326static inline int cpu_has_vmx_invept_individual_addr(void)
327{
Sheng Yang04547152009-04-01 15:52:31 +0800328 return !!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT);
Sheng Yangd56f5462008-04-25 10:13:16 +0800329}
330
331static inline int cpu_has_vmx_invept_context(void)
332{
Sheng Yang04547152009-04-01 15:52:31 +0800333 return !!(vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT);
Sheng Yangd56f5462008-04-25 10:13:16 +0800334}
335
336static inline int cpu_has_vmx_invept_global(void)
337{
Sheng Yang04547152009-04-01 15:52:31 +0800338 return !!(vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT);
Sheng Yangd56f5462008-04-25 10:13:16 +0800339}
340
341static inline int cpu_has_vmx_ept(void)
342{
Sheng Yang04547152009-04-01 15:52:31 +0800343 return vmcs_config.cpu_based_2nd_exec_ctrl &
344 SECONDARY_EXEC_ENABLE_EPT;
Sheng Yangd56f5462008-04-25 10:13:16 +0800345}
346
Nitin A Kamble3a624e22009-06-08 11:34:16 -0700347static inline int cpu_has_vmx_unrestricted_guest(void)
348{
349 return vmcs_config.cpu_based_2nd_exec_ctrl &
350 SECONDARY_EXEC_UNRESTRICTED_GUEST;
351}
352
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +0800353static inline int cpu_has_vmx_ple(void)
354{
355 return vmcs_config.cpu_based_2nd_exec_ctrl &
356 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
357}
358
Sheng Yangf78e0e22007-10-29 09:40:42 +0800359static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm)
360{
Sheng Yang04547152009-04-01 15:52:31 +0800361 return flexpriority_enabled &&
362 (cpu_has_vmx_virtualize_apic_accesses()) &&
363 (irqchip_in_kernel(kvm));
Sheng Yangf78e0e22007-10-29 09:40:42 +0800364}
365
Sheng Yang2384d2b2008-01-17 15:14:33 +0800366static inline int cpu_has_vmx_vpid(void)
367{
Sheng Yang04547152009-04-01 15:52:31 +0800368 return vmcs_config.cpu_based_2nd_exec_ctrl &
369 SECONDARY_EXEC_ENABLE_VPID;
Sheng Yang2384d2b2008-01-17 15:14:33 +0800370}
371
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800372static inline int cpu_has_vmx_rdtscp(void)
373{
374 return vmcs_config.cpu_based_2nd_exec_ctrl &
375 SECONDARY_EXEC_RDTSCP;
376}
377
Sheng Yangf08864b2008-05-15 18:23:25 +0800378static inline int cpu_has_virtual_nmis(void)
379{
380 return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
381}
382
Sheng Yang04547152009-04-01 15:52:31 +0800383static inline bool report_flexpriority(void)
384{
385 return flexpriority_enabled;
386}
387
Rusty Russell8b9cf982007-07-30 16:31:43 +1000388static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
Avi Kivity7725f0b2006-12-13 00:34:01 -0800389{
390 int i;
391
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400392 for (i = 0; i < vmx->nmsrs; ++i)
Avi Kivity26bb0982009-09-07 11:14:12 +0300393 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
Eddie Donga75beee2007-05-17 18:55:15 +0300394 return i;
395 return -1;
396}
397
Sheng Yang2384d2b2008-01-17 15:14:33 +0800398static inline void __invvpid(int ext, u16 vpid, gva_t gva)
399{
400 struct {
401 u64 vpid : 16;
402 u64 rsvd : 48;
403 u64 gva;
404 } operand = { vpid, 0, gva };
405
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300406 asm volatile (__ex(ASM_VMX_INVVPID)
Sheng Yang2384d2b2008-01-17 15:14:33 +0800407 /* CF==1 or ZF==1 --> rc = -1 */
408 "; ja 1f ; ud2 ; 1:"
409 : : "a"(&operand), "c"(ext) : "cc", "memory");
410}
411
Sheng Yang14394422008-04-28 12:24:45 +0800412static inline void __invept(int ext, u64 eptp, gpa_t gpa)
413{
414 struct {
415 u64 eptp, gpa;
416 } operand = {eptp, gpa};
417
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300418 asm volatile (__ex(ASM_VMX_INVEPT)
Sheng Yang14394422008-04-28 12:24:45 +0800419 /* CF==1 or ZF==1 --> rc = -1 */
420 "; ja 1f ; ud2 ; 1:\n"
421 : : "a" (&operand), "c" (ext) : "cc", "memory");
422}
423
Avi Kivity26bb0982009-09-07 11:14:12 +0300424static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
Eddie Donga75beee2007-05-17 18:55:15 +0300425{
426 int i;
427
Rusty Russell8b9cf982007-07-30 16:31:43 +1000428 i = __find_msr_index(vmx, msr);
Eddie Donga75beee2007-05-17 18:55:15 +0300429 if (i >= 0)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400430 return &vmx->guest_msrs[i];
Al Viro8b6d44c2007-02-09 16:38:40 +0000431 return NULL;
Avi Kivity7725f0b2006-12-13 00:34:01 -0800432}
433
Avi Kivity6aa8b732006-12-10 02:21:36 -0800434static void vmcs_clear(struct vmcs *vmcs)
435{
436 u64 phys_addr = __pa(vmcs);
437 u8 error;
438
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300439 asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
Avi Kivity6aa8b732006-12-10 02:21:36 -0800440 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
441 : "cc", "memory");
442 if (error)
443 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
444 vmcs, phys_addr);
445}
446
447static void __vcpu_clear(void *arg)
448{
Rusty Russell8b9cf982007-07-30 16:31:43 +1000449 struct vcpu_vmx *vmx = arg;
Ingo Molnard3b2c332007-01-05 16:36:23 -0800450 int cpu = raw_smp_processor_id();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800451
Rusty Russell8b9cf982007-07-30 16:31:43 +1000452 if (vmx->vcpu.cpu == cpu)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400453 vmcs_clear(vmx->vmcs);
454 if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800455 per_cpu(current_vmcs, cpu) = NULL;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800456 rdtscll(vmx->vcpu.arch.host_tsc);
Avi Kivity543e4242008-05-13 16:22:47 +0300457 list_del(&vmx->local_vcpus_link);
458 vmx->vcpu.cpu = -1;
459 vmx->launched = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800460}
461
Rusty Russell8b9cf982007-07-30 16:31:43 +1000462static void vcpu_clear(struct vcpu_vmx *vmx)
Avi Kivity8d0be2b2007-02-12 00:54:46 -0800463{
Avi Kivityeae5ecb2007-09-30 10:50:12 +0200464 if (vmx->vcpu.cpu == -1)
465 return;
Jens Axboe8691e5a2008-06-06 11:18:06 +0200466 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
Avi Kivity8d0be2b2007-02-12 00:54:46 -0800467}
468
Sheng Yang2384d2b2008-01-17 15:14:33 +0800469static inline void vpid_sync_vcpu_all(struct vcpu_vmx *vmx)
470{
471 if (vmx->vpid == 0)
472 return;
473
474 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
475}
476
Sheng Yang14394422008-04-28 12:24:45 +0800477static inline void ept_sync_global(void)
478{
479 if (cpu_has_vmx_invept_global())
480 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
481}
482
483static inline void ept_sync_context(u64 eptp)
484{
Avi Kivity089d0342009-03-23 18:26:32 +0200485 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +0800486 if (cpu_has_vmx_invept_context())
487 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
488 else
489 ept_sync_global();
490 }
491}
492
493static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
494{
Avi Kivity089d0342009-03-23 18:26:32 +0200495 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +0800496 if (cpu_has_vmx_invept_individual_addr())
497 __invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
498 eptp, gpa);
499 else
500 ept_sync_context(eptp);
501 }
502}
503
Avi Kivity6aa8b732006-12-10 02:21:36 -0800504static unsigned long vmcs_readl(unsigned long field)
505{
506 unsigned long value;
507
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300508 asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800509 : "=a"(value) : "d"(field) : "cc");
510 return value;
511}
512
513static u16 vmcs_read16(unsigned long field)
514{
515 return vmcs_readl(field);
516}
517
518static u32 vmcs_read32(unsigned long field)
519{
520 return vmcs_readl(field);
521}
522
523static u64 vmcs_read64(unsigned long field)
524{
Avi Kivity05b3e0c2006-12-13 00:33:45 -0800525#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -0800526 return vmcs_readl(field);
527#else
528 return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
529#endif
530}
531
Avi Kivitye52de1b2007-01-05 16:36:56 -0800532static noinline void vmwrite_error(unsigned long field, unsigned long value)
533{
534 printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
535 field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
536 dump_stack();
537}
538
Avi Kivity6aa8b732006-12-10 02:21:36 -0800539static void vmcs_writel(unsigned long field, unsigned long value)
540{
541 u8 error;
542
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300543 asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
Mike Dayd77c26f2007-10-08 09:02:08 -0400544 : "=q"(error) : "a"(value), "d"(field) : "cc");
Avi Kivitye52de1b2007-01-05 16:36:56 -0800545 if (unlikely(error))
546 vmwrite_error(field, value);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800547}
548
549static void vmcs_write16(unsigned long field, u16 value)
550{
551 vmcs_writel(field, value);
552}
553
554static void vmcs_write32(unsigned long field, u32 value)
555{
556 vmcs_writel(field, value);
557}
558
559static void vmcs_write64(unsigned long field, u64 value)
560{
Avi Kivity6aa8b732006-12-10 02:21:36 -0800561 vmcs_writel(field, value);
Avi Kivity7682f2d2008-05-12 19:25:43 +0300562#ifndef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -0800563 asm volatile ("");
564 vmcs_writel(field+1, value >> 32);
565#endif
566}
567
Anthony Liguori2ab455c2007-04-27 09:29:49 +0300568static void vmcs_clear_bits(unsigned long field, u32 mask)
569{
570 vmcs_writel(field, vmcs_readl(field) & ~mask);
571}
572
573static void vmcs_set_bits(unsigned long field, u32 mask)
574{
575 vmcs_writel(field, vmcs_readl(field) | mask);
576}
577
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300578static void update_exception_bitmap(struct kvm_vcpu *vcpu)
579{
580 u32 eb;
581
Jan Kiszkafd7373c2010-01-20 18:20:20 +0100582 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
583 (1u << NM_VECTOR) | (1u << DB_VECTOR);
584 if ((vcpu->guest_debug &
585 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
586 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
587 eb |= 1u << BP_VECTOR;
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300588 if (to_vmx(vcpu)->rmode.vm86_active)
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300589 eb = ~0;
Avi Kivity089d0342009-03-23 18:26:32 +0200590 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +0800591 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
Avi Kivity02daab22009-12-30 12:40:26 +0200592 if (vcpu->fpu_active)
593 eb &= ~(1u << NM_VECTOR);
Avi Kivityabd3f2d2007-05-02 17:57:40 +0300594 vmcs_write32(EXCEPTION_BITMAP, eb);
595}
596
Avi Kivity33ed6322007-05-02 16:54:03 +0300597static void reload_tss(void)
598{
Avi Kivity33ed6322007-05-02 16:54:03 +0300599 /*
600 * VT restores TR but not its size. Useless.
601 */
602 struct descriptor_table gdt;
Avi Kivitya5f61302008-02-20 17:57:21 +0200603 struct desc_struct *descs;
Avi Kivity33ed6322007-05-02 16:54:03 +0300604
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300605 kvm_get_gdt(&gdt);
Avi Kivity33ed6322007-05-02 16:54:03 +0300606 descs = (void *)gdt.base;
607 descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
608 load_TR_desc();
Avi Kivity33ed6322007-05-02 16:54:03 +0300609}
610
Avi Kivity92c0d902009-10-29 11:00:16 +0200611static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
Eddie Dong2cc51562007-05-21 07:28:09 +0300612{
Roel Kluin3a34a882009-08-04 02:08:45 -0700613 u64 guest_efer;
Avi Kivity51c6cf62007-08-29 03:48:05 +0300614 u64 ignore_bits;
Eddie Dong2cc51562007-05-21 07:28:09 +0300615
Avi Kivityf6801df2010-01-21 15:31:50 +0200616 guest_efer = vmx->vcpu.arch.efer;
Roel Kluin3a34a882009-08-04 02:08:45 -0700617
Avi Kivity51c6cf62007-08-29 03:48:05 +0300618 /*
619 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
620 * outside long mode
621 */
622 ignore_bits = EFER_NX | EFER_SCE;
623#ifdef CONFIG_X86_64
624 ignore_bits |= EFER_LMA | EFER_LME;
625 /* SCE is meaningful only in long mode on Intel */
626 if (guest_efer & EFER_LMA)
627 ignore_bits &= ~(u64)EFER_SCE;
628#endif
Avi Kivity51c6cf62007-08-29 03:48:05 +0300629 guest_efer &= ~ignore_bits;
630 guest_efer |= host_efer & ignore_bits;
Avi Kivity26bb0982009-09-07 11:14:12 +0300631 vmx->guest_msrs[efer_offset].data = guest_efer;
Avi Kivityd5696722009-12-02 12:28:47 +0200632 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
Avi Kivity26bb0982009-09-07 11:14:12 +0300633 return true;
Avi Kivity51c6cf62007-08-29 03:48:05 +0300634}
635
Avi Kivity04d2cc72007-09-10 18:10:54 +0300636static void vmx_save_host_state(struct kvm_vcpu *vcpu)
Avi Kivity33ed6322007-05-02 16:54:03 +0300637{
Avi Kivity04d2cc72007-09-10 18:10:54 +0300638 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +0300639 int i;
Avi Kivity04d2cc72007-09-10 18:10:54 +0300640
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400641 if (vmx->host_state.loaded)
Avi Kivity33ed6322007-05-02 16:54:03 +0300642 return;
643
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400644 vmx->host_state.loaded = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300645 /*
646 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
647 * allow segment selectors with cpl > 0 or ti == 1.
648 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300649 vmx->host_state.ldt_sel = kvm_read_ldt();
Laurent Vivier152d3f22007-08-23 16:33:11 +0200650 vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300651 vmx->host_state.fs_sel = kvm_read_fs();
Laurent Vivier152d3f22007-08-23 16:33:11 +0200652 if (!(vmx->host_state.fs_sel & 7)) {
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400653 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200654 vmx->host_state.fs_reload_needed = 0;
655 } else {
Avi Kivity33ed6322007-05-02 16:54:03 +0300656 vmcs_write16(HOST_FS_SELECTOR, 0);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200657 vmx->host_state.fs_reload_needed = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300658 }
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300659 vmx->host_state.gs_sel = kvm_read_gs();
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400660 if (!(vmx->host_state.gs_sel & 7))
661 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300662 else {
663 vmcs_write16(HOST_GS_SELECTOR, 0);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200664 vmx->host_state.gs_ldt_reload_needed = 1;
Avi Kivity33ed6322007-05-02 16:54:03 +0300665 }
666
667#ifdef CONFIG_X86_64
668 vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
669 vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
670#else
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400671 vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
672 vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
Avi Kivity33ed6322007-05-02 16:54:03 +0300673#endif
Avi Kivity707c0872007-05-02 17:33:43 +0300674
675#ifdef CONFIG_X86_64
Avi Kivity44ea2b12009-09-06 15:55:37 +0300676 if (is_long_mode(&vmx->vcpu)) {
677 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
678 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
679 }
Avi Kivity707c0872007-05-02 17:33:43 +0300680#endif
Avi Kivity26bb0982009-09-07 11:14:12 +0300681 for (i = 0; i < vmx->save_nmsrs; ++i)
682 kvm_set_shared_msr(vmx->guest_msrs[i].index,
Avi Kivityd5696722009-12-02 12:28:47 +0200683 vmx->guest_msrs[i].data,
684 vmx->guest_msrs[i].mask);
Avi Kivity33ed6322007-05-02 16:54:03 +0300685}
686
Avi Kivitya9b21b62008-06-24 11:48:49 +0300687static void __vmx_load_host_state(struct vcpu_vmx *vmx)
Avi Kivity33ed6322007-05-02 16:54:03 +0300688{
Avi Kivity15ad7142007-07-11 18:17:21 +0300689 unsigned long flags;
Avi Kivity33ed6322007-05-02 16:54:03 +0300690
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400691 if (!vmx->host_state.loaded)
Avi Kivity33ed6322007-05-02 16:54:03 +0300692 return;
693
Avi Kivitye1beb1d2007-11-18 13:50:24 +0200694 ++vmx->vcpu.stat.host_state_reload;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400695 vmx->host_state.loaded = 0;
Laurent Vivier152d3f22007-08-23 16:33:11 +0200696 if (vmx->host_state.fs_reload_needed)
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300697 kvm_load_fs(vmx->host_state.fs_sel);
Laurent Vivier152d3f22007-08-23 16:33:11 +0200698 if (vmx->host_state.gs_ldt_reload_needed) {
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300699 kvm_load_ldt(vmx->host_state.ldt_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300700 /*
701 * If we have to reload gs, we must take care to
702 * preserve our gs base.
703 */
Avi Kivity15ad7142007-07-11 18:17:21 +0300704 local_irq_save(flags);
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300705 kvm_load_gs(vmx->host_state.gs_sel);
Avi Kivity33ed6322007-05-02 16:54:03 +0300706#ifdef CONFIG_X86_64
707 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
708#endif
Avi Kivity15ad7142007-07-11 18:17:21 +0300709 local_irq_restore(flags);
Avi Kivity33ed6322007-05-02 16:54:03 +0300710 }
Laurent Vivier152d3f22007-08-23 16:33:11 +0200711 reload_tss();
Avi Kivity44ea2b12009-09-06 15:55:37 +0300712#ifdef CONFIG_X86_64
713 if (is_long_mode(&vmx->vcpu)) {
714 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
715 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
716 }
717#endif
Avi Kivity33ed6322007-05-02 16:54:03 +0300718}
719
Avi Kivitya9b21b62008-06-24 11:48:49 +0300720static void vmx_load_host_state(struct vcpu_vmx *vmx)
721{
722 preempt_disable();
723 __vmx_load_host_state(vmx);
724 preempt_enable();
725}
726
Avi Kivity6aa8b732006-12-10 02:21:36 -0800727/*
728 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
729 * vcpu mutex is already taken.
730 */
Avi Kivity15ad7142007-07-11 18:17:21 +0300731static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800732{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400733 struct vcpu_vmx *vmx = to_vmx(vcpu);
734 u64 phys_addr = __pa(vmx->vmcs);
Avi Kivity019960a2008-03-04 10:44:51 +0200735 u64 tsc_this, delta, new_offset;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800736
Eddie Donga3d7f852007-09-03 16:15:12 +0300737 if (vcpu->cpu != cpu) {
Rusty Russell8b9cf982007-07-30 16:31:43 +1000738 vcpu_clear(vmx);
Marcelo Tosatti2f599712008-05-27 12:10:20 -0300739 kvm_migrate_timers(vcpu);
Marcelo Tosattieb5109e2009-10-01 19:16:58 -0300740 set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests);
Avi Kivity543e4242008-05-13 16:22:47 +0300741 local_irq_disable();
742 list_add(&vmx->local_vcpus_link,
743 &per_cpu(vcpus_on_cpu, cpu));
744 local_irq_enable();
Eddie Donga3d7f852007-09-03 16:15:12 +0300745 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800746
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400747 if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800748 u8 error;
749
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400750 per_cpu(current_vmcs, cpu) = vmx->vmcs;
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300751 asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
Avi Kivity6aa8b732006-12-10 02:21:36 -0800752 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
753 : "cc");
754 if (error)
755 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400756 vmx->vmcs, phys_addr);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800757 }
758
759 if (vcpu->cpu != cpu) {
760 struct descriptor_table dt;
761 unsigned long sysenter_esp;
762
763 vcpu->cpu = cpu;
764 /*
765 * Linux uses per-cpu TSS and GDT, so set these when switching
766 * processors.
767 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +0300768 vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
769 kvm_get_gdt(&dt);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800770 vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */
771
772 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
773 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
Avi Kivity77002702007-06-13 19:55:28 +0300774
775 /*
776 * Make sure the time stamp counter is monotonous.
777 */
778 rdtscll(tsc_this);
Avi Kivity019960a2008-03-04 10:44:51 +0200779 if (tsc_this < vcpu->arch.host_tsc) {
780 delta = vcpu->arch.host_tsc - tsc_this;
781 new_offset = vmcs_read64(TSC_OFFSET) + delta;
782 vmcs_write64(TSC_OFFSET, new_offset);
783 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800784 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800785}
786
787static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
788{
Avi Kivitya9b21b62008-06-24 11:48:49 +0300789 __vmx_load_host_state(to_vmx(vcpu));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800790}
791
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300792static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
793{
Avi Kivity81231c62010-01-24 16:26:40 +0200794 ulong cr0;
795
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300796 if (vcpu->fpu_active)
797 return;
798 vcpu->fpu_active = 1;
Avi Kivity81231c62010-01-24 16:26:40 +0200799 cr0 = vmcs_readl(GUEST_CR0);
800 cr0 &= ~(X86_CR0_TS | X86_CR0_MP);
801 cr0 |= kvm_read_cr0_bits(vcpu, X86_CR0_TS | X86_CR0_MP);
802 vmcs_writel(GUEST_CR0, cr0);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300803 update_exception_bitmap(vcpu);
Avi Kivityedcafe32009-12-30 18:07:40 +0200804 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
805 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300806}
807
Avi Kivityedcafe32009-12-30 18:07:40 +0200808static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
809
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300810static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
811{
Avi Kivityedcafe32009-12-30 18:07:40 +0200812 vmx_decache_cr0_guest_bits(vcpu);
Avi Kivity81231c62010-01-24 16:26:40 +0200813 vmcs_set_bits(GUEST_CR0, X86_CR0_TS | X86_CR0_MP);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300814 update_exception_bitmap(vcpu);
Avi Kivityedcafe32009-12-30 18:07:40 +0200815 vcpu->arch.cr0_guest_owned_bits = 0;
816 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
817 vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
Avi Kivity5fd86fc2007-05-02 20:40:00 +0300818}
819
Avi Kivity6aa8b732006-12-10 02:21:36 -0800820static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
821{
Avi Kivity345dcaa2009-08-12 15:29:37 +0300822 unsigned long rflags;
823
824 rflags = vmcs_readl(GUEST_RFLAGS);
825 if (to_vmx(vcpu)->rmode.vm86_active)
826 rflags &= ~(unsigned long)(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
827 return rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800828}
829
830static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
831{
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300832 if (to_vmx(vcpu)->rmode.vm86_active)
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100833 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800834 vmcs_writel(GUEST_RFLAGS, rflags);
835}
836
Glauber Costa2809f5d2009-05-12 16:21:05 -0400837static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
838{
839 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
840 int ret = 0;
841
842 if (interruptibility & GUEST_INTR_STATE_STI)
843 ret |= X86_SHADOW_INT_STI;
844 if (interruptibility & GUEST_INTR_STATE_MOV_SS)
845 ret |= X86_SHADOW_INT_MOV_SS;
846
847 return ret & mask;
848}
849
850static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
851{
852 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
853 u32 interruptibility = interruptibility_old;
854
855 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
856
857 if (mask & X86_SHADOW_INT_MOV_SS)
858 interruptibility |= GUEST_INTR_STATE_MOV_SS;
859 if (mask & X86_SHADOW_INT_STI)
860 interruptibility |= GUEST_INTR_STATE_STI;
861
862 if ((interruptibility != interruptibility_old))
863 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
864}
865
Avi Kivity6aa8b732006-12-10 02:21:36 -0800866static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
867{
868 unsigned long rip;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800869
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300870 rip = kvm_rip_read(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800871 rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -0300872 kvm_rip_write(vcpu, rip);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800873
Glauber Costa2809f5d2009-05-12 16:21:05 -0400874 /* skipping an emulated instruction also counts */
875 vmx_set_interrupt_shadow(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800876}
877
Avi Kivity298101d2007-11-25 13:41:11 +0200878static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
879 bool has_error_code, u32 error_code)
880{
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200881 struct vcpu_vmx *vmx = to_vmx(vcpu);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100882 u32 intr_info = nr | INTR_INFO_VALID_MASK;
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200883
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100884 if (has_error_code) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200885 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100886 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
887 }
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200888
Avi Kivity7ffd92c2009-06-09 14:10:45 +0300889 if (vmx->rmode.vm86_active) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200890 vmx->rmode.irq.pending = true;
891 vmx->rmode.irq.vector = nr;
892 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
Gleb Natapovae0bb3e2009-05-19 11:07:10 +0300893 if (kvm_exception_is_soft(nr))
894 vmx->rmode.irq.rip +=
895 vmx->vcpu.arch.event_exit_inst_len;
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100896 intr_info |= INTR_TYPE_SOFT_INTR;
897 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
Jan Kiszka77ab6db2008-07-14 12:28:51 +0200898 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
899 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
900 return;
901 }
902
Gleb Natapov66fd3f72009-05-11 13:35:50 +0300903 if (kvm_exception_is_soft(nr)) {
904 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
905 vmx->vcpu.arch.event_exit_inst_len);
Jan Kiszka8ab2d2e2008-12-15 13:52:10 +0100906 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
907 } else
908 intr_info |= INTR_TYPE_HARD_EXCEPTION;
909
910 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
Avi Kivity298101d2007-11-25 13:41:11 +0200911}
912
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800913static bool vmx_rdtscp_supported(void)
914{
915 return cpu_has_vmx_rdtscp();
916}
917
Avi Kivity6aa8b732006-12-10 02:21:36 -0800918/*
Eddie Donga75beee2007-05-17 18:55:15 +0300919 * Swap MSR entry in host/guest MSR entry array.
920 */
Rusty Russell8b9cf982007-07-30 16:31:43 +1000921static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
Eddie Donga75beee2007-05-17 18:55:15 +0300922{
Avi Kivity26bb0982009-09-07 11:14:12 +0300923 struct shared_msr_entry tmp;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400924
925 tmp = vmx->guest_msrs[to];
926 vmx->guest_msrs[to] = vmx->guest_msrs[from];
927 vmx->guest_msrs[from] = tmp;
Eddie Donga75beee2007-05-17 18:55:15 +0300928}
929
930/*
Avi Kivitye38aea32007-04-19 13:22:48 +0300931 * Set up the vmcs to automatically save and restore system
932 * msrs. Don't touch the 64-bit msrs if the guest is in legacy
933 * mode, as fiddling with msrs is very expensive.
934 */
Rusty Russell8b9cf982007-07-30 16:31:43 +1000935static void setup_msrs(struct vcpu_vmx *vmx)
Avi Kivitye38aea32007-04-19 13:22:48 +0300936{
Avi Kivity26bb0982009-09-07 11:14:12 +0300937 int save_nmsrs, index;
Avi Kivity58972972009-02-24 22:26:47 +0200938 unsigned long *msr_bitmap;
Avi Kivitye38aea32007-04-19 13:22:48 +0300939
Avi Kivity33f9c502008-02-27 16:06:57 +0200940 vmx_load_host_state(vmx);
Eddie Donga75beee2007-05-17 18:55:15 +0300941 save_nmsrs = 0;
Avi Kivity4d56c8a2007-04-19 14:28:44 +0300942#ifdef CONFIG_X86_64
Rusty Russell8b9cf982007-07-30 16:31:43 +1000943 if (is_long_mode(&vmx->vcpu)) {
Rusty Russell8b9cf982007-07-30 16:31:43 +1000944 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
Eddie Donga75beee2007-05-17 18:55:15 +0300945 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +1000946 move_msr_up(vmx, index, save_nmsrs++);
947 index = __find_msr_index(vmx, MSR_LSTAR);
Eddie Donga75beee2007-05-17 18:55:15 +0300948 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +1000949 move_msr_up(vmx, index, save_nmsrs++);
950 index = __find_msr_index(vmx, MSR_CSTAR);
Eddie Donga75beee2007-05-17 18:55:15 +0300951 if (index >= 0)
Rusty Russell8b9cf982007-07-30 16:31:43 +1000952 move_msr_up(vmx, index, save_nmsrs++);
Sheng Yang4e47c7a2009-12-18 16:48:47 +0800953 index = __find_msr_index(vmx, MSR_TSC_AUX);
954 if (index >= 0 && vmx->rdtscp_enabled)
955 move_msr_up(vmx, index, save_nmsrs++);
Eddie Donga75beee2007-05-17 18:55:15 +0300956 /*
957 * MSR_K6_STAR is only needed on long mode guests, and only
958 * if efer.sce is enabled.
959 */
Rusty Russell8b9cf982007-07-30 16:31:43 +1000960 index = __find_msr_index(vmx, MSR_K6_STAR);
Avi Kivityf6801df2010-01-21 15:31:50 +0200961 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
Rusty Russell8b9cf982007-07-30 16:31:43 +1000962 move_msr_up(vmx, index, save_nmsrs++);
Avi Kivity4d56c8a2007-04-19 14:28:44 +0300963 }
Eddie Donga75beee2007-05-17 18:55:15 +0300964#endif
Avi Kivity92c0d902009-10-29 11:00:16 +0200965 index = __find_msr_index(vmx, MSR_EFER);
966 if (index >= 0 && update_transition_efer(vmx, index))
Avi Kivity26bb0982009-09-07 11:14:12 +0300967 move_msr_up(vmx, index, save_nmsrs++);
Avi Kivity4d56c8a2007-04-19 14:28:44 +0300968
Avi Kivity26bb0982009-09-07 11:14:12 +0300969 vmx->save_nmsrs = save_nmsrs;
Avi Kivity58972972009-02-24 22:26:47 +0200970
971 if (cpu_has_vmx_msr_bitmap()) {
972 if (is_long_mode(&vmx->vcpu))
973 msr_bitmap = vmx_msr_bitmap_longmode;
974 else
975 msr_bitmap = vmx_msr_bitmap_legacy;
976
977 vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
978 }
Avi Kivitye38aea32007-04-19 13:22:48 +0300979}
980
981/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800982 * reads and returns guest's timestamp counter "register"
983 * guest_tsc = host_tsc + tsc_offset -- 21.3
984 */
985static u64 guest_read_tsc(void)
986{
987 u64 host_tsc, tsc_offset;
988
989 rdtscll(host_tsc);
990 tsc_offset = vmcs_read64(TSC_OFFSET);
991 return host_tsc + tsc_offset;
992}
993
994/*
995 * writes 'guest_tsc' into guest's timestamp counter "register"
996 * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
997 */
Marcelo Tosatti53f658b2008-12-11 20:45:05 +0100998static void guest_write_tsc(u64 guest_tsc, u64 host_tsc)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800999{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001000 vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
1001}
1002
Avi Kivity6aa8b732006-12-10 02:21:36 -08001003/*
1004 * Reads an msr value (of 'msr_index') into 'pdata'.
1005 * Returns 0 on success, non-0 otherwise.
1006 * Assumes vcpu_load() was already called.
1007 */
1008static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1009{
1010 u64 data;
Avi Kivity26bb0982009-09-07 11:14:12 +03001011 struct shared_msr_entry *msr;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001012
1013 if (!pdata) {
1014 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
1015 return -EINVAL;
1016 }
1017
1018 switch (msr_index) {
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001019#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001020 case MSR_FS_BASE:
1021 data = vmcs_readl(GUEST_FS_BASE);
1022 break;
1023 case MSR_GS_BASE:
1024 data = vmcs_readl(GUEST_GS_BASE);
1025 break;
Avi Kivity44ea2b12009-09-06 15:55:37 +03001026 case MSR_KERNEL_GS_BASE:
1027 vmx_load_host_state(to_vmx(vcpu));
1028 data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
1029 break;
Avi Kivity26bb0982009-09-07 11:14:12 +03001030#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -08001031 case MSR_EFER:
Avi Kivity3bab1f52006-12-29 16:49:48 -08001032 return kvm_get_msr_common(vcpu, msr_index, pdata);
Jaswinder Singh Rajputaf24a4e2009-05-15 18:42:05 +05301033 case MSR_IA32_TSC:
Avi Kivity6aa8b732006-12-10 02:21:36 -08001034 data = guest_read_tsc();
1035 break;
1036 case MSR_IA32_SYSENTER_CS:
1037 data = vmcs_read32(GUEST_SYSENTER_CS);
1038 break;
1039 case MSR_IA32_SYSENTER_EIP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001040 data = vmcs_readl(GUEST_SYSENTER_EIP);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001041 break;
1042 case MSR_IA32_SYSENTER_ESP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001043 data = vmcs_readl(GUEST_SYSENTER_ESP);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001044 break;
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001045 case MSR_TSC_AUX:
1046 if (!to_vmx(vcpu)->rdtscp_enabled)
1047 return 1;
1048 /* Otherwise falls through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001049 default:
Avi Kivity26bb0982009-09-07 11:14:12 +03001050 vmx_load_host_state(to_vmx(vcpu));
Rusty Russell8b9cf982007-07-30 16:31:43 +10001051 msr = find_msr_entry(to_vmx(vcpu), msr_index);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001052 if (msr) {
Gleb Natapov542423b2009-08-27 15:07:30 +03001053 vmx_load_host_state(to_vmx(vcpu));
Avi Kivity3bab1f52006-12-29 16:49:48 -08001054 data = msr->data;
1055 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001056 }
Avi Kivity3bab1f52006-12-29 16:49:48 -08001057 return kvm_get_msr_common(vcpu, msr_index, pdata);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001058 }
1059
1060 *pdata = data;
1061 return 0;
1062}
1063
1064/*
1065 * Writes msr value into into the appropriate "register".
1066 * Returns 0 on success, non-0 otherwise.
1067 * Assumes vcpu_load() was already called.
1068 */
1069static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1070{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001071 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +03001072 struct shared_msr_entry *msr;
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01001073 u64 host_tsc;
Eddie Dong2cc51562007-05-21 07:28:09 +03001074 int ret = 0;
1075
Avi Kivity6aa8b732006-12-10 02:21:36 -08001076 switch (msr_index) {
Avi Kivity3bab1f52006-12-29 16:49:48 -08001077 case MSR_EFER:
Avi Kivitya9b21b62008-06-24 11:48:49 +03001078 vmx_load_host_state(vmx);
Eddie Dong2cc51562007-05-21 07:28:09 +03001079 ret = kvm_set_msr_common(vcpu, msr_index, data);
Eddie Dong2cc51562007-05-21 07:28:09 +03001080 break;
Avi Kivity16175a72009-03-23 22:13:44 +02001081#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001082 case MSR_FS_BASE:
1083 vmcs_writel(GUEST_FS_BASE, data);
1084 break;
1085 case MSR_GS_BASE:
1086 vmcs_writel(GUEST_GS_BASE, data);
1087 break;
Avi Kivity44ea2b12009-09-06 15:55:37 +03001088 case MSR_KERNEL_GS_BASE:
1089 vmx_load_host_state(vmx);
1090 vmx->msr_guest_kernel_gs_base = data;
1091 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001092#endif
1093 case MSR_IA32_SYSENTER_CS:
1094 vmcs_write32(GUEST_SYSENTER_CS, data);
1095 break;
1096 case MSR_IA32_SYSENTER_EIP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001097 vmcs_writel(GUEST_SYSENTER_EIP, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001098 break;
1099 case MSR_IA32_SYSENTER_ESP:
Avi Kivityf5b42c32007-03-06 12:05:53 +02001100 vmcs_writel(GUEST_SYSENTER_ESP, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001101 break;
Jaswinder Singh Rajputaf24a4e2009-05-15 18:42:05 +05301102 case MSR_IA32_TSC:
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01001103 rdtscll(host_tsc);
1104 guest_write_tsc(data, host_tsc);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001105 break;
Sheng Yang468d4722008-10-09 16:01:55 +08001106 case MSR_IA32_CR_PAT:
1107 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
1108 vmcs_write64(GUEST_IA32_PAT, data);
1109 vcpu->arch.pat = data;
1110 break;
1111 }
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001112 ret = kvm_set_msr_common(vcpu, msr_index, data);
1113 break;
1114 case MSR_TSC_AUX:
1115 if (!vmx->rdtscp_enabled)
1116 return 1;
1117 /* Check reserved bit, higher 32 bits should be zero */
1118 if ((data >> 32) != 0)
1119 return 1;
1120 /* Otherwise falls through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001121 default:
Rusty Russell8b9cf982007-07-30 16:31:43 +10001122 msr = find_msr_entry(vmx, msr_index);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001123 if (msr) {
Gleb Natapov542423b2009-08-27 15:07:30 +03001124 vmx_load_host_state(vmx);
Avi Kivity3bab1f52006-12-29 16:49:48 -08001125 msr->data = data;
1126 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001127 }
Eddie Dong2cc51562007-05-21 07:28:09 +03001128 ret = kvm_set_msr_common(vcpu, msr_index, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001129 }
1130
Eddie Dong2cc51562007-05-21 07:28:09 +03001131 return ret;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001132}
1133
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001134static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001135{
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001136 __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
1137 switch (reg) {
1138 case VCPU_REGS_RSP:
1139 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
1140 break;
1141 case VCPU_REGS_RIP:
1142 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
1143 break;
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001144 case VCPU_EXREG_PDPTR:
1145 if (enable_ept)
1146 ept_save_pdptrs(vcpu);
1147 break;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03001148 default:
1149 break;
1150 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001151}
1152
Jan Kiszka355be0b2009-10-03 00:31:21 +02001153static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001154{
Jan Kiszkaae675ef2008-12-15 13:52:10 +01001155 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1156 vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
1157 else
1158 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
1159
Avi Kivityabd3f2d2007-05-02 17:57:40 +03001160 update_exception_bitmap(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001161}
1162
1163static __init int cpu_has_kvm_support(void)
1164{
Eduardo Habkost6210e372008-11-17 19:03:16 -02001165 return cpu_has_vmx();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001166}
1167
1168static __init int vmx_disabled_by_bios(void)
1169{
1170 u64 msr;
1171
1172 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
Sheng Yang9ea542f2008-09-11 15:27:49 +08001173 return (msr & (FEATURE_CONTROL_LOCKED |
1174 FEATURE_CONTROL_VMXON_ENABLED))
1175 == FEATURE_CONTROL_LOCKED;
Yang, Sheng62b3ffb2007-07-25 12:17:06 +03001176 /* locked but not enabled */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001177}
1178
Alexander Graf10474ae2009-09-15 11:37:46 +02001179static int hardware_enable(void *garbage)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001180{
1181 int cpu = raw_smp_processor_id();
1182 u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1183 u64 old;
1184
Alexander Graf10474ae2009-09-15 11:37:46 +02001185 if (read_cr4() & X86_CR4_VMXE)
1186 return -EBUSY;
1187
Avi Kivity543e4242008-05-13 16:22:47 +03001188 INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001189 rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
Sheng Yang9ea542f2008-09-11 15:27:49 +08001190 if ((old & (FEATURE_CONTROL_LOCKED |
1191 FEATURE_CONTROL_VMXON_ENABLED))
1192 != (FEATURE_CONTROL_LOCKED |
1193 FEATURE_CONTROL_VMXON_ENABLED))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001194 /* enable and lock */
Yang, Sheng62b3ffb2007-07-25 12:17:06 +03001195 wrmsrl(MSR_IA32_FEATURE_CONTROL, old |
Sheng Yang9ea542f2008-09-11 15:27:49 +08001196 FEATURE_CONTROL_LOCKED |
1197 FEATURE_CONTROL_VMXON_ENABLED);
Rusty Russell66aee912007-07-17 23:34:16 +10001198 write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001199 asm volatile (ASM_VMX_VMXON_RAX
1200 : : "a"(&phys_addr), "m"(phys_addr)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001201 : "memory", "cc");
Alexander Graf10474ae2009-09-15 11:37:46 +02001202
1203 ept_sync_global();
1204
1205 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001206}
1207
Avi Kivity543e4242008-05-13 16:22:47 +03001208static void vmclear_local_vcpus(void)
1209{
1210 int cpu = raw_smp_processor_id();
1211 struct vcpu_vmx *vmx, *n;
1212
1213 list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
1214 local_vcpus_link)
1215 __vcpu_clear(vmx);
1216}
1217
Eduardo Habkost710ff4a2008-11-17 19:03:18 -02001218
1219/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
1220 * tricks.
1221 */
1222static void kvm_cpu_vmxoff(void)
1223{
1224 asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
1225 write_cr4(read_cr4() & ~X86_CR4_VMXE);
1226}
1227
Avi Kivity6aa8b732006-12-10 02:21:36 -08001228static void hardware_disable(void *garbage)
1229{
Avi Kivity543e4242008-05-13 16:22:47 +03001230 vmclear_local_vcpus();
Eduardo Habkost710ff4a2008-11-17 19:03:18 -02001231 kvm_cpu_vmxoff();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001232}
1233
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001234static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
Mike Dayd77c26f2007-10-08 09:02:08 -04001235 u32 msr, u32 *result)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001236{
1237 u32 vmx_msr_low, vmx_msr_high;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001238 u32 ctl = ctl_min | ctl_opt;
1239
1240 rdmsr(msr, vmx_msr_low, vmx_msr_high);
1241
1242 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
1243 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
1244
1245 /* Ensure minimum (required) set of control bits are supported. */
1246 if (ctl_min & ~ctl)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001247 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001248
1249 *result = ctl;
1250 return 0;
1251}
1252
Yang, Sheng002c7f72007-07-31 14:23:01 +03001253static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001254{
1255 u32 vmx_msr_low, vmx_msr_high;
Sheng Yangd56f5462008-04-25 10:13:16 +08001256 u32 min, opt, min2, opt2;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001257 u32 _pin_based_exec_control = 0;
1258 u32 _cpu_based_exec_control = 0;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001259 u32 _cpu_based_2nd_exec_control = 0;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001260 u32 _vmexit_control = 0;
1261 u32 _vmentry_control = 0;
1262
1263 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
Sheng Yangf08864b2008-05-15 18:23:25 +08001264 opt = PIN_BASED_VIRTUAL_NMIS;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001265 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
1266 &_pin_based_exec_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001267 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001268
1269 min = CPU_BASED_HLT_EXITING |
1270#ifdef CONFIG_X86_64
1271 CPU_BASED_CR8_LOAD_EXITING |
1272 CPU_BASED_CR8_STORE_EXITING |
1273#endif
Sheng Yangd56f5462008-04-25 10:13:16 +08001274 CPU_BASED_CR3_LOAD_EXITING |
1275 CPU_BASED_CR3_STORE_EXITING |
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001276 CPU_BASED_USE_IO_BITMAPS |
1277 CPU_BASED_MOV_DR_EXITING |
Marcelo Tosattia7052892008-09-23 13:18:35 -03001278 CPU_BASED_USE_TSC_OFFSETING |
Sheng Yang59708672009-12-15 13:29:54 +08001279 CPU_BASED_MWAIT_EXITING |
1280 CPU_BASED_MONITOR_EXITING |
Marcelo Tosattia7052892008-09-23 13:18:35 -03001281 CPU_BASED_INVLPG_EXITING;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001282 opt = CPU_BASED_TPR_SHADOW |
Sheng Yang25c5f222008-03-28 13:18:56 +08001283 CPU_BASED_USE_MSR_BITMAPS |
Sheng Yangf78e0e22007-10-29 09:40:42 +08001284 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001285 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
1286 &_cpu_based_exec_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001287 return -EIO;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08001288#ifdef CONFIG_X86_64
1289 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
1290 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
1291 ~CPU_BASED_CR8_STORE_EXITING;
1292#endif
Sheng Yangf78e0e22007-10-29 09:40:42 +08001293 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
Sheng Yangd56f5462008-04-25 10:13:16 +08001294 min2 = 0;
1295 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
Sheng Yang2384d2b2008-01-17 15:14:33 +08001296 SECONDARY_EXEC_WBINVD_EXITING |
Sheng Yangd56f5462008-04-25 10:13:16 +08001297 SECONDARY_EXEC_ENABLE_VPID |
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001298 SECONDARY_EXEC_ENABLE_EPT |
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08001299 SECONDARY_EXEC_UNRESTRICTED_GUEST |
Sheng Yang4e47c7a2009-12-18 16:48:47 +08001300 SECONDARY_EXEC_PAUSE_LOOP_EXITING |
1301 SECONDARY_EXEC_RDTSCP;
Sheng Yangd56f5462008-04-25 10:13:16 +08001302 if (adjust_vmx_controls(min2, opt2,
1303 MSR_IA32_VMX_PROCBASED_CTLS2,
Sheng Yangf78e0e22007-10-29 09:40:42 +08001304 &_cpu_based_2nd_exec_control) < 0)
1305 return -EIO;
1306 }
1307#ifndef CONFIG_X86_64
1308 if (!(_cpu_based_2nd_exec_control &
1309 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
1310 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
1311#endif
Sheng Yangd56f5462008-04-25 10:13:16 +08001312 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
Marcelo Tosattia7052892008-09-23 13:18:35 -03001313 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
1314 enabled */
Gleb Natapov5fff7d22009-08-27 18:41:30 +03001315 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
1316 CPU_BASED_CR3_STORE_EXITING |
1317 CPU_BASED_INVLPG_EXITING);
Sheng Yangd56f5462008-04-25 10:13:16 +08001318 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
1319 vmx_capability.ept, vmx_capability.vpid);
1320 }
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001321
1322 min = 0;
1323#ifdef CONFIG_X86_64
1324 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
1325#endif
Sheng Yang468d4722008-10-09 16:01:55 +08001326 opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001327 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
1328 &_vmexit_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001329 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001330
Sheng Yang468d4722008-10-09 16:01:55 +08001331 min = 0;
1332 opt = VM_ENTRY_LOAD_IA32_PAT;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001333 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
1334 &_vmentry_control) < 0)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001335 return -EIO;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001336
Nguyen Anh Quynhc68876f2006-12-29 16:49:54 -08001337 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001338
1339 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
1340 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001341 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001342
1343#ifdef CONFIG_X86_64
1344 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
1345 if (vmx_msr_high & (1u<<16))
Yang, Sheng002c7f72007-07-31 14:23:01 +03001346 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001347#endif
1348
1349 /* Require Write-Back (WB) memory type for VMCS accesses. */
1350 if (((vmx_msr_high >> 18) & 15) != 6)
Yang, Sheng002c7f72007-07-31 14:23:01 +03001351 return -EIO;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001352
Yang, Sheng002c7f72007-07-31 14:23:01 +03001353 vmcs_conf->size = vmx_msr_high & 0x1fff;
1354 vmcs_conf->order = get_order(vmcs_config.size);
1355 vmcs_conf->revision_id = vmx_msr_low;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001356
Yang, Sheng002c7f72007-07-31 14:23:01 +03001357 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
1358 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
Sheng Yangf78e0e22007-10-29 09:40:42 +08001359 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
Yang, Sheng002c7f72007-07-31 14:23:01 +03001360 vmcs_conf->vmexit_ctrl = _vmexit_control;
1361 vmcs_conf->vmentry_ctrl = _vmentry_control;
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001362
1363 return 0;
Nguyen Anh Quynhc68876f2006-12-29 16:49:54 -08001364}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001365
1366static struct vmcs *alloc_vmcs_cpu(int cpu)
1367{
1368 int node = cpu_to_node(cpu);
1369 struct page *pages;
1370 struct vmcs *vmcs;
1371
Mel Gorman6484eb32009-06-16 15:31:54 -07001372 pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001373 if (!pages)
1374 return NULL;
1375 vmcs = page_address(pages);
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001376 memset(vmcs, 0, vmcs_config.size);
1377 vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001378 return vmcs;
1379}
1380
1381static struct vmcs *alloc_vmcs(void)
1382{
Ingo Molnard3b2c332007-01-05 16:36:23 -08001383 return alloc_vmcs_cpu(raw_smp_processor_id());
Avi Kivity6aa8b732006-12-10 02:21:36 -08001384}
1385
1386static void free_vmcs(struct vmcs *vmcs)
1387{
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03001388 free_pages((unsigned long)vmcs, vmcs_config.order);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001389}
1390
Sam Ravnborg39959582007-06-01 00:47:13 -07001391static void free_kvm_area(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001392{
1393 int cpu;
1394
Zachary Amsden3230bb42009-09-29 11:38:37 -10001395 for_each_possible_cpu(cpu) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001396 free_vmcs(per_cpu(vmxarea, cpu));
Zachary Amsden3230bb42009-09-29 11:38:37 -10001397 per_cpu(vmxarea, cpu) = NULL;
1398 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001399}
1400
Avi Kivity6aa8b732006-12-10 02:21:36 -08001401static __init int alloc_kvm_area(void)
1402{
1403 int cpu;
1404
Zachary Amsden3230bb42009-09-29 11:38:37 -10001405 for_each_possible_cpu(cpu) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001406 struct vmcs *vmcs;
1407
1408 vmcs = alloc_vmcs_cpu(cpu);
1409 if (!vmcs) {
1410 free_kvm_area();
1411 return -ENOMEM;
1412 }
1413
1414 per_cpu(vmxarea, cpu) = vmcs;
1415 }
1416 return 0;
1417}
1418
1419static __init int hardware_setup(void)
1420{
Yang, Sheng002c7f72007-07-31 14:23:01 +03001421 if (setup_vmcs_config(&vmcs_config) < 0)
1422 return -EIO;
Joerg Roedel50a37eb2008-01-31 14:57:38 +01001423
1424 if (boot_cpu_has(X86_FEATURE_NX))
1425 kvm_enable_efer_bits(EFER_NX);
1426
Sheng Yang93ba03c2009-04-01 15:52:32 +08001427 if (!cpu_has_vmx_vpid())
1428 enable_vpid = 0;
1429
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001430 if (!cpu_has_vmx_ept()) {
Sheng Yang93ba03c2009-04-01 15:52:32 +08001431 enable_ept = 0;
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001432 enable_unrestricted_guest = 0;
1433 }
1434
1435 if (!cpu_has_vmx_unrestricted_guest())
1436 enable_unrestricted_guest = 0;
Sheng Yang93ba03c2009-04-01 15:52:32 +08001437
1438 if (!cpu_has_vmx_flexpriority())
1439 flexpriority_enabled = 0;
1440
Gleb Natapov95ba8273132009-04-21 17:45:08 +03001441 if (!cpu_has_vmx_tpr_shadow())
1442 kvm_x86_ops->update_cr8_intercept = NULL;
1443
Marcelo Tosatti54dee992009-06-11 12:07:44 -03001444 if (enable_ept && !cpu_has_vmx_ept_2m_page())
1445 kvm_disable_largepages();
1446
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08001447 if (!cpu_has_vmx_ple())
1448 ple_gap = 0;
1449
Avi Kivity6aa8b732006-12-10 02:21:36 -08001450 return alloc_kvm_area();
1451}
1452
1453static __exit void hardware_unsetup(void)
1454{
1455 free_kvm_area();
1456}
1457
Avi Kivity6aa8b732006-12-10 02:21:36 -08001458static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
1459{
1460 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1461
Avi Kivity6af11b92007-03-19 13:18:10 +02001462 if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001463 vmcs_write16(sf->selector, save->selector);
1464 vmcs_writel(sf->base, save->base);
1465 vmcs_write32(sf->limit, save->limit);
1466 vmcs_write32(sf->ar_bytes, save->ar);
1467 } else {
1468 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
1469 << AR_DPL_SHIFT;
1470 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
1471 }
1472}
1473
1474static void enter_pmode(struct kvm_vcpu *vcpu)
1475{
1476 unsigned long flags;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001477 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001478
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001479 vmx->emulation_required = 1;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001480 vmx->rmode.vm86_active = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001481
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001482 vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
1483 vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
1484 vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001485
1486 flags = vmcs_readl(GUEST_RFLAGS);
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +01001487 flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001488 flags |= (vmx->rmode.save_iopl << IOPL_SHIFT);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001489 vmcs_writel(GUEST_RFLAGS, flags);
1490
Rusty Russell66aee912007-07-17 23:34:16 +10001491 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
1492 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001493
1494 update_exception_bitmap(vcpu);
1495
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001496 if (emulate_invalid_guest_state)
1497 return;
1498
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001499 fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
1500 fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
1501 fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
1502 fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001503
1504 vmcs_write16(GUEST_SS_SELECTOR, 0);
1505 vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
1506
1507 vmcs_write16(GUEST_CS_SELECTOR,
1508 vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
1509 vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1510}
1511
Mike Dayd77c26f2007-10-08 09:02:08 -04001512static gva_t rmode_tss_base(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001513{
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08001514 if (!kvm->arch.tss_addr) {
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001515 struct kvm_memslots *slots;
1516 gfn_t base_gfn;
1517
1518 slots = rcu_dereference(kvm->memslots);
1519 base_gfn = kvm->memslots->memslots[0].base_gfn +
Marcelo Tosatti46a26bf2009-12-23 14:35:16 -02001520 kvm->memslots->memslots[0].npages - 3;
Izik Eiduscbc94022007-10-25 00:29:55 +02001521 return base_gfn << PAGE_SHIFT;
1522 }
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08001523 return kvm->arch.tss_addr;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001524}
1525
1526static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
1527{
1528 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1529
1530 save->selector = vmcs_read16(sf->selector);
1531 save->base = vmcs_readl(sf->base);
1532 save->limit = vmcs_read32(sf->limit);
1533 save->ar = vmcs_read32(sf->ar_bytes);
Jan Kiszka15b00f32007-11-19 10:21:45 +01001534 vmcs_write16(sf->selector, save->base >> 4);
1535 vmcs_write32(sf->base, save->base & 0xfffff);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001536 vmcs_write32(sf->limit, 0xffff);
1537 vmcs_write32(sf->ar_bytes, 0xf3);
1538}
1539
1540static void enter_rmode(struct kvm_vcpu *vcpu)
1541{
1542 unsigned long flags;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001543 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001544
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001545 if (enable_unrestricted_guest)
1546 return;
1547
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001548 vmx->emulation_required = 1;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001549 vmx->rmode.vm86_active = 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001550
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001551 vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001552 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
1553
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001554 vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001555 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
1556
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001557 vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001558 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1559
1560 flags = vmcs_readl(GUEST_RFLAGS);
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001561 vmx->rmode.save_iopl
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001562 = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001563
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +01001564 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001565
1566 vmcs_writel(GUEST_RFLAGS, flags);
Rusty Russell66aee912007-07-17 23:34:16 +10001567 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001568 update_exception_bitmap(vcpu);
1569
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001570 if (emulate_invalid_guest_state)
1571 goto continue_rmode;
1572
Avi Kivity6aa8b732006-12-10 02:21:36 -08001573 vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
1574 vmcs_write32(GUEST_SS_LIMIT, 0xffff);
1575 vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
1576
1577 vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
Michael Riepeabacf8d2006-12-22 01:05:45 -08001578 vmcs_write32(GUEST_CS_LIMIT, 0xffff);
Avi Kivity8cb5b032007-03-20 18:40:40 +02001579 if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
1580 vmcs_writel(GUEST_CS_BASE, 0xf0000);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001581 vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
1582
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001583 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
1584 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
1585 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
1586 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
Avi Kivity75880a02007-06-20 11:20:04 +03001587
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03001588continue_rmode:
Eddie Dong8668a3c2007-10-10 14:26:45 +08001589 kvm_mmu_reset_context(vcpu);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08001590 init_rmode(vcpu->kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001591}
1592
Amit Shah401d10d2009-02-20 22:53:37 +05301593static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1594{
1595 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity26bb0982009-09-07 11:14:12 +03001596 struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
1597
1598 if (!msr)
1599 return;
Amit Shah401d10d2009-02-20 22:53:37 +05301600
Avi Kivity44ea2b12009-09-06 15:55:37 +03001601 /*
1602 * Force kernel_gs_base reloading before EFER changes, as control
1603 * of this msr depends on is_long_mode().
1604 */
1605 vmx_load_host_state(to_vmx(vcpu));
Avi Kivityf6801df2010-01-21 15:31:50 +02001606 vcpu->arch.efer = efer;
Amit Shah401d10d2009-02-20 22:53:37 +05301607 if (!msr)
1608 return;
1609 if (efer & EFER_LMA) {
1610 vmcs_write32(VM_ENTRY_CONTROLS,
1611 vmcs_read32(VM_ENTRY_CONTROLS) |
1612 VM_ENTRY_IA32E_MODE);
1613 msr->data = efer;
1614 } else {
1615 vmcs_write32(VM_ENTRY_CONTROLS,
1616 vmcs_read32(VM_ENTRY_CONTROLS) &
1617 ~VM_ENTRY_IA32E_MODE);
1618
1619 msr->data = efer & ~EFER_LME;
1620 }
1621 setup_msrs(vmx);
1622}
1623
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001624#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001625
1626static void enter_lmode(struct kvm_vcpu *vcpu)
1627{
1628 u32 guest_tr_ar;
1629
1630 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
1631 if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
1632 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001633 __func__);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001634 vmcs_write32(GUEST_TR_AR_BYTES,
1635 (guest_tr_ar & ~AR_TYPE_MASK)
1636 | AR_TYPE_BUSY_64_TSS);
1637 }
Avi Kivityf6801df2010-01-21 15:31:50 +02001638 vcpu->arch.efer |= EFER_LMA;
1639 vmx_set_efer(vcpu, vcpu->arch.efer);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001640}
1641
1642static void exit_lmode(struct kvm_vcpu *vcpu)
1643{
Avi Kivityf6801df2010-01-21 15:31:50 +02001644 vcpu->arch.efer &= ~EFER_LMA;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001645
1646 vmcs_write32(VM_ENTRY_CONTROLS,
1647 vmcs_read32(VM_ENTRY_CONTROLS)
Li, Xin B1e4e6e02007-08-01 21:49:10 +03001648 & ~VM_ENTRY_IA32E_MODE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001649}
1650
1651#endif
1652
Sheng Yang2384d2b2008-01-17 15:14:33 +08001653static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
1654{
1655 vpid_sync_vcpu_all(to_vmx(vcpu));
Avi Kivity089d0342009-03-23 18:26:32 +02001656 if (enable_ept)
Sheng Yang4e1096d2008-07-06 19:16:51 +08001657 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
Sheng Yang2384d2b2008-01-17 15:14:33 +08001658}
1659
Avi Kivitye8467fd2009-12-29 18:43:06 +02001660static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1661{
1662 ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
1663
1664 vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
1665 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
1666}
1667
Anthony Liguori25c4c272007-04-27 09:29:21 +03001668static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
Avi Kivity399badf2007-01-05 16:36:38 -08001669{
Avi Kivityfc78f512009-12-07 12:16:48 +02001670 ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
1671
1672 vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
1673 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
Avi Kivity399badf2007-01-05 16:36:38 -08001674}
1675
Sheng Yang14394422008-04-28 12:24:45 +08001676static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
1677{
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001678 if (!test_bit(VCPU_EXREG_PDPTR,
1679 (unsigned long *)&vcpu->arch.regs_dirty))
1680 return;
1681
Sheng Yang14394422008-04-28 12:24:45 +08001682 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
Sheng Yang14394422008-04-28 12:24:45 +08001683 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]);
1684 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]);
1685 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]);
1686 vmcs_write64(GUEST_PDPTR3, vcpu->arch.pdptrs[3]);
1687 }
1688}
1689
Avi Kivity8f5d5492009-05-31 18:41:29 +03001690static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
1691{
1692 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1693 vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
1694 vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
1695 vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
1696 vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1697 }
Avi Kivity6de4f3a2009-05-31 22:58:47 +03001698
1699 __set_bit(VCPU_EXREG_PDPTR,
1700 (unsigned long *)&vcpu->arch.regs_avail);
1701 __set_bit(VCPU_EXREG_PDPTR,
1702 (unsigned long *)&vcpu->arch.regs_dirty);
Avi Kivity8f5d5492009-05-31 18:41:29 +03001703}
1704
Sheng Yang14394422008-04-28 12:24:45 +08001705static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1706
1707static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
1708 unsigned long cr0,
1709 struct kvm_vcpu *vcpu)
1710{
1711 if (!(cr0 & X86_CR0_PG)) {
1712 /* From paging/starting to nonpaging */
1713 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
Sheng Yang65267ea2008-06-18 14:43:38 +08001714 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
Sheng Yang14394422008-04-28 12:24:45 +08001715 (CPU_BASED_CR3_LOAD_EXITING |
1716 CPU_BASED_CR3_STORE_EXITING));
1717 vcpu->arch.cr0 = cr0;
Avi Kivityfc78f512009-12-07 12:16:48 +02001718 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
Sheng Yang14394422008-04-28 12:24:45 +08001719 } else if (!is_paging(vcpu)) {
1720 /* From nonpaging to paging */
1721 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
Sheng Yang65267ea2008-06-18 14:43:38 +08001722 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
Sheng Yang14394422008-04-28 12:24:45 +08001723 ~(CPU_BASED_CR3_LOAD_EXITING |
1724 CPU_BASED_CR3_STORE_EXITING));
1725 vcpu->arch.cr0 = cr0;
Avi Kivityfc78f512009-12-07 12:16:48 +02001726 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
Sheng Yang14394422008-04-28 12:24:45 +08001727 }
Sheng Yang95eb84a2009-08-19 09:52:18 +08001728
1729 if (!(cr0 & X86_CR0_WP))
1730 *hw_cr0 &= ~X86_CR0_WP;
Sheng Yang14394422008-04-28 12:24:45 +08001731}
1732
Avi Kivity6aa8b732006-12-10 02:21:36 -08001733static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1734{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001735 struct vcpu_vmx *vmx = to_vmx(vcpu);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001736 unsigned long hw_cr0;
1737
1738 if (enable_unrestricted_guest)
1739 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
1740 | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
1741 else
1742 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
Sheng Yang14394422008-04-28 12:24:45 +08001743
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001744 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001745 enter_pmode(vcpu);
1746
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001747 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001748 enter_rmode(vcpu);
1749
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001750#ifdef CONFIG_X86_64
Avi Kivityf6801df2010-01-21 15:31:50 +02001751 if (vcpu->arch.efer & EFER_LME) {
Rusty Russell707d92f2007-07-17 23:19:08 +10001752 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001753 enter_lmode(vcpu);
Rusty Russell707d92f2007-07-17 23:19:08 +10001754 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001755 exit_lmode(vcpu);
1756 }
1757#endif
1758
Avi Kivity089d0342009-03-23 18:26:32 +02001759 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +08001760 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
1761
Avi Kivity02daab22009-12-30 12:40:26 +02001762 if (!vcpu->fpu_active)
Avi Kivity81231c62010-01-24 16:26:40 +02001763 hw_cr0 |= X86_CR0_TS | X86_CR0_MP;
Avi Kivity02daab22009-12-30 12:40:26 +02001764
Avi Kivity6aa8b732006-12-10 02:21:36 -08001765 vmcs_writel(CR0_READ_SHADOW, cr0);
Sheng Yang14394422008-04-28 12:24:45 +08001766 vmcs_writel(GUEST_CR0, hw_cr0);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001767 vcpu->arch.cr0 = cr0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001768}
1769
Sheng Yang14394422008-04-28 12:24:45 +08001770static u64 construct_eptp(unsigned long root_hpa)
1771{
1772 u64 eptp;
1773
1774 /* TODO write the value reading from MSR */
1775 eptp = VMX_EPT_DEFAULT_MT |
1776 VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
1777 eptp |= (root_hpa & PAGE_MASK);
1778
1779 return eptp;
1780}
1781
Avi Kivity6aa8b732006-12-10 02:21:36 -08001782static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1783{
Sheng Yang14394422008-04-28 12:24:45 +08001784 unsigned long guest_cr3;
1785 u64 eptp;
1786
1787 guest_cr3 = cr3;
Avi Kivity089d0342009-03-23 18:26:32 +02001788 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +08001789 eptp = construct_eptp(cr3);
1790 vmcs_write64(EPT_POINTER, eptp);
Sheng Yang14394422008-04-28 12:24:45 +08001791 guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
Sheng Yangb927a3c2009-07-21 10:42:48 +08001792 vcpu->kvm->arch.ept_identity_map_addr;
Marcelo Tosatti7c93be42009-10-26 16:48:33 -02001793 ept_load_pdptrs(vcpu);
Sheng Yang14394422008-04-28 12:24:45 +08001794 }
1795
Sheng Yang2384d2b2008-01-17 15:14:33 +08001796 vmx_flush_tlb(vcpu);
Sheng Yang14394422008-04-28 12:24:45 +08001797 vmcs_writel(GUEST_CR3, guest_cr3);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001798}
1799
1800static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1801{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001802 unsigned long hw_cr4 = cr4 | (to_vmx(vcpu)->rmode.vm86_active ?
Sheng Yang14394422008-04-28 12:24:45 +08001803 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
1804
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001805 vcpu->arch.cr4 = cr4;
Avi Kivitybc230082009-12-08 12:14:42 +02001806 if (enable_ept) {
1807 if (!is_paging(vcpu)) {
1808 hw_cr4 &= ~X86_CR4_PAE;
1809 hw_cr4 |= X86_CR4_PSE;
1810 } else if (!(cr4 & X86_CR4_PAE)) {
1811 hw_cr4 &= ~X86_CR4_PAE;
1812 }
1813 }
Sheng Yang14394422008-04-28 12:24:45 +08001814
1815 vmcs_writel(CR4_READ_SHADOW, cr4);
1816 vmcs_writel(GUEST_CR4, hw_cr4);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001817}
1818
Avi Kivity6aa8b732006-12-10 02:21:36 -08001819static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1820{
1821 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1822
1823 return vmcs_readl(sf->base);
1824}
1825
1826static void vmx_get_segment(struct kvm_vcpu *vcpu,
1827 struct kvm_segment *var, int seg)
1828{
1829 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1830 u32 ar;
1831
1832 var->base = vmcs_readl(sf->base);
1833 var->limit = vmcs_read32(sf->limit);
1834 var->selector = vmcs_read16(sf->selector);
1835 ar = vmcs_read32(sf->ar_bytes);
Avi Kivity9fd4a3b2009-01-04 23:43:42 +02001836 if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001837 ar = 0;
1838 var->type = ar & 15;
1839 var->s = (ar >> 4) & 1;
1840 var->dpl = (ar >> 5) & 3;
1841 var->present = (ar >> 7) & 1;
1842 var->avl = (ar >> 12) & 1;
1843 var->l = (ar >> 13) & 1;
1844 var->db = (ar >> 14) & 1;
1845 var->g = (ar >> 15) & 1;
1846 var->unusable = (ar >> 16) & 1;
1847}
1848
Izik Eidus2e4d2652008-03-24 19:38:34 +02001849static int vmx_get_cpl(struct kvm_vcpu *vcpu)
1850{
Avi Kivity3eeb3282010-01-21 15:31:48 +02001851 if (!is_protmode(vcpu))
Izik Eidus2e4d2652008-03-24 19:38:34 +02001852 return 0;
1853
1854 if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
1855 return 3;
1856
Avi Kivityeab4b8a2009-08-04 15:02:54 +03001857 return vmcs_read16(GUEST_CS_SELECTOR) & 3;
Izik Eidus2e4d2652008-03-24 19:38:34 +02001858}
1859
Avi Kivity653e3102007-05-07 10:55:37 +03001860static u32 vmx_segment_access_rights(struct kvm_segment *var)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001861{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001862 u32 ar;
1863
Avi Kivity653e3102007-05-07 10:55:37 +03001864 if (var->unusable)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001865 ar = 1 << 16;
1866 else {
1867 ar = var->type & 15;
1868 ar |= (var->s & 1) << 4;
1869 ar |= (var->dpl & 3) << 5;
1870 ar |= (var->present & 1) << 7;
1871 ar |= (var->avl & 1) << 12;
1872 ar |= (var->l & 1) << 13;
1873 ar |= (var->db & 1) << 14;
1874 ar |= (var->g & 1) << 15;
1875 }
Uri Lublinf7fbf1f2006-12-13 00:34:00 -08001876 if (ar == 0) /* a 0 value means unusable */
1877 ar = AR_UNUSABLE_MASK;
Avi Kivity653e3102007-05-07 10:55:37 +03001878
1879 return ar;
1880}
1881
1882static void vmx_set_segment(struct kvm_vcpu *vcpu,
1883 struct kvm_segment *var, int seg)
1884{
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001885 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity653e3102007-05-07 10:55:37 +03001886 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1887 u32 ar;
1888
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001889 if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
1890 vmx->rmode.tr.selector = var->selector;
1891 vmx->rmode.tr.base = var->base;
1892 vmx->rmode.tr.limit = var->limit;
1893 vmx->rmode.tr.ar = vmx_segment_access_rights(var);
Avi Kivity653e3102007-05-07 10:55:37 +03001894 return;
1895 }
1896 vmcs_writel(sf->base, var->base);
1897 vmcs_write32(sf->limit, var->limit);
1898 vmcs_write16(sf->selector, var->selector);
Avi Kivity7ffd92c2009-06-09 14:10:45 +03001899 if (vmx->rmode.vm86_active && var->s) {
Avi Kivity653e3102007-05-07 10:55:37 +03001900 /*
1901 * Hack real-mode segments into vm86 compatibility.
1902 */
1903 if (var->base == 0xffff0000 && var->selector == 0xf000)
1904 vmcs_writel(sf->base, 0xf0000);
1905 ar = 0xf3;
1906 } else
1907 ar = vmx_segment_access_rights(var);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07001908
1909 /*
1910 * Fix the "Accessed" bit in AR field of segment registers for older
1911 * qemu binaries.
1912 * IA32 arch specifies that at the time of processor reset the
1913 * "Accessed" bit in the AR field of segment registers is 1. And qemu
1914 * is setting it to 0 in the usedland code. This causes invalid guest
1915 * state vmexit when "unrestricted guest" mode is turned on.
1916 * Fix for this setup issue in cpu_reset is being pushed in the qemu
1917 * tree. Newer qemu binaries with that qemu fix would not need this
1918 * kvm hack.
1919 */
1920 if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
1921 ar |= 0x1; /* Accessed */
1922
Avi Kivity6aa8b732006-12-10 02:21:36 -08001923 vmcs_write32(sf->ar_bytes, ar);
1924}
1925
Avi Kivity6aa8b732006-12-10 02:21:36 -08001926static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1927{
1928 u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
1929
1930 *db = (ar >> 14) & 1;
1931 *l = (ar >> 13) & 1;
1932}
1933
1934static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1935{
1936 dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
1937 dt->base = vmcs_readl(GUEST_IDTR_BASE);
1938}
1939
1940static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1941{
1942 vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
1943 vmcs_writel(GUEST_IDTR_BASE, dt->base);
1944}
1945
1946static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1947{
1948 dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
1949 dt->base = vmcs_readl(GUEST_GDTR_BASE);
1950}
1951
1952static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1953{
1954 vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1955 vmcs_writel(GUEST_GDTR_BASE, dt->base);
1956}
1957
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03001958static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
1959{
1960 struct kvm_segment var;
1961 u32 ar;
1962
1963 vmx_get_segment(vcpu, &var, seg);
1964 ar = vmx_segment_access_rights(&var);
1965
1966 if (var.base != (var.selector << 4))
1967 return false;
1968 if (var.limit != 0xffff)
1969 return false;
1970 if (ar != 0xf3)
1971 return false;
1972
1973 return true;
1974}
1975
1976static bool code_segment_valid(struct kvm_vcpu *vcpu)
1977{
1978 struct kvm_segment cs;
1979 unsigned int cs_rpl;
1980
1981 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
1982 cs_rpl = cs.selector & SELECTOR_RPL_MASK;
1983
Avi Kivity1872a3f2009-01-04 23:26:52 +02001984 if (cs.unusable)
1985 return false;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03001986 if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
1987 return false;
1988 if (!cs.s)
1989 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02001990 if (cs.type & AR_TYPE_WRITEABLE_MASK) {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03001991 if (cs.dpl > cs_rpl)
1992 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02001993 } else {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03001994 if (cs.dpl != cs_rpl)
1995 return false;
1996 }
1997 if (!cs.present)
1998 return false;
1999
2000 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
2001 return true;
2002}
2003
2004static bool stack_segment_valid(struct kvm_vcpu *vcpu)
2005{
2006 struct kvm_segment ss;
2007 unsigned int ss_rpl;
2008
2009 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2010 ss_rpl = ss.selector & SELECTOR_RPL_MASK;
2011
Avi Kivity1872a3f2009-01-04 23:26:52 +02002012 if (ss.unusable)
2013 return true;
2014 if (ss.type != 3 && ss.type != 7)
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002015 return false;
2016 if (!ss.s)
2017 return false;
2018 if (ss.dpl != ss_rpl) /* DPL != RPL */
2019 return false;
2020 if (!ss.present)
2021 return false;
2022
2023 return true;
2024}
2025
2026static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
2027{
2028 struct kvm_segment var;
2029 unsigned int rpl;
2030
2031 vmx_get_segment(vcpu, &var, seg);
2032 rpl = var.selector & SELECTOR_RPL_MASK;
2033
Avi Kivity1872a3f2009-01-04 23:26:52 +02002034 if (var.unusable)
2035 return true;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002036 if (!var.s)
2037 return false;
2038 if (!var.present)
2039 return false;
2040 if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
2041 if (var.dpl < rpl) /* DPL < RPL */
2042 return false;
2043 }
2044
2045 /* TODO: Add other members to kvm_segment_field to allow checking for other access
2046 * rights flags
2047 */
2048 return true;
2049}
2050
2051static bool tr_valid(struct kvm_vcpu *vcpu)
2052{
2053 struct kvm_segment tr;
2054
2055 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
2056
Avi Kivity1872a3f2009-01-04 23:26:52 +02002057 if (tr.unusable)
2058 return false;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002059 if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */
2060 return false;
Avi Kivity1872a3f2009-01-04 23:26:52 +02002061 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002062 return false;
2063 if (!tr.present)
2064 return false;
2065
2066 return true;
2067}
2068
2069static bool ldtr_valid(struct kvm_vcpu *vcpu)
2070{
2071 struct kvm_segment ldtr;
2072
2073 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
2074
Avi Kivity1872a3f2009-01-04 23:26:52 +02002075 if (ldtr.unusable)
2076 return true;
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002077 if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */
2078 return false;
2079 if (ldtr.type != 2)
2080 return false;
2081 if (!ldtr.present)
2082 return false;
2083
2084 return true;
2085}
2086
2087static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
2088{
2089 struct kvm_segment cs, ss;
2090
2091 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
2092 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
2093
2094 return ((cs.selector & SELECTOR_RPL_MASK) ==
2095 (ss.selector & SELECTOR_RPL_MASK));
2096}
2097
2098/*
2099 * Check if guest state is valid. Returns true if valid, false if
2100 * not.
2101 * We assume that registers are always usable
2102 */
2103static bool guest_state_valid(struct kvm_vcpu *vcpu)
2104{
2105 /* real mode guest state checks */
Avi Kivity3eeb3282010-01-21 15:31:48 +02002106 if (!is_protmode(vcpu)) {
Mohammed Gamal648dfaa2008-08-17 16:38:32 +03002107 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
2108 return false;
2109 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
2110 return false;
2111 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
2112 return false;
2113 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
2114 return false;
2115 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
2116 return false;
2117 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
2118 return false;
2119 } else {
2120 /* protected mode guest state checks */
2121 if (!cs_ss_rpl_check(vcpu))
2122 return false;
2123 if (!code_segment_valid(vcpu))
2124 return false;
2125 if (!stack_segment_valid(vcpu))
2126 return false;
2127 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
2128 return false;
2129 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
2130 return false;
2131 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
2132 return false;
2133 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
2134 return false;
2135 if (!tr_valid(vcpu))
2136 return false;
2137 if (!ldtr_valid(vcpu))
2138 return false;
2139 }
2140 /* TODO:
2141 * - Add checks on RIP
2142 * - Add checks on RFLAGS
2143 */
2144
2145 return true;
2146}
2147
Mike Dayd77c26f2007-10-08 09:02:08 -04002148static int init_rmode_tss(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002149{
Avi Kivity6aa8b732006-12-10 02:21:36 -08002150 gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
Izik Eidus195aefd2007-10-01 22:14:18 +02002151 u16 data = 0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002152 int ret = 0;
Izik Eidus195aefd2007-10-01 22:14:18 +02002153 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002154
Izik Eidus195aefd2007-10-01 22:14:18 +02002155 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2156 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002157 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002158 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
Sheng Yang464d17c2008-08-13 14:10:33 +08002159 r = kvm_write_guest_page(kvm, fn++, &data,
2160 TSS_IOPB_BASE_OFFSET, sizeof(u16));
Izik Eidus195aefd2007-10-01 22:14:18 +02002161 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002162 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002163 r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
2164 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002165 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002166 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2167 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002168 goto out;
Izik Eidus195aefd2007-10-01 22:14:18 +02002169 data = ~0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002170 r = kvm_write_guest_page(kvm, fn, &data,
2171 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
2172 sizeof(u8));
Izik Eidus195aefd2007-10-01 22:14:18 +02002173 if (r < 0)
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002174 goto out;
2175
2176 ret = 1;
2177out:
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002178 return ret;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002179}
2180
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002181static int init_rmode_identity_map(struct kvm *kvm)
2182{
2183 int i, r, ret;
2184 pfn_t identity_map_pfn;
2185 u32 tmp;
2186
Avi Kivity089d0342009-03-23 18:26:32 +02002187 if (!enable_ept)
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002188 return 1;
2189 if (unlikely(!kvm->arch.ept_identity_pagetable)) {
2190 printk(KERN_ERR "EPT: identity-mapping pagetable "
2191 "haven't been allocated!\n");
2192 return 0;
2193 }
2194 if (likely(kvm->arch.ept_identity_pagetable_done))
2195 return 1;
2196 ret = 0;
Sheng Yangb927a3c2009-07-21 10:42:48 +08002197 identity_map_pfn = kvm->arch.ept_identity_map_addr >> PAGE_SHIFT;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002198 r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
2199 if (r < 0)
2200 goto out;
2201 /* Set up identity-mapping pagetable for EPT in real mode */
2202 for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
2203 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
2204 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
2205 r = kvm_write_guest_page(kvm, identity_map_pfn,
2206 &tmp, i * sizeof(tmp), sizeof(tmp));
2207 if (r < 0)
2208 goto out;
2209 }
2210 kvm->arch.ept_identity_pagetable_done = true;
2211 ret = 1;
2212out:
2213 return ret;
2214}
2215
Avi Kivity6aa8b732006-12-10 02:21:36 -08002216static void seg_setup(int seg)
2217{
2218 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002219 unsigned int ar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002220
2221 vmcs_write16(sf->selector, 0);
2222 vmcs_writel(sf->base, 0);
2223 vmcs_write32(sf->limit, 0xffff);
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002224 if (enable_unrestricted_guest) {
2225 ar = 0x93;
2226 if (seg == VCPU_SREG_CS)
2227 ar |= 0x08; /* code segment */
2228 } else
2229 ar = 0xf3;
2230
2231 vmcs_write32(sf->ar_bytes, ar);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002232}
2233
Sheng Yangf78e0e22007-10-29 09:40:42 +08002234static int alloc_apic_access_page(struct kvm *kvm)
2235{
2236 struct kvm_userspace_memory_region kvm_userspace_mem;
2237 int r = 0;
2238
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002239 mutex_lock(&kvm->slots_lock);
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002240 if (kvm->arch.apic_access_page)
Sheng Yangf78e0e22007-10-29 09:40:42 +08002241 goto out;
2242 kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
2243 kvm_userspace_mem.flags = 0;
2244 kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
2245 kvm_userspace_mem.memory_size = PAGE_SIZE;
2246 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2247 if (r)
2248 goto out;
Izik Eidus72dc67a2008-02-10 18:04:15 +02002249
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002250 kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002251out:
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002252 mutex_unlock(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002253 return r;
2254}
2255
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002256static int alloc_identity_pagetable(struct kvm *kvm)
2257{
2258 struct kvm_userspace_memory_region kvm_userspace_mem;
2259 int r = 0;
2260
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002261 mutex_lock(&kvm->slots_lock);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002262 if (kvm->arch.ept_identity_pagetable)
2263 goto out;
2264 kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
2265 kvm_userspace_mem.flags = 0;
Sheng Yangb927a3c2009-07-21 10:42:48 +08002266 kvm_userspace_mem.guest_phys_addr =
2267 kvm->arch.ept_identity_map_addr;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002268 kvm_userspace_mem.memory_size = PAGE_SIZE;
2269 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2270 if (r)
2271 goto out;
2272
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002273 kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
Sheng Yangb927a3c2009-07-21 10:42:48 +08002274 kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002275out:
Marcelo Tosatti79fac952009-12-23 14:35:26 -02002276 mutex_unlock(&kvm->slots_lock);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002277 return r;
2278}
2279
Sheng Yang2384d2b2008-01-17 15:14:33 +08002280static void allocate_vpid(struct vcpu_vmx *vmx)
2281{
2282 int vpid;
2283
2284 vmx->vpid = 0;
Avi Kivity919818a2009-03-23 18:01:29 +02002285 if (!enable_vpid)
Sheng Yang2384d2b2008-01-17 15:14:33 +08002286 return;
2287 spin_lock(&vmx_vpid_lock);
2288 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
2289 if (vpid < VMX_NR_VPIDS) {
2290 vmx->vpid = vpid;
2291 __set_bit(vpid, vmx_vpid_bitmap);
2292 }
2293 spin_unlock(&vmx_vpid_lock);
2294}
2295
Avi Kivity58972972009-02-24 22:26:47 +02002296static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
Sheng Yang25c5f222008-03-28 13:18:56 +08002297{
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002298 int f = sizeof(unsigned long);
Sheng Yang25c5f222008-03-28 13:18:56 +08002299
2300 if (!cpu_has_vmx_msr_bitmap())
2301 return;
2302
2303 /*
2304 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
2305 * have the write-low and read-high bitmap offsets the wrong way round.
2306 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
2307 */
Sheng Yang25c5f222008-03-28 13:18:56 +08002308 if (msr <= 0x1fff) {
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002309 __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
2310 __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
Sheng Yang25c5f222008-03-28 13:18:56 +08002311 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2312 msr &= 0x1fff;
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002313 __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
2314 __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
Sheng Yang25c5f222008-03-28 13:18:56 +08002315 }
Sheng Yang25c5f222008-03-28 13:18:56 +08002316}
2317
Avi Kivity58972972009-02-24 22:26:47 +02002318static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
2319{
2320 if (!longmode_only)
2321 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
2322 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
2323}
2324
Avi Kivity6aa8b732006-12-10 02:21:36 -08002325/*
2326 * Sets up the vmcs for emulated real mode.
2327 */
Rusty Russell8b9cf982007-07-30 16:31:43 +10002328static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002329{
Sheng Yang468d4722008-10-09 16:01:55 +08002330 u32 host_sysenter_cs, msr_low, msr_high;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002331 u32 junk;
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01002332 u64 host_pat, tsc_this, tsc_base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002333 unsigned long a;
2334 struct descriptor_table dt;
2335 int i;
Avi Kivitycd2276a2007-05-14 20:41:13 +03002336 unsigned long kvm_vmx_return;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002337 u32 exec_control;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002338
Avi Kivity6aa8b732006-12-10 02:21:36 -08002339 /* I/O */
Avi Kivity3e7c73e2009-02-24 21:46:19 +02002340 vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
2341 vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002342
Sheng Yang25c5f222008-03-28 13:18:56 +08002343 if (cpu_has_vmx_msr_bitmap())
Avi Kivity58972972009-02-24 22:26:47 +02002344 vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
Sheng Yang25c5f222008-03-28 13:18:56 +08002345
Avi Kivity6aa8b732006-12-10 02:21:36 -08002346 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
2347
Avi Kivity6aa8b732006-12-10 02:21:36 -08002348 /* Control */
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002349 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
2350 vmcs_config.pin_based_exec_ctrl);
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002351
2352 exec_control = vmcs_config.cpu_based_exec_ctrl;
2353 if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
2354 exec_control &= ~CPU_BASED_TPR_SHADOW;
2355#ifdef CONFIG_X86_64
2356 exec_control |= CPU_BASED_CR8_STORE_EXITING |
2357 CPU_BASED_CR8_LOAD_EXITING;
2358#endif
2359 }
Avi Kivity089d0342009-03-23 18:26:32 +02002360 if (!enable_ept)
Sheng Yangd56f5462008-04-25 10:13:16 +08002361 exec_control |= CPU_BASED_CR3_STORE_EXITING |
Marcelo Tosatti83dbc832008-10-07 17:01:27 -03002362 CPU_BASED_CR3_LOAD_EXITING |
2363 CPU_BASED_INVLPG_EXITING;
Yang, Sheng6e5d8652007-09-12 18:03:11 +08002364 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002365
Sheng Yang83ff3b92007-11-21 14:33:25 +08002366 if (cpu_has_secondary_exec_ctrls()) {
2367 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
2368 if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2369 exec_control &=
2370 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
Sheng Yang2384d2b2008-01-17 15:14:33 +08002371 if (vmx->vpid == 0)
2372 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
Sheng Yang046d8712009-11-27 16:46:26 +08002373 if (!enable_ept) {
Sheng Yangd56f5462008-04-25 10:13:16 +08002374 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
Sheng Yang046d8712009-11-27 16:46:26 +08002375 enable_unrestricted_guest = 0;
2376 }
Nitin A Kamble3a624e22009-06-08 11:34:16 -07002377 if (!enable_unrestricted_guest)
2378 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08002379 if (!ple_gap)
2380 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
Sheng Yang83ff3b92007-11-21 14:33:25 +08002381 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
2382 }
Sheng Yangf78e0e22007-10-29 09:40:42 +08002383
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08002384 if (ple_gap) {
2385 vmcs_write32(PLE_GAP, ple_gap);
2386 vmcs_write32(PLE_WINDOW, ple_window);
2387 }
2388
Avi Kivityc7addb92007-09-16 18:58:32 +02002389 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
2390 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002391 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
2392
2393 vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */
2394 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
2395 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
2396
2397 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
2398 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
2399 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
Avi Kivityd6e88ae2008-07-10 16:53:33 +03002400 vmcs_write16(HOST_FS_SELECTOR, kvm_read_fs()); /* 22.2.4 */
2401 vmcs_write16(HOST_GS_SELECTOR, kvm_read_gs()); /* 22.2.4 */
Avi Kivity6aa8b732006-12-10 02:21:36 -08002402 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
Avi Kivity05b3e0c2006-12-13 00:33:45 -08002403#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08002404 rdmsrl(MSR_FS_BASE, a);
2405 vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
2406 rdmsrl(MSR_GS_BASE, a);
2407 vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
2408#else
2409 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
2410 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
2411#endif
2412
2413 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
2414
Avi Kivityd6e88ae2008-07-10 16:53:33 +03002415 kvm_get_idt(&dt);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002416 vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */
2417
Mike Dayd77c26f2007-10-08 09:02:08 -04002418 asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
Avi Kivitycd2276a2007-05-14 20:41:13 +03002419 vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
Eddie Dong2cc51562007-05-21 07:28:09 +03002420 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
2421 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2422 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002423
2424 rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
2425 vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
2426 rdmsrl(MSR_IA32_SYSENTER_ESP, a);
2427 vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */
2428 rdmsrl(MSR_IA32_SYSENTER_EIP, a);
2429 vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */
2430
Sheng Yang468d4722008-10-09 16:01:55 +08002431 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
2432 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2433 host_pat = msr_low | ((u64) msr_high << 32);
2434 vmcs_write64(HOST_IA32_PAT, host_pat);
2435 }
2436 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2437 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2438 host_pat = msr_low | ((u64) msr_high << 32);
2439 /* Write the default value follow host pat */
2440 vmcs_write64(GUEST_IA32_PAT, host_pat);
2441 /* Keep arch.pat sync with GUEST_IA32_PAT */
2442 vmx->vcpu.arch.pat = host_pat;
2443 }
2444
Avi Kivity6aa8b732006-12-10 02:21:36 -08002445 for (i = 0; i < NR_VMX_MSR; ++i) {
2446 u32 index = vmx_msr_index[i];
2447 u32 data_low, data_high;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002448 int j = vmx->nmsrs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002449
2450 if (rdmsr_safe(index, &data_low, &data_high) < 0)
2451 continue;
Avi Kivity432bd6c2007-01-31 23:48:13 -08002452 if (wrmsr_safe(index, data_low, data_high) < 0)
2453 continue;
Avi Kivity26bb0982009-09-07 11:14:12 +03002454 vmx->guest_msrs[j].index = i;
2455 vmx->guest_msrs[j].data = 0;
Avi Kivityd5696722009-12-02 12:28:47 +02002456 vmx->guest_msrs[j].mask = -1ull;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04002457 ++vmx->nmsrs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002458 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002459
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002460 vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002461
2462 /* 22.2.1, 20.8.1 */
Yang, Sheng1c3d14f2007-07-29 11:07:42 +03002463 vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
2464
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002465 vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
Avi Kivity4c386092009-12-07 12:26:18 +02002466 vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
Avi Kivityce03e4f2009-12-07 12:29:14 +02002467 if (enable_ept)
2468 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
Avi Kivity4c386092009-12-07 12:26:18 +02002469 vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002470
Marcelo Tosatti53f658b2008-12-11 20:45:05 +01002471 tsc_base = vmx->vcpu.kvm->arch.vm_init_tsc;
2472 rdtscll(tsc_this);
2473 if (tsc_this < vmx->vcpu.kvm->arch.vm_init_tsc)
2474 tsc_base = tsc_this;
2475
2476 guest_write_tsc(0, tsc_base);
Sheng Yangf78e0e22007-10-29 09:40:42 +08002477
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002478 return 0;
2479}
2480
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002481static int init_rmode(struct kvm *kvm)
2482{
2483 if (!init_rmode_tss(kvm))
2484 return 0;
2485 if (!init_rmode_identity_map(kvm))
2486 return 0;
2487 return 1;
2488}
2489
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002490static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
2491{
2492 struct vcpu_vmx *vmx = to_vmx(vcpu);
2493 u64 msr;
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002494 int ret, idx;
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002495
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002496 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002497 idx = srcu_read_lock(&vcpu->kvm->srcu);
Sheng Yangb7ebfb02008-04-25 21:44:52 +08002498 if (!init_rmode(vmx->vcpu.kvm)) {
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002499 ret = -ENOMEM;
2500 goto out;
2501 }
2502
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002503 vmx->rmode.vm86_active = 0;
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002504
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002505 vmx->soft_vnmi_blocked = 0;
2506
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002507 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02002508 kvm_set_cr8(&vmx->vcpu, 0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002509 msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002510 if (kvm_vcpu_is_bsp(&vmx->vcpu))
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002511 msr |= MSR_IA32_APICBASE_BSP;
2512 kvm_set_apic_base(&vmx->vcpu, msr);
2513
2514 fx_init(&vmx->vcpu);
2515
Avi Kivity5706be02008-08-20 15:07:31 +03002516 seg_setup(VCPU_SREG_CS);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002517 /*
2518 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
2519 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh.
2520 */
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002521 if (kvm_vcpu_is_bsp(&vmx->vcpu)) {
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002522 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
2523 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
2524 } else {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002525 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
2526 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002527 }
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002528
2529 seg_setup(VCPU_SREG_DS);
2530 seg_setup(VCPU_SREG_ES);
2531 seg_setup(VCPU_SREG_FS);
2532 seg_setup(VCPU_SREG_GS);
2533 seg_setup(VCPU_SREG_SS);
2534
2535 vmcs_write16(GUEST_TR_SELECTOR, 0);
2536 vmcs_writel(GUEST_TR_BASE, 0);
2537 vmcs_write32(GUEST_TR_LIMIT, 0xffff);
2538 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2539
2540 vmcs_write16(GUEST_LDTR_SELECTOR, 0);
2541 vmcs_writel(GUEST_LDTR_BASE, 0);
2542 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
2543 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
2544
2545 vmcs_write32(GUEST_SYSENTER_CS, 0);
2546 vmcs_writel(GUEST_SYSENTER_ESP, 0);
2547 vmcs_writel(GUEST_SYSENTER_EIP, 0);
2548
2549 vmcs_writel(GUEST_RFLAGS, 0x02);
Gleb Natapovc5af89b2009-06-09 15:56:26 +03002550 if (kvm_vcpu_is_bsp(&vmx->vcpu))
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002551 kvm_rip_write(vcpu, 0xfff0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002552 else
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002553 kvm_rip_write(vcpu, 0);
2554 kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002555
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002556 vmcs_writel(GUEST_DR7, 0x400);
2557
2558 vmcs_writel(GUEST_GDTR_BASE, 0);
2559 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
2560
2561 vmcs_writel(GUEST_IDTR_BASE, 0);
2562 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
2563
2564 vmcs_write32(GUEST_ACTIVITY_STATE, 0);
2565 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
2566 vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
2567
Avi Kivitye00c8cf2007-10-21 11:00:39 +02002568 /* Special registers */
2569 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
2570
2571 setup_msrs(vmx);
2572
Avi Kivity6aa8b732006-12-10 02:21:36 -08002573 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
2574
Sheng Yangf78e0e22007-10-29 09:40:42 +08002575 if (cpu_has_vmx_tpr_shadow()) {
2576 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
2577 if (vm_need_tpr_shadow(vmx->vcpu.kvm))
2578 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002579 page_to_phys(vmx->vcpu.arch.apic->regs_page));
Sheng Yangf78e0e22007-10-29 09:40:42 +08002580 vmcs_write32(TPR_THRESHOLD, 0);
2581 }
2582
2583 if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2584 vmcs_write64(APIC_ACCESS_ADDR,
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002585 page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002586
Sheng Yang2384d2b2008-01-17 15:14:33 +08002587 if (vmx->vpid != 0)
2588 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2589
Eduardo Habkostfa400522009-10-24 02:49:58 -02002590 vmx->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
Avi Kivity4d4ec082009-12-29 18:07:30 +02002591 vmx_set_cr0(&vmx->vcpu, kvm_read_cr0(vcpu)); /* enter rmode */
Rusty Russell8b9cf982007-07-30 16:31:43 +10002592 vmx_set_cr4(&vmx->vcpu, 0);
Rusty Russell8b9cf982007-07-30 16:31:43 +10002593 vmx_set_efer(&vmx->vcpu, 0);
Rusty Russell8b9cf982007-07-30 16:31:43 +10002594 vmx_fpu_activate(&vmx->vcpu);
2595 update_exception_bitmap(&vmx->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002596
Sheng Yang2384d2b2008-01-17 15:14:33 +08002597 vpid_sync_vcpu_all(vmx);
2598
Marcelo Tosatti3200f402008-03-29 20:17:59 -03002599 ret = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002600
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03002601 /* HACK: Don't enable emulation on guest boot/reset */
2602 vmx->emulation_required = 0;
2603
Avi Kivity6aa8b732006-12-10 02:21:36 -08002604out:
Marcelo Tosattif656ce02009-12-23 14:35:25 -02002605 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002606 return ret;
2607}
2608
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002609static void enable_irq_window(struct kvm_vcpu *vcpu)
2610{
2611 u32 cpu_based_vm_exec_control;
2612
2613 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2614 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
2615 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2616}
2617
2618static void enable_nmi_window(struct kvm_vcpu *vcpu)
2619{
2620 u32 cpu_based_vm_exec_control;
2621
2622 if (!cpu_has_virtual_nmis()) {
2623 enable_irq_window(vcpu);
2624 return;
2625 }
2626
2627 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2628 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
2629 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2630}
2631
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002632static void vmx_inject_irq(struct kvm_vcpu *vcpu)
Eddie Dong85f455f2007-07-06 12:20:49 +03002633{
Avi Kivity9c8cba32007-11-22 11:42:59 +02002634 struct vcpu_vmx *vmx = to_vmx(vcpu);
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002635 uint32_t intr;
2636 int irq = vcpu->arch.interrupt.nr;
Avi Kivity9c8cba32007-11-22 11:42:59 +02002637
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002638 trace_kvm_inj_virq(irq);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04002639
Avi Kivityfa89a812008-09-01 15:57:51 +03002640 ++vcpu->stat.irq_injections;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002641 if (vmx->rmode.vm86_active) {
Avi Kivity9c8cba32007-11-22 11:42:59 +02002642 vmx->rmode.irq.pending = true;
2643 vmx->rmode.irq.vector = irq;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002644 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
Gleb Natapovae0bb3e2009-05-19 11:07:10 +03002645 if (vcpu->arch.interrupt.soft)
2646 vmx->rmode.irq.rip +=
2647 vmx->vcpu.arch.event_exit_inst_len;
Avi Kivity9c5623e2007-11-08 18:19:20 +02002648 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2649 irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);
2650 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002651 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
Eddie Dong85f455f2007-07-06 12:20:49 +03002652 return;
2653 }
Gleb Natapov66fd3f72009-05-11 13:35:50 +03002654 intr = irq | INTR_INFO_VALID_MASK;
2655 if (vcpu->arch.interrupt.soft) {
2656 intr |= INTR_TYPE_SOFT_INTR;
2657 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2658 vmx->vcpu.arch.event_exit_inst_len);
2659 } else
2660 intr |= INTR_TYPE_EXT_INTR;
2661 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
Eddie Dong85f455f2007-07-06 12:20:49 +03002662}
2663
Sheng Yangf08864b2008-05-15 18:23:25 +08002664static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
2665{
Jan Kiszka66a5a342008-09-26 09:30:51 +02002666 struct vcpu_vmx *vmx = to_vmx(vcpu);
2667
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002668 if (!cpu_has_virtual_nmis()) {
2669 /*
2670 * Tracking the NMI-blocked state in software is built upon
2671 * finding the next open IRQ window. This, in turn, depends on
2672 * well-behaving guests: They have to keep IRQs disabled at
2673 * least as long as the NMI handler runs. Otherwise we may
2674 * cause NMI nesting, maybe breaking the guest. But as this is
2675 * highly unlikely, we can live with the residual risk.
2676 */
2677 vmx->soft_vnmi_blocked = 1;
2678 vmx->vnmi_blocked_time = 0;
2679 }
2680
Jan Kiszka487b3912008-09-26 09:30:56 +02002681 ++vcpu->stat.nmi_injections;
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002682 if (vmx->rmode.vm86_active) {
Jan Kiszka66a5a342008-09-26 09:30:51 +02002683 vmx->rmode.irq.pending = true;
2684 vmx->rmode.irq.vector = NMI_VECTOR;
2685 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
2686 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2687 NMI_VECTOR | INTR_TYPE_SOFT_INTR |
2688 INTR_INFO_VALID_MASK);
2689 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
2690 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
2691 return;
2692 }
Sheng Yangf08864b2008-05-15 18:23:25 +08002693 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2694 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
Sheng Yangf08864b2008-05-15 18:23:25 +08002695}
2696
Gleb Natapovc4282df2009-04-21 17:45:07 +03002697static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
Jan Kiszka33f089c2008-09-26 09:30:49 +02002698{
Jan Kiszka3b86cd92008-09-26 09:30:57 +02002699 if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
Gleb Natapovc4282df2009-04-21 17:45:07 +03002700 return 0;
Jan Kiszka33f089c2008-09-26 09:30:49 +02002701
Gleb Natapovc4282df2009-04-21 17:45:07 +03002702 return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2703 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS |
2704 GUEST_INTR_STATE_NMI));
Jan Kiszka33f089c2008-09-26 09:30:49 +02002705}
2706
Jan Kiszka3cfc3092009-11-12 01:04:25 +01002707static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
2708{
2709 if (!cpu_has_virtual_nmis())
2710 return to_vmx(vcpu)->soft_vnmi_blocked;
2711 else
2712 return !!(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2713 GUEST_INTR_STATE_NMI);
2714}
2715
2716static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
2717{
2718 struct vcpu_vmx *vmx = to_vmx(vcpu);
2719
2720 if (!cpu_has_virtual_nmis()) {
2721 if (vmx->soft_vnmi_blocked != masked) {
2722 vmx->soft_vnmi_blocked = masked;
2723 vmx->vnmi_blocked_time = 0;
2724 }
2725 } else {
2726 if (masked)
2727 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
2728 GUEST_INTR_STATE_NMI);
2729 else
2730 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
2731 GUEST_INTR_STATE_NMI);
2732 }
2733}
2734
Gleb Natapov78646122009-03-23 12:12:11 +02002735static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
2736{
Gleb Natapovc4282df2009-04-21 17:45:07 +03002737 return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
2738 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2739 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
Gleb Natapov78646122009-03-23 12:12:11 +02002740}
2741
Izik Eiduscbc94022007-10-25 00:29:55 +02002742static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
2743{
2744 int ret;
2745 struct kvm_userspace_memory_region tss_mem = {
Sheng Yang6fe63972008-10-16 17:30:58 +08002746 .slot = TSS_PRIVATE_MEMSLOT,
Izik Eiduscbc94022007-10-25 00:29:55 +02002747 .guest_phys_addr = addr,
2748 .memory_size = PAGE_SIZE * 3,
2749 .flags = 0,
2750 };
2751
2752 ret = kvm_set_memory_region(kvm, &tss_mem, 0);
2753 if (ret)
2754 return ret;
Zhang Xiantaobfc6d222007-12-14 10:20:16 +08002755 kvm->arch.tss_addr = addr;
Izik Eiduscbc94022007-10-25 00:29:55 +02002756 return 0;
2757}
2758
Avi Kivity6aa8b732006-12-10 02:21:36 -08002759static int handle_rmode_exception(struct kvm_vcpu *vcpu,
2760 int vec, u32 err_code)
2761{
Nitin A Kambleb3f37702007-05-17 15:50:34 +03002762 /*
2763 * Instruction with address size override prefix opcode 0x67
2764 * Cause the #SS fault with 0 error code in VM86 mode.
2765 */
2766 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
Avi Kivity851ba692009-08-24 11:10:17 +03002767 if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002768 return 1;
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002769 /*
2770 * Forward all other exceptions that are valid in real mode.
2771 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
2772 * the required debugging infrastructure rework.
2773 */
2774 switch (vec) {
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002775 case DB_VECTOR:
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002776 if (vcpu->guest_debug &
2777 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
2778 return 0;
2779 kvm_queue_exception(vcpu, vec);
2780 return 1;
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002781 case BP_VECTOR:
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002782 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2783 return 0;
2784 /* fall through */
2785 case DE_VECTOR:
Jan Kiszka77ab6db2008-07-14 12:28:51 +02002786 case OF_VECTOR:
2787 case BR_VECTOR:
2788 case UD_VECTOR:
2789 case DF_VECTOR:
2790 case SS_VECTOR:
2791 case GP_VECTOR:
2792 case MF_VECTOR:
2793 kvm_queue_exception(vcpu, vec);
2794 return 1;
2795 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002796 return 0;
2797}
2798
Andi Kleena0861c02009-06-08 17:37:09 +08002799/*
2800 * Trigger machine check on the host. We assume all the MSRs are already set up
2801 * by the CPU and that we still run on the same CPU as the MCE occurred on.
2802 * We pass a fake environment to the machine check handler because we want
2803 * the guest to be always treated like user space, no matter what context
2804 * it used internally.
2805 */
2806static void kvm_machine_check(void)
2807{
2808#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
2809 struct pt_regs regs = {
2810 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
2811 .flags = X86_EFLAGS_IF,
2812 };
2813
2814 do_machine_check(&regs, 0);
2815#endif
2816}
2817
Avi Kivity851ba692009-08-24 11:10:17 +03002818static int handle_machine_check(struct kvm_vcpu *vcpu)
Andi Kleena0861c02009-06-08 17:37:09 +08002819{
2820 /* already handled by vcpu_run */
2821 return 1;
2822}
2823
Avi Kivity851ba692009-08-24 11:10:17 +03002824static int handle_exception(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002825{
Avi Kivity1155f762007-11-22 11:30:47 +02002826 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivity851ba692009-08-24 11:10:17 +03002827 struct kvm_run *kvm_run = vcpu->run;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002828 u32 intr_info, ex_no, error_code;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002829 unsigned long cr2, rip, dr6;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002830 u32 vect_info;
2831 enum emulation_result er;
2832
Avi Kivity1155f762007-11-22 11:30:47 +02002833 vect_info = vmx->idt_vectoring_info;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002834 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
2835
Andi Kleena0861c02009-06-08 17:37:09 +08002836 if (is_machine_check(intr_info))
Avi Kivity851ba692009-08-24 11:10:17 +03002837 return handle_machine_check(vcpu);
Andi Kleena0861c02009-06-08 17:37:09 +08002838
Avi Kivity6aa8b732006-12-10 02:21:36 -08002839 if ((vect_info & VECTORING_INFO_VALID_MASK) &&
Avi Kivity65ac7262009-11-04 11:59:01 +02002840 !is_page_fault(intr_info)) {
2841 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2842 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
2843 vcpu->run->internal.ndata = 2;
2844 vcpu->run->internal.data[0] = vect_info;
2845 vcpu->run->internal.data[1] = intr_info;
2846 return 0;
2847 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002848
Jan Kiszkae4a41882008-09-26 09:30:46 +02002849 if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
Avi Kivity1b6269d2007-10-09 12:12:19 +02002850 return 1; /* already handled by vmx_vcpu_run() */
Anthony Liguori2ab455c2007-04-27 09:29:49 +03002851
2852 if (is_no_device(intr_info)) {
Avi Kivity5fd86fc2007-05-02 20:40:00 +03002853 vmx_fpu_activate(vcpu);
Anthony Liguori2ab455c2007-04-27 09:29:49 +03002854 return 1;
2855 }
2856
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002857 if (is_invalid_opcode(intr_info)) {
Avi Kivity851ba692009-08-24 11:10:17 +03002858 er = emulate_instruction(vcpu, 0, 0, EMULTYPE_TRAP_UD);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002859 if (er != EMULATE_DONE)
Avi Kivity7ee5d9402007-11-25 15:22:50 +02002860 kvm_queue_exception(vcpu, UD_VECTOR);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05002861 return 1;
2862 }
2863
Avi Kivity6aa8b732006-12-10 02:21:36 -08002864 error_code = 0;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03002865 rip = kvm_rip_read(vcpu);
Ryan Harper2e113842008-02-11 10:26:38 -06002866 if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002867 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
2868 if (is_page_fault(intr_info)) {
Sheng Yang14394422008-04-28 12:24:45 +08002869 /* EPT won't cause page fault directly */
Avi Kivity089d0342009-03-23 18:26:32 +02002870 if (enable_ept)
Sheng Yang14394422008-04-28 12:24:45 +08002871 BUG();
Avi Kivity6aa8b732006-12-10 02:21:36 -08002872 cr2 = vmcs_readl(EXIT_QUALIFICATION);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002873 trace_kvm_page_fault(cr2, error_code);
2874
Gleb Natapov3298b752009-05-11 13:35:46 +03002875 if (kvm_event_needs_reinjection(vcpu))
Avi Kivity577bdc42008-07-19 08:57:05 +03002876 kvm_mmu_unprotect_page_virt(vcpu, cr2);
Avi Kivity30677142007-10-28 18:48:59 +02002877 return kvm_mmu_page_fault(vcpu, cr2, error_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002878 }
2879
Avi Kivity7ffd92c2009-06-09 14:10:45 +03002880 if (vmx->rmode.vm86_active &&
Avi Kivity6aa8b732006-12-10 02:21:36 -08002881 handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
Avi Kivity72d6e5a2007-06-05 16:15:51 +03002882 error_code)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002883 if (vcpu->arch.halt_request) {
2884 vcpu->arch.halt_request = 0;
Avi Kivity72d6e5a2007-06-05 16:15:51 +03002885 return kvm_emulate_halt(vcpu);
2886 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002887 return 1;
Avi Kivity72d6e5a2007-06-05 16:15:51 +03002888 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002889
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002890 ex_no = intr_info & INTR_INFO_VECTOR_MASK;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002891 switch (ex_no) {
2892 case DB_VECTOR:
2893 dr6 = vmcs_readl(EXIT_QUALIFICATION);
2894 if (!(vcpu->guest_debug &
2895 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
2896 vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
2897 kvm_queue_exception(vcpu, DB_VECTOR);
2898 return 1;
2899 }
2900 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
2901 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
2902 /* fall through */
2903 case BP_VECTOR:
Avi Kivity6aa8b732006-12-10 02:21:36 -08002904 kvm_run->exit_reason = KVM_EXIT_DEBUG;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002905 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
2906 kvm_run->debug.arch.exception = ex_no;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002907 break;
2908 default:
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002909 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
2910 kvm_run->ex.exception = ex_no;
2911 kvm_run->ex.error_code = error_code;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01002912 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002913 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002914 return 0;
2915}
2916
Avi Kivity851ba692009-08-24 11:10:17 +03002917static int handle_external_interrupt(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002918{
Avi Kivity1165f5f2007-04-19 17:27:43 +03002919 ++vcpu->stat.irq_exits;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002920 return 1;
2921}
2922
Avi Kivity851ba692009-08-24 11:10:17 +03002923static int handle_triple_fault(struct kvm_vcpu *vcpu)
Avi Kivity988ad742007-02-12 00:54:36 -08002924{
Avi Kivity851ba692009-08-24 11:10:17 +03002925 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
Avi Kivity988ad742007-02-12 00:54:36 -08002926 return 0;
2927}
Avi Kivity6aa8b732006-12-10 02:21:36 -08002928
Avi Kivity851ba692009-08-24 11:10:17 +03002929static int handle_io(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002930{
He, Qingbfdaab02007-09-12 14:18:28 +08002931 unsigned long exit_qualification;
Jan Kiszka34c33d12009-02-08 13:28:15 +01002932 int size, in, string;
Avi Kivity039576c2007-03-20 12:46:50 +02002933 unsigned port;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002934
Avi Kivity1165f5f2007-04-19 17:27:43 +03002935 ++vcpu->stat.io_exits;
He, Qingbfdaab02007-09-12 14:18:28 +08002936 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Avi Kivity039576c2007-03-20 12:46:50 +02002937 string = (exit_qualification & 16) != 0;
Laurent Viviere70669a2007-08-05 10:36:40 +03002938
2939 if (string) {
Avi Kivity851ba692009-08-24 11:10:17 +03002940 if (emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DO_MMIO)
Laurent Viviere70669a2007-08-05 10:36:40 +03002941 return 0;
2942 return 1;
2943 }
2944
2945 size = (exit_qualification & 7) + 1;
2946 in = (exit_qualification & 8) != 0;
Avi Kivity039576c2007-03-20 12:46:50 +02002947 port = exit_qualification >> 16;
Laurent Viviere70669a2007-08-05 10:36:40 +03002948
Guillaume Thouvenine93f36b2008-10-28 10:51:30 +01002949 skip_emulated_instruction(vcpu);
Avi Kivity851ba692009-08-24 11:10:17 +03002950 return kvm_emulate_pio(vcpu, in, size, port);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002951}
2952
Ingo Molnar102d8322007-02-19 14:37:47 +02002953static void
2954vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
2955{
2956 /*
2957 * Patch in the VMCALL instruction:
2958 */
2959 hypercall[0] = 0x0f;
2960 hypercall[1] = 0x01;
2961 hypercall[2] = 0xc1;
Ingo Molnar102d8322007-02-19 14:37:47 +02002962}
2963
Avi Kivity851ba692009-08-24 11:10:17 +03002964static int handle_cr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002965{
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002966 unsigned long exit_qualification, val;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002967 int cr;
2968 int reg;
2969
He, Qingbfdaab02007-09-12 14:18:28 +08002970 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002971 cr = exit_qualification & 15;
2972 reg = (exit_qualification >> 8) & 15;
2973 switch ((exit_qualification >> 4) & 3) {
2974 case 0: /* mov to cr */
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002975 val = kvm_register_read(vcpu, reg);
2976 trace_kvm_cr_write(cr, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002977 switch (cr) {
2978 case 0:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002979 kvm_set_cr0(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002980 skip_emulated_instruction(vcpu);
2981 return 1;
2982 case 3:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002983 kvm_set_cr3(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002984 skip_emulated_instruction(vcpu);
2985 return 1;
2986 case 4:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03002987 kvm_set_cr4(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002988 skip_emulated_instruction(vcpu);
2989 return 1;
Gleb Natapov0a5fff192009-04-21 17:45:06 +03002990 case 8: {
2991 u8 cr8_prev = kvm_get_cr8(vcpu);
2992 u8 cr8 = kvm_register_read(vcpu, reg);
2993 kvm_set_cr8(vcpu, cr8);
2994 skip_emulated_instruction(vcpu);
2995 if (irqchip_in_kernel(vcpu->kvm))
2996 return 1;
2997 if (cr8_prev <= cr8)
2998 return 1;
Avi Kivity851ba692009-08-24 11:10:17 +03002999 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
Gleb Natapov0a5fff192009-04-21 17:45:06 +03003000 return 0;
3001 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003002 };
3003 break;
Anthony Liguori25c4c272007-04-27 09:29:21 +03003004 case 2: /* clts */
Avi Kivityedcafe32009-12-30 18:07:40 +02003005 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
Avi Kivity4d4ec082009-12-29 18:07:30 +02003006 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
Anthony Liguori25c4c272007-04-27 09:29:21 +03003007 skip_emulated_instruction(vcpu);
Avi Kivity6b52d182010-01-21 15:31:47 +02003008 vmx_fpu_activate(vcpu);
Anthony Liguori25c4c272007-04-27 09:29:21 +03003009 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003010 case 1: /*mov from cr*/
3011 switch (cr) {
3012 case 3:
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003013 kvm_register_write(vcpu, reg, vcpu->arch.cr3);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003014 trace_kvm_cr_read(cr, vcpu->arch.cr3);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003015 skip_emulated_instruction(vcpu);
3016 return 1;
3017 case 8:
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003018 val = kvm_get_cr8(vcpu);
3019 kvm_register_write(vcpu, reg, val);
3020 trace_kvm_cr_read(cr, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003021 skip_emulated_instruction(vcpu);
3022 return 1;
3023 }
3024 break;
3025 case 3: /* lmsw */
Avi Kivitya1f83a72009-12-29 17:33:58 +02003026 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
Avi Kivity4d4ec082009-12-29 18:07:30 +02003027 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
Avi Kivitya1f83a72009-12-29 17:33:58 +02003028 kvm_lmsw(vcpu, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003029
3030 skip_emulated_instruction(vcpu);
3031 return 1;
3032 default:
3033 break;
3034 }
Avi Kivity851ba692009-08-24 11:10:17 +03003035 vcpu->run->exit_reason = 0;
Rusty Russellf0242472007-08-01 10:48:02 +10003036 pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
Avi Kivity6aa8b732006-12-10 02:21:36 -08003037 (int)(exit_qualification >> 4) & 3, cr);
3038 return 0;
3039}
3040
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003041static int check_dr_alias(struct kvm_vcpu *vcpu)
3042{
3043 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE)) {
3044 kvm_queue_exception(vcpu, UD_VECTOR);
3045 return -1;
3046 }
3047 return 0;
3048}
3049
Avi Kivity851ba692009-08-24 11:10:17 +03003050static int handle_dr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003051{
He, Qingbfdaab02007-09-12 14:18:28 +08003052 unsigned long exit_qualification;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003053 unsigned long val;
3054 int dr, reg;
3055
Jan Kiszkaf2483412010-01-20 18:20:20 +01003056 /* Do not handle if the CPL > 0, will trigger GP on re-entry */
Avi Kivity0a79b002009-09-01 12:03:25 +03003057 if (!kvm_require_cpl(vcpu, 0))
3058 return 1;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003059 dr = vmcs_readl(GUEST_DR7);
3060 if (dr & DR7_GD) {
3061 /*
3062 * As the vm-exit takes precedence over the debug trap, we
3063 * need to emulate the latter, either for the host or the
3064 * guest debugging itself.
3065 */
3066 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
Avi Kivity851ba692009-08-24 11:10:17 +03003067 vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
3068 vcpu->run->debug.arch.dr7 = dr;
3069 vcpu->run->debug.arch.pc =
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003070 vmcs_readl(GUEST_CS_BASE) +
3071 vmcs_readl(GUEST_RIP);
Avi Kivity851ba692009-08-24 11:10:17 +03003072 vcpu->run->debug.arch.exception = DB_VECTOR;
3073 vcpu->run->exit_reason = KVM_EXIT_DEBUG;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003074 return 0;
3075 } else {
3076 vcpu->arch.dr7 &= ~DR7_GD;
3077 vcpu->arch.dr6 |= DR6_BD;
3078 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
3079 kvm_queue_exception(vcpu, DB_VECTOR);
3080 return 1;
3081 }
3082 }
3083
He, Qingbfdaab02007-09-12 14:18:28 +08003084 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003085 dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
3086 reg = DEBUG_REG_ACCESS_REG(exit_qualification);
3087 if (exit_qualification & TYPE_MOV_FROM_DR) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08003088 switch (dr) {
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003089 case 0 ... 3:
3090 val = vcpu->arch.db[dr];
3091 break;
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003092 case 4:
3093 if (check_dr_alias(vcpu) < 0)
3094 return 1;
3095 /* fall through */
Avi Kivity6aa8b732006-12-10 02:21:36 -08003096 case 6:
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003097 val = vcpu->arch.dr6;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003098 break;
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003099 case 5:
3100 if (check_dr_alias(vcpu) < 0)
3101 return 1;
3102 /* fall through */
3103 default: /* 7 */
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003104 val = vcpu->arch.dr7;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003105 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003106 }
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003107 kvm_register_write(vcpu, reg, val);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003108 } else {
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003109 val = vcpu->arch.regs[reg];
3110 switch (dr) {
3111 case 0 ... 3:
3112 vcpu->arch.db[dr] = val;
3113 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
3114 vcpu->arch.eff_db[dr] = val;
3115 break;
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003116 case 4:
3117 if (check_dr_alias(vcpu) < 0)
Jan Kiszkaf2483412010-01-20 18:20:20 +01003118 return 1;
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003119 /* fall through */
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003120 case 6:
3121 if (val & 0xffffffff00000000ULL) {
Jan Kiszkaf2483412010-01-20 18:20:20 +01003122 kvm_inject_gp(vcpu, 0);
3123 return 1;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003124 }
3125 vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
3126 break;
Jan Kiszka138ac8d2010-01-20 18:20:20 +01003127 case 5:
3128 if (check_dr_alias(vcpu) < 0)
3129 return 1;
3130 /* fall through */
3131 default: /* 7 */
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003132 if (val & 0xffffffff00000000ULL) {
Jan Kiszkaf2483412010-01-20 18:20:20 +01003133 kvm_inject_gp(vcpu, 0);
3134 return 1;
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003135 }
3136 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
3137 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
3138 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
3139 vcpu->arch.switch_db_regs =
3140 (val & DR7_BP_EN_MASK);
3141 }
3142 break;
3143 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003144 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003145 skip_emulated_instruction(vcpu);
3146 return 1;
3147}
3148
Avi Kivity851ba692009-08-24 11:10:17 +03003149static int handle_cpuid(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003150{
Avi Kivity06465c52007-02-28 20:46:53 +02003151 kvm_emulate_cpuid(vcpu);
3152 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003153}
3154
Avi Kivity851ba692009-08-24 11:10:17 +03003155static int handle_rdmsr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003156{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003157 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
Avi Kivity6aa8b732006-12-10 02:21:36 -08003158 u64 data;
3159
3160 if (vmx_get_msr(vcpu, ecx, &data)) {
Avi Kivity59200272010-01-25 19:47:02 +02003161 trace_kvm_msr_read_ex(ecx);
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02003162 kvm_inject_gp(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003163 return 1;
3164 }
3165
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003166 trace_kvm_msr_read(ecx, data);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003167
Avi Kivity6aa8b732006-12-10 02:21:36 -08003168 /* FIXME: handling of bits 32:63 of rax, rdx */
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003169 vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
3170 vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003171 skip_emulated_instruction(vcpu);
3172 return 1;
3173}
3174
Avi Kivity851ba692009-08-24 11:10:17 +03003175static int handle_wrmsr(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003176{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003177 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
3178 u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
3179 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003180
3181 if (vmx_set_msr(vcpu, ecx, data) != 0) {
Avi Kivity59200272010-01-25 19:47:02 +02003182 trace_kvm_msr_write_ex(ecx, data);
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02003183 kvm_inject_gp(vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003184 return 1;
3185 }
3186
Avi Kivity59200272010-01-25 19:47:02 +02003187 trace_kvm_msr_write(ecx, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003188 skip_emulated_instruction(vcpu);
3189 return 1;
3190}
3191
Avi Kivity851ba692009-08-24 11:10:17 +03003192static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003193{
3194 return 1;
3195}
3196
Avi Kivity851ba692009-08-24 11:10:17 +03003197static int handle_interrupt_window(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003198{
Eddie Dong85f455f2007-07-06 12:20:49 +03003199 u32 cpu_based_vm_exec_control;
3200
3201 /* clear pending irq */
3202 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3203 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
3204 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003205
Jan Kiszkaa26bf122008-09-26 09:30:45 +02003206 ++vcpu->stat.irq_window_exits;
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003207
Dor Laorc1150d82007-01-05 16:36:24 -08003208 /*
3209 * If the user space waits to inject interrupts, exit as soon as
3210 * possible
3211 */
Gleb Natapov80618232009-04-21 17:44:56 +03003212 if (!irqchip_in_kernel(vcpu->kvm) &&
Avi Kivity851ba692009-08-24 11:10:17 +03003213 vcpu->run->request_interrupt_window &&
Gleb Natapov80618232009-04-21 17:44:56 +03003214 !kvm_cpu_has_interrupt(vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003215 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
Dor Laorc1150d82007-01-05 16:36:24 -08003216 return 0;
3217 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003218 return 1;
3219}
3220
Avi Kivity851ba692009-08-24 11:10:17 +03003221static int handle_halt(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003222{
3223 skip_emulated_instruction(vcpu);
Avi Kivityd3bef152007-06-05 15:53:05 +03003224 return kvm_emulate_halt(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003225}
3226
Avi Kivity851ba692009-08-24 11:10:17 +03003227static int handle_vmcall(struct kvm_vcpu *vcpu)
Ingo Molnarc21415e2007-02-19 14:37:47 +02003228{
Dor Laor510043d2007-02-19 18:25:43 +02003229 skip_emulated_instruction(vcpu);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05003230 kvm_emulate_hypercall(vcpu);
3231 return 1;
Ingo Molnarc21415e2007-02-19 14:37:47 +02003232}
3233
Avi Kivity851ba692009-08-24 11:10:17 +03003234static int handle_vmx_insn(struct kvm_vcpu *vcpu)
Avi Kivitye3c7cb62009-06-16 14:19:52 +03003235{
3236 kvm_queue_exception(vcpu, UD_VECTOR);
3237 return 1;
3238}
3239
Avi Kivity851ba692009-08-24 11:10:17 +03003240static int handle_invlpg(struct kvm_vcpu *vcpu)
Marcelo Tosattia7052892008-09-23 13:18:35 -03003241{
Sheng Yangf9c617f2009-03-25 10:08:52 +08003242 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Marcelo Tosattia7052892008-09-23 13:18:35 -03003243
3244 kvm_mmu_invlpg(vcpu, exit_qualification);
3245 skip_emulated_instruction(vcpu);
3246 return 1;
3247}
3248
Avi Kivity851ba692009-08-24 11:10:17 +03003249static int handle_wbinvd(struct kvm_vcpu *vcpu)
Eddie Donge5edaa02007-11-11 12:28:35 +02003250{
3251 skip_emulated_instruction(vcpu);
3252 /* TODO: Add support for VT-d/pass-through device */
3253 return 1;
3254}
3255
Avi Kivity851ba692009-08-24 11:10:17 +03003256static int handle_apic_access(struct kvm_vcpu *vcpu)
Sheng Yangf78e0e22007-10-29 09:40:42 +08003257{
Sheng Yangf9c617f2009-03-25 10:08:52 +08003258 unsigned long exit_qualification;
Sheng Yangf78e0e22007-10-29 09:40:42 +08003259 enum emulation_result er;
3260 unsigned long offset;
3261
Sheng Yangf9c617f2009-03-25 10:08:52 +08003262 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Sheng Yangf78e0e22007-10-29 09:40:42 +08003263 offset = exit_qualification & 0xffful;
3264
Avi Kivity851ba692009-08-24 11:10:17 +03003265 er = emulate_instruction(vcpu, 0, 0, 0);
Sheng Yangf78e0e22007-10-29 09:40:42 +08003266
3267 if (er != EMULATE_DONE) {
3268 printk(KERN_ERR
3269 "Fail to handle apic access vmexit! Offset is 0x%lx\n",
3270 offset);
Jan Kiszka7f582ab2009-07-22 23:53:01 +02003271 return -ENOEXEC;
Sheng Yangf78e0e22007-10-29 09:40:42 +08003272 }
3273 return 1;
3274}
3275
Avi Kivity851ba692009-08-24 11:10:17 +03003276static int handle_task_switch(struct kvm_vcpu *vcpu)
Izik Eidus37817f22008-03-24 23:14:53 +02003277{
Jan Kiszka60637aa2008-09-26 09:30:47 +02003278 struct vcpu_vmx *vmx = to_vmx(vcpu);
Izik Eidus37817f22008-03-24 23:14:53 +02003279 unsigned long exit_qualification;
3280 u16 tss_selector;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003281 int reason, type, idt_v;
3282
3283 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
3284 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
Izik Eidus37817f22008-03-24 23:14:53 +02003285
3286 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3287
3288 reason = (u32)exit_qualification >> 30;
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003289 if (reason == TASK_SWITCH_GATE && idt_v) {
3290 switch (type) {
3291 case INTR_TYPE_NMI_INTR:
3292 vcpu->arch.nmi_injected = false;
3293 if (cpu_has_virtual_nmis())
3294 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3295 GUEST_INTR_STATE_NMI);
3296 break;
3297 case INTR_TYPE_EXT_INTR:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003298 case INTR_TYPE_SOFT_INTR:
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003299 kvm_clear_interrupt_queue(vcpu);
3300 break;
3301 case INTR_TYPE_HARD_EXCEPTION:
3302 case INTR_TYPE_SOFT_EXCEPTION:
3303 kvm_clear_exception_queue(vcpu);
3304 break;
3305 default:
3306 break;
3307 }
Jan Kiszka60637aa2008-09-26 09:30:47 +02003308 }
Izik Eidus37817f22008-03-24 23:14:53 +02003309 tss_selector = exit_qualification;
3310
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003311 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
3312 type != INTR_TYPE_EXT_INTR &&
3313 type != INTR_TYPE_NMI_INTR))
3314 skip_emulated_instruction(vcpu);
3315
Jan Kiszka42dbaa52008-12-15 13:52:10 +01003316 if (!kvm_task_switch(vcpu, tss_selector, reason))
3317 return 0;
3318
3319 /* clear all local breakpoint enable flags */
3320 vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);
3321
3322 /*
3323 * TODO: What about debug traps on tss switch?
3324 * Are we supposed to inject them and update dr6?
3325 */
3326
3327 return 1;
Izik Eidus37817f22008-03-24 23:14:53 +02003328}
3329
Avi Kivity851ba692009-08-24 11:10:17 +03003330static int handle_ept_violation(struct kvm_vcpu *vcpu)
Sheng Yang14394422008-04-28 12:24:45 +08003331{
Sheng Yangf9c617f2009-03-25 10:08:52 +08003332 unsigned long exit_qualification;
Sheng Yang14394422008-04-28 12:24:45 +08003333 gpa_t gpa;
Sheng Yang14394422008-04-28 12:24:45 +08003334 int gla_validity;
Sheng Yang14394422008-04-28 12:24:45 +08003335
Sheng Yangf9c617f2009-03-25 10:08:52 +08003336 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
Sheng Yang14394422008-04-28 12:24:45 +08003337
3338 if (exit_qualification & (1 << 6)) {
3339 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
Jan Kiszka7f582ab2009-07-22 23:53:01 +02003340 return -EINVAL;
Sheng Yang14394422008-04-28 12:24:45 +08003341 }
3342
3343 gla_validity = (exit_qualification >> 7) & 0x3;
3344 if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
3345 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
3346 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
3347 (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
Sheng Yangf9c617f2009-03-25 10:08:52 +08003348 vmcs_readl(GUEST_LINEAR_ADDRESS));
Sheng Yang14394422008-04-28 12:24:45 +08003349 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
3350 (long unsigned int)exit_qualification);
Avi Kivity851ba692009-08-24 11:10:17 +03003351 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3352 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
Avi Kivity596ae892009-06-03 14:12:10 +03003353 return 0;
Sheng Yang14394422008-04-28 12:24:45 +08003354 }
3355
3356 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003357 trace_kvm_page_fault(gpa, exit_qualification);
Sheng Yang49cd7d22009-02-11 13:50:40 +08003358 return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
Sheng Yang14394422008-04-28 12:24:45 +08003359}
3360
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003361static u64 ept_rsvd_mask(u64 spte, int level)
3362{
3363 int i;
3364 u64 mask = 0;
3365
3366 for (i = 51; i > boot_cpu_data.x86_phys_bits; i--)
3367 mask |= (1ULL << i);
3368
3369 if (level > 2)
3370 /* bits 7:3 reserved */
3371 mask |= 0xf8;
3372 else if (level == 2) {
3373 if (spte & (1ULL << 7))
3374 /* 2MB ref, bits 20:12 reserved */
3375 mask |= 0x1ff000;
3376 else
3377 /* bits 6:3 reserved */
3378 mask |= 0x78;
3379 }
3380
3381 return mask;
3382}
3383
3384static void ept_misconfig_inspect_spte(struct kvm_vcpu *vcpu, u64 spte,
3385 int level)
3386{
3387 printk(KERN_ERR "%s: spte 0x%llx level %d\n", __func__, spte, level);
3388
3389 /* 010b (write-only) */
3390 WARN_ON((spte & 0x7) == 0x2);
3391
3392 /* 110b (write/execute) */
3393 WARN_ON((spte & 0x7) == 0x6);
3394
3395 /* 100b (execute-only) and value not supported by logical processor */
3396 if (!cpu_has_vmx_ept_execute_only())
3397 WARN_ON((spte & 0x7) == 0x4);
3398
3399 /* not 000b */
3400 if ((spte & 0x7)) {
3401 u64 rsvd_bits = spte & ept_rsvd_mask(spte, level);
3402
3403 if (rsvd_bits != 0) {
3404 printk(KERN_ERR "%s: rsvd_bits = 0x%llx\n",
3405 __func__, rsvd_bits);
3406 WARN_ON(1);
3407 }
3408
3409 if (level == 1 || (level == 2 && (spte & (1ULL << 7)))) {
3410 u64 ept_mem_type = (spte & 0x38) >> 3;
3411
3412 if (ept_mem_type == 2 || ept_mem_type == 3 ||
3413 ept_mem_type == 7) {
3414 printk(KERN_ERR "%s: ept_mem_type=0x%llx\n",
3415 __func__, ept_mem_type);
3416 WARN_ON(1);
3417 }
3418 }
3419 }
3420}
3421
Avi Kivity851ba692009-08-24 11:10:17 +03003422static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003423{
3424 u64 sptes[4];
3425 int nr_sptes, i;
3426 gpa_t gpa;
3427
3428 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
3429
3430 printk(KERN_ERR "EPT: Misconfiguration.\n");
3431 printk(KERN_ERR "EPT: GPA: 0x%llx\n", gpa);
3432
3433 nr_sptes = kvm_mmu_get_spte_hierarchy(vcpu, gpa, sptes);
3434
3435 for (i = PT64_ROOT_LEVEL; i > PT64_ROOT_LEVEL - nr_sptes; --i)
3436 ept_misconfig_inspect_spte(vcpu, sptes[i-1], i);
3437
Avi Kivity851ba692009-08-24 11:10:17 +03003438 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3439 vcpu->run->hw.hardware_exit_reason = EXIT_REASON_EPT_MISCONFIG;
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003440
3441 return 0;
3442}
3443
Avi Kivity851ba692009-08-24 11:10:17 +03003444static int handle_nmi_window(struct kvm_vcpu *vcpu)
Sheng Yangf08864b2008-05-15 18:23:25 +08003445{
3446 u32 cpu_based_vm_exec_control;
3447
3448 /* clear pending NMI */
3449 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3450 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
3451 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
3452 ++vcpu->stat.nmi_window_exits;
3453
3454 return 1;
3455}
3456
Mohammed Gamal80ced182009-09-01 12:48:18 +02003457static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003458{
Avi Kivity8b3079a2009-01-05 12:10:54 +02003459 struct vcpu_vmx *vmx = to_vmx(vcpu);
3460 enum emulation_result err = EMULATE_DONE;
Mohammed Gamal80ced182009-09-01 12:48:18 +02003461 int ret = 1;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003462
3463 while (!guest_state_valid(vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003464 err = emulate_instruction(vcpu, 0, 0, 0);
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003465
Mohammed Gamal80ced182009-09-01 12:48:18 +02003466 if (err == EMULATE_DO_MMIO) {
3467 ret = 0;
3468 goto out;
3469 }
Guillaume Thouvenin1d5a4d92008-10-29 09:39:42 +01003470
3471 if (err != EMULATE_DONE) {
Mohammed Gamal80ced182009-09-01 12:48:18 +02003472 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3473 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
Avi Kivitya9c73992009-11-04 11:54:59 +02003474 vcpu->run->internal.ndata = 0;
Mohammed Gamal80ced182009-09-01 12:48:18 +02003475 ret = 0;
3476 goto out;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003477 }
3478
3479 if (signal_pending(current))
Mohammed Gamal80ced182009-09-01 12:48:18 +02003480 goto out;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003481 if (need_resched())
3482 schedule();
3483 }
3484
Mohammed Gamal80ced182009-09-01 12:48:18 +02003485 vmx->emulation_required = 0;
3486out:
3487 return ret;
Mohammed Gamalea953ef2008-08-17 16:47:05 +03003488}
3489
Avi Kivity6aa8b732006-12-10 02:21:36 -08003490/*
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003491 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
3492 * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
3493 */
Marcelo Tosatti9fb41ba2009-10-12 19:37:31 -03003494static int handle_pause(struct kvm_vcpu *vcpu)
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003495{
3496 skip_emulated_instruction(vcpu);
3497 kvm_vcpu_on_spin(vcpu);
3498
3499 return 1;
3500}
3501
Sheng Yang59708672009-12-15 13:29:54 +08003502static int handle_invalid_op(struct kvm_vcpu *vcpu)
3503{
3504 kvm_queue_exception(vcpu, UD_VECTOR);
3505 return 1;
3506}
3507
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003508/*
Avi Kivity6aa8b732006-12-10 02:21:36 -08003509 * The exit handlers return 1 if the exit was handled fully and guest execution
3510 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
3511 * to be done to userspace and return 0.
3512 */
Avi Kivity851ba692009-08-24 11:10:17 +03003513static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08003514 [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
3515 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
Avi Kivity988ad742007-02-12 00:54:36 -08003516 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
Sheng Yangf08864b2008-05-15 18:23:25 +08003517 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003518 [EXIT_REASON_IO_INSTRUCTION] = handle_io,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003519 [EXIT_REASON_CR_ACCESS] = handle_cr,
3520 [EXIT_REASON_DR_ACCESS] = handle_dr,
3521 [EXIT_REASON_CPUID] = handle_cpuid,
3522 [EXIT_REASON_MSR_READ] = handle_rdmsr,
3523 [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
3524 [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
3525 [EXIT_REASON_HLT] = handle_halt,
Marcelo Tosattia7052892008-09-23 13:18:35 -03003526 [EXIT_REASON_INVLPG] = handle_invlpg,
Ingo Molnarc21415e2007-02-19 14:37:47 +02003527 [EXIT_REASON_VMCALL] = handle_vmcall,
Avi Kivitye3c7cb62009-06-16 14:19:52 +03003528 [EXIT_REASON_VMCLEAR] = handle_vmx_insn,
3529 [EXIT_REASON_VMLAUNCH] = handle_vmx_insn,
3530 [EXIT_REASON_VMPTRLD] = handle_vmx_insn,
3531 [EXIT_REASON_VMPTRST] = handle_vmx_insn,
3532 [EXIT_REASON_VMREAD] = handle_vmx_insn,
3533 [EXIT_REASON_VMRESUME] = handle_vmx_insn,
3534 [EXIT_REASON_VMWRITE] = handle_vmx_insn,
3535 [EXIT_REASON_VMOFF] = handle_vmx_insn,
3536 [EXIT_REASON_VMON] = handle_vmx_insn,
Sheng Yangf78e0e22007-10-29 09:40:42 +08003537 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
3538 [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
Eddie Donge5edaa02007-11-11 12:28:35 +02003539 [EXIT_REASON_WBINVD] = handle_wbinvd,
Izik Eidus37817f22008-03-24 23:14:53 +02003540 [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
Andi Kleena0861c02009-06-08 17:37:09 +08003541 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
Marcelo Tosatti68f89402009-06-11 12:07:43 -03003542 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
3543 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig,
Zhai, Edwin4b8d54f2009-10-09 18:03:20 +08003544 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause,
Sheng Yang59708672009-12-15 13:29:54 +08003545 [EXIT_REASON_MWAIT_INSTRUCTION] = handle_invalid_op,
3546 [EXIT_REASON_MONITOR_INSTRUCTION] = handle_invalid_op,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003547};
3548
3549static const int kvm_vmx_max_exit_handlers =
Robert P. J. Day50a34852007-06-03 13:35:29 -04003550 ARRAY_SIZE(kvm_vmx_exit_handlers);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003551
3552/*
3553 * The guest has exited. See if we can fix it or if we need userspace
3554 * assistance.
3555 */
Avi Kivity851ba692009-08-24 11:10:17 +03003556static int vmx_handle_exit(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003557{
Avi Kivity29bd8a72007-09-10 17:27:03 +03003558 struct vcpu_vmx *vmx = to_vmx(vcpu);
Andi Kleena0861c02009-06-08 17:37:09 +08003559 u32 exit_reason = vmx->exit_reason;
Avi Kivity1155f762007-11-22 11:30:47 +02003560 u32 vectoring_info = vmx->idt_vectoring_info;
Avi Kivity29bd8a72007-09-10 17:27:03 +03003561
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003562 trace_kvm_exit(exit_reason, kvm_rip_read(vcpu));
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04003563
Mohammed Gamal80ced182009-09-01 12:48:18 +02003564 /* If guest state is invalid, start emulating */
3565 if (vmx->emulation_required && emulate_invalid_guest_state)
3566 return handle_invalid_guest_state(vcpu);
Guillaume Thouvenin1d5a4d92008-10-29 09:39:42 +01003567
Sheng Yang14394422008-04-28 12:24:45 +08003568 /* Access CR3 don't cause VMExit in paging mode, so we need
3569 * to sync with guest real CR3. */
Avi Kivity6de4f3a2009-05-31 22:58:47 +03003570 if (enable_ept && is_paging(vcpu))
Sheng Yang14394422008-04-28 12:24:45 +08003571 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
Sheng Yang14394422008-04-28 12:24:45 +08003572
Avi Kivity29bd8a72007-09-10 17:27:03 +03003573 if (unlikely(vmx->fail)) {
Avi Kivity851ba692009-08-24 11:10:17 +03003574 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3575 vcpu->run->fail_entry.hardware_entry_failure_reason
Avi Kivity29bd8a72007-09-10 17:27:03 +03003576 = vmcs_read32(VM_INSTRUCTION_ERROR);
3577 return 0;
3578 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08003579
Mike Dayd77c26f2007-10-08 09:02:08 -04003580 if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
Sheng Yang14394422008-04-28 12:24:45 +08003581 (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
Jan Kiszka60637aa2008-09-26 09:30:47 +02003582 exit_reason != EXIT_REASON_EPT_VIOLATION &&
3583 exit_reason != EXIT_REASON_TASK_SWITCH))
3584 printk(KERN_WARNING "%s: unexpected, valid vectoring info "
3585 "(0x%x) and exit reason is 0x%x\n",
3586 __func__, vectoring_info, exit_reason);
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003587
3588 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
Gleb Natapovc4282df2009-04-21 17:45:07 +03003589 if (vmx_interrupt_allowed(vcpu)) {
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003590 vmx->soft_vnmi_blocked = 0;
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003591 } else if (vmx->vnmi_blocked_time > 1000000000LL &&
Jan Kiszka45312202008-12-11 16:54:54 +01003592 vcpu->arch.nmi_pending) {
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003593 /*
3594 * This CPU don't support us in finding the end of an
3595 * NMI-blocked window if the guest runs with IRQs
3596 * disabled. So we pull the trigger after 1 s of
3597 * futile waiting, but inform the user about this.
3598 */
3599 printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
3600 "state on VCPU %d after 1 s timeout\n",
3601 __func__, vcpu->vcpu_id);
3602 vmx->soft_vnmi_blocked = 0;
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003603 }
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003604 }
3605
Avi Kivity6aa8b732006-12-10 02:21:36 -08003606 if (exit_reason < kvm_vmx_max_exit_handlers
3607 && kvm_vmx_exit_handlers[exit_reason])
Avi Kivity851ba692009-08-24 11:10:17 +03003608 return kvm_vmx_exit_handlers[exit_reason](vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003609 else {
Avi Kivity851ba692009-08-24 11:10:17 +03003610 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
3611 vcpu->run->hw.hardware_exit_reason = exit_reason;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003612 }
3613 return 0;
3614}
3615
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003616static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003617{
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003618 if (irr == -1 || tpr < irr) {
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003619 vmcs_write32(TPR_THRESHOLD, 0);
3620 return;
3621 }
3622
Gleb Natapov95ba8273132009-04-21 17:45:08 +03003623 vmcs_write32(TPR_THRESHOLD, irr);
Yang, Sheng6e5d8652007-09-12 18:03:11 +08003624}
3625
Avi Kivitycf393f72008-07-01 16:20:21 +03003626static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
3627{
3628 u32 exit_intr_info;
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003629 u32 idt_vectoring_info = vmx->idt_vectoring_info;
Avi Kivitycf393f72008-07-01 16:20:21 +03003630 bool unblock_nmi;
3631 u8 vector;
Avi Kivity668f6122008-07-02 09:28:55 +03003632 int type;
3633 bool idtv_info_valid;
Avi Kivitycf393f72008-07-01 16:20:21 +03003634
3635 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
Gleb Natapov20f65982009-05-11 13:35:55 +03003636
Andi Kleena0861c02009-06-08 17:37:09 +08003637 vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
3638
3639 /* Handle machine checks before interrupts are enabled */
3640 if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
3641 || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
3642 && is_machine_check(exit_intr_info)))
3643 kvm_machine_check();
3644
Gleb Natapov20f65982009-05-11 13:35:55 +03003645 /* We need to handle NMIs before interrupts are enabled */
3646 if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
Marcelo Tosatti229456f2009-06-17 09:22:14 -03003647 (exit_intr_info & INTR_INFO_VALID_MASK))
Gleb Natapov20f65982009-05-11 13:35:55 +03003648 asm("int $2");
Gleb Natapov20f65982009-05-11 13:35:55 +03003649
3650 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3651
Avi Kivitycf393f72008-07-01 16:20:21 +03003652 if (cpu_has_virtual_nmis()) {
3653 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
3654 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
3655 /*
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003656 * SDM 3: 27.7.1.2 (September 2008)
Avi Kivitycf393f72008-07-01 16:20:21 +03003657 * Re-set bit "block by NMI" before VM entry if vmexit caused by
3658 * a guest IRET fault.
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003659 * SDM 3: 23.2.2 (September 2008)
3660 * Bit 12 is undefined in any of the following cases:
3661 * If the VM exit sets the valid bit in the IDT-vectoring
3662 * information field.
3663 * If the VM exit is due to a double fault.
Avi Kivitycf393f72008-07-01 16:20:21 +03003664 */
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003665 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
3666 vector != DF_VECTOR && !idtv_info_valid)
Avi Kivitycf393f72008-07-01 16:20:21 +03003667 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3668 GUEST_INTR_STATE_NMI);
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003669 } else if (unlikely(vmx->soft_vnmi_blocked))
3670 vmx->vnmi_blocked_time +=
3671 ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
Avi Kivity668f6122008-07-02 09:28:55 +03003672
Gleb Natapov37b96e92009-03-30 16:03:13 +03003673 vmx->vcpu.arch.nmi_injected = false;
3674 kvm_clear_exception_queue(&vmx->vcpu);
3675 kvm_clear_interrupt_queue(&vmx->vcpu);
3676
3677 if (!idtv_info_valid)
3678 return;
3679
Avi Kivity668f6122008-07-02 09:28:55 +03003680 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
3681 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
Gleb Natapov37b96e92009-03-30 16:03:13 +03003682
Gleb Natapov64a7ec02009-03-30 16:03:29 +03003683 switch (type) {
Gleb Natapov37b96e92009-03-30 16:03:13 +03003684 case INTR_TYPE_NMI_INTR:
3685 vmx->vcpu.arch.nmi_injected = true;
Avi Kivity668f6122008-07-02 09:28:55 +03003686 /*
Gleb Natapov7b4a25c2009-03-30 16:03:08 +03003687 * SDM 3: 27.7.1.2 (September 2008)
Gleb Natapov37b96e92009-03-30 16:03:13 +03003688 * Clear bit "block by NMI" before VM entry if a NMI
3689 * delivery faulted.
Avi Kivity668f6122008-07-02 09:28:55 +03003690 */
Gleb Natapov37b96e92009-03-30 16:03:13 +03003691 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
3692 GUEST_INTR_STATE_NMI);
3693 break;
Gleb Natapov37b96e92009-03-30 16:03:13 +03003694 case INTR_TYPE_SOFT_EXCEPTION:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003695 vmx->vcpu.arch.event_exit_inst_len =
3696 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3697 /* fall through */
3698 case INTR_TYPE_HARD_EXCEPTION:
Avi Kivity35920a32008-07-03 14:50:12 +03003699 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
Gleb Natapov37b96e92009-03-30 16:03:13 +03003700 u32 err = vmcs_read32(IDT_VECTORING_ERROR_CODE);
3701 kvm_queue_exception_e(&vmx->vcpu, vector, err);
Avi Kivity35920a32008-07-03 14:50:12 +03003702 } else
3703 kvm_queue_exception(&vmx->vcpu, vector);
Gleb Natapov37b96e92009-03-30 16:03:13 +03003704 break;
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003705 case INTR_TYPE_SOFT_INTR:
3706 vmx->vcpu.arch.event_exit_inst_len =
3707 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3708 /* fall through */
Gleb Natapov37b96e92009-03-30 16:03:13 +03003709 case INTR_TYPE_EXT_INTR:
Gleb Natapov66fd3f72009-05-11 13:35:50 +03003710 kvm_queue_interrupt(&vmx->vcpu, vector,
3711 type == INTR_TYPE_SOFT_INTR);
Gleb Natapov37b96e92009-03-30 16:03:13 +03003712 break;
3713 default:
3714 break;
Avi Kivityf7d92382008-07-03 16:14:28 +03003715 }
Avi Kivitycf393f72008-07-01 16:20:21 +03003716}
3717
Avi Kivity9c8cba32007-11-22 11:42:59 +02003718/*
3719 * Failure to inject an interrupt should give us the information
3720 * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs
3721 * when fetching the interrupt redirection bitmap in the real-mode
3722 * tss, this doesn't happen. So we do it ourselves.
3723 */
3724static void fixup_rmode_irq(struct vcpu_vmx *vmx)
3725{
3726 vmx->rmode.irq.pending = 0;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003727 if (kvm_rip_read(&vmx->vcpu) + 1 != vmx->rmode.irq.rip)
Avi Kivity9c8cba32007-11-22 11:42:59 +02003728 return;
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003729 kvm_rip_write(&vmx->vcpu, vmx->rmode.irq.rip);
Avi Kivity9c8cba32007-11-22 11:42:59 +02003730 if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {
3731 vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK;
3732 vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR;
3733 return;
3734 }
3735 vmx->idt_vectoring_info =
3736 VECTORING_INFO_VALID_MASK
3737 | INTR_TYPE_EXT_INTR
3738 | vmx->rmode.irq.vector;
3739}
3740
Avi Kivityc8019492008-07-14 14:44:59 +03003741#ifdef CONFIG_X86_64
3742#define R "r"
3743#define Q "q"
3744#else
3745#define R "e"
3746#define Q "l"
3747#endif
3748
Avi Kivity851ba692009-08-24 11:10:17 +03003749static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003750{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003751 struct vcpu_vmx *vmx = to_vmx(vcpu);
Avi Kivitye6adf282007-04-30 16:07:54 +03003752
Jan Kiszka3b86cd92008-09-26 09:30:57 +02003753 /* Record the guest's net vcpu time for enforced NMI injections. */
3754 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
3755 vmx->entry_time = ktime_get();
3756
Mohammed Gamal80ced182009-09-01 12:48:18 +02003757 /* Don't enter VMX if guest state is invalid, let the exit handler
3758 start emulation until we arrive back to a valid state */
3759 if (vmx->emulation_required && emulate_invalid_guest_state)
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03003760 return;
Mohammed Gamala89a8fb2008-08-17 16:42:16 +03003761
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003762 if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
3763 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
3764 if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
3765 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
3766
Gleb Natapov787ff732009-05-18 11:44:06 +03003767 /* When single-stepping over STI and MOV SS, we must clear the
3768 * corresponding interruptibility bits in the guest state. Otherwise
3769 * vmentry fails as it then expects bit 14 (BS) in pending debug
3770 * exceptions being set, but that's not correct for the guest debugging
3771 * case. */
3772 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
3773 vmx_set_interrupt_shadow(vcpu, 0);
3774
Avi Kivitye6adf282007-04-30 16:07:54 +03003775 /*
3776 * Loading guest fpu may have cleared host cr0.ts
3777 */
3778 vmcs_writel(HOST_CR0, read_cr0());
3779
Mike Dayd77c26f2007-10-08 09:02:08 -04003780 asm(
Avi Kivity6aa8b732006-12-10 02:21:36 -08003781 /* Store host registers */
Avi Kivityc8019492008-07-14 14:44:59 +03003782 "push %%"R"dx; push %%"R"bp;"
3783 "push %%"R"cx \n\t"
Avi Kivity313dbd42008-07-17 18:04:30 +03003784 "cmp %%"R"sp, %c[host_rsp](%0) \n\t"
3785 "je 1f \n\t"
3786 "mov %%"R"sp, %c[host_rsp](%0) \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003787 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
Avi Kivity313dbd42008-07-17 18:04:30 +03003788 "1: \n\t"
Avi Kivityd3edefc2009-06-16 12:33:56 +03003789 /* Reload cr2 if changed */
3790 "mov %c[cr2](%0), %%"R"ax \n\t"
3791 "mov %%cr2, %%"R"dx \n\t"
3792 "cmp %%"R"ax, %%"R"dx \n\t"
3793 "je 2f \n\t"
3794 "mov %%"R"ax, %%cr2 \n\t"
3795 "2: \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003796 /* Check if vmlaunch of vmresume is needed */
Avi Kivitye08aa782007-11-15 18:06:18 +02003797 "cmpl $0, %c[launched](%0) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003798 /* Load guest registers. Don't clobber flags. */
Avi Kivityc8019492008-07-14 14:44:59 +03003799 "mov %c[rax](%0), %%"R"ax \n\t"
3800 "mov %c[rbx](%0), %%"R"bx \n\t"
3801 "mov %c[rdx](%0), %%"R"dx \n\t"
3802 "mov %c[rsi](%0), %%"R"si \n\t"
3803 "mov %c[rdi](%0), %%"R"di \n\t"
3804 "mov %c[rbp](%0), %%"R"bp \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003805#ifdef CONFIG_X86_64
Avi Kivitye08aa782007-11-15 18:06:18 +02003806 "mov %c[r8](%0), %%r8 \n\t"
3807 "mov %c[r9](%0), %%r9 \n\t"
3808 "mov %c[r10](%0), %%r10 \n\t"
3809 "mov %c[r11](%0), %%r11 \n\t"
3810 "mov %c[r12](%0), %%r12 \n\t"
3811 "mov %c[r13](%0), %%r13 \n\t"
3812 "mov %c[r14](%0), %%r14 \n\t"
3813 "mov %c[r15](%0), %%r15 \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003814#endif
Avi Kivityc8019492008-07-14 14:44:59 +03003815 "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */
3816
Avi Kivity6aa8b732006-12-10 02:21:36 -08003817 /* Enter guest mode */
Avi Kivitycd2276a2007-05-14 20:41:13 +03003818 "jne .Llaunched \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003819 __ex(ASM_VMX_VMLAUNCH) "\n\t"
Avi Kivitycd2276a2007-05-14 20:41:13 +03003820 "jmp .Lkvm_vmx_return \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03003821 ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
Avi Kivitycd2276a2007-05-14 20:41:13 +03003822 ".Lkvm_vmx_return: "
Avi Kivity6aa8b732006-12-10 02:21:36 -08003823 /* Save guest registers, load host registers, keep flags */
Avi Kivityc8019492008-07-14 14:44:59 +03003824 "xchg %0, (%%"R"sp) \n\t"
3825 "mov %%"R"ax, %c[rax](%0) \n\t"
3826 "mov %%"R"bx, %c[rbx](%0) \n\t"
3827 "push"Q" (%%"R"sp); pop"Q" %c[rcx](%0) \n\t"
3828 "mov %%"R"dx, %c[rdx](%0) \n\t"
3829 "mov %%"R"si, %c[rsi](%0) \n\t"
3830 "mov %%"R"di, %c[rdi](%0) \n\t"
3831 "mov %%"R"bp, %c[rbp](%0) \n\t"
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003832#ifdef CONFIG_X86_64
Avi Kivitye08aa782007-11-15 18:06:18 +02003833 "mov %%r8, %c[r8](%0) \n\t"
3834 "mov %%r9, %c[r9](%0) \n\t"
3835 "mov %%r10, %c[r10](%0) \n\t"
3836 "mov %%r11, %c[r11](%0) \n\t"
3837 "mov %%r12, %c[r12](%0) \n\t"
3838 "mov %%r13, %c[r13](%0) \n\t"
3839 "mov %%r14, %c[r14](%0) \n\t"
3840 "mov %%r15, %c[r15](%0) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08003841#endif
Avi Kivityc8019492008-07-14 14:44:59 +03003842 "mov %%cr2, %%"R"ax \n\t"
3843 "mov %%"R"ax, %c[cr2](%0) \n\t"
3844
3845 "pop %%"R"bp; pop %%"R"bp; pop %%"R"dx \n\t"
Avi Kivitye08aa782007-11-15 18:06:18 +02003846 "setbe %c[fail](%0) \n\t"
3847 : : "c"(vmx), "d"((unsigned long)HOST_RSP),
3848 [launched]"i"(offsetof(struct vcpu_vmx, launched)),
3849 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
Avi Kivity313dbd42008-07-17 18:04:30 +03003850 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003851 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
3852 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
3853 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
3854 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
3855 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
3856 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
3857 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
Avi Kivity05b3e0c2006-12-13 00:33:45 -08003858#ifdef CONFIG_X86_64
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003859 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
3860 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
3861 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
3862 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
3863 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
3864 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
3865 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
3866 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
Avi Kivity6aa8b732006-12-10 02:21:36 -08003867#endif
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003868 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2))
Laurent Vivierc2036302007-10-25 14:18:52 +02003869 : "cc", "memory"
Avi Kivityc8019492008-07-14 14:44:59 +03003870 , R"bx", R"di", R"si"
Laurent Vivierc2036302007-10-25 14:18:52 +02003871#ifdef CONFIG_X86_64
Laurent Vivierc2036302007-10-25 14:18:52 +02003872 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
3873#endif
3874 );
Avi Kivity6aa8b732006-12-10 02:21:36 -08003875
Avi Kivity6de4f3a2009-05-31 22:58:47 +03003876 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
3877 | (1 << VCPU_EXREG_PDPTR));
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03003878 vcpu->arch.regs_dirty = 0;
3879
Avi Kivity1155f762007-11-22 11:30:47 +02003880 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
Avi Kivity9c8cba32007-11-22 11:42:59 +02003881 if (vmx->rmode.irq.pending)
3882 fixup_rmode_irq(vmx);
Avi Kivity1155f762007-11-22 11:30:47 +02003883
Mike Dayd77c26f2007-10-08 09:02:08 -04003884 asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
Avi Kivity15ad7142007-07-11 18:17:21 +03003885 vmx->launched = 1;
Avi Kivity1b6269d2007-10-09 12:12:19 +02003886
Avi Kivitycf393f72008-07-01 16:20:21 +03003887 vmx_complete_interrupts(vmx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003888}
3889
Avi Kivityc8019492008-07-14 14:44:59 +03003890#undef R
3891#undef Q
3892
Avi Kivity6aa8b732006-12-10 02:21:36 -08003893static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
3894{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003895 struct vcpu_vmx *vmx = to_vmx(vcpu);
3896
3897 if (vmx->vmcs) {
Avi Kivity543e4242008-05-13 16:22:47 +03003898 vcpu_clear(vmx);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003899 free_vmcs(vmx->vmcs);
3900 vmx->vmcs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003901 }
3902}
3903
3904static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
3905{
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003906 struct vcpu_vmx *vmx = to_vmx(vcpu);
3907
Sheng Yang2384d2b2008-01-17 15:14:33 +08003908 spin_lock(&vmx_vpid_lock);
3909 if (vmx->vpid != 0)
3910 __clear_bit(vmx->vpid, vmx_vpid_bitmap);
3911 spin_unlock(&vmx_vpid_lock);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003912 vmx_free_vmcs(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003913 kfree(vmx->guest_msrs);
3914 kvm_vcpu_uninit(vcpu);
Rusty Russella4770342007-08-01 14:46:11 +10003915 kmem_cache_free(kvm_vcpu_cache, vmx);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003916}
3917
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003918static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
Avi Kivity6aa8b732006-12-10 02:21:36 -08003919{
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003920 int err;
Rusty Russellc16f8622007-07-30 21:12:19 +10003921 struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
Avi Kivity15ad7142007-07-11 18:17:21 +03003922 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003923
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003924 if (!vmx)
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003925 return ERR_PTR(-ENOMEM);
3926
Sheng Yang2384d2b2008-01-17 15:14:33 +08003927 allocate_vpid(vmx);
3928
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003929 err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
3930 if (err)
3931 goto free_vcpu;
Ingo Molnar965b58a2007-01-05 16:36:23 -08003932
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003933 vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003934 if (!vmx->guest_msrs) {
3935 err = -ENOMEM;
3936 goto uninit_vcpu;
3937 }
Ingo Molnar965b58a2007-01-05 16:36:23 -08003938
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003939 vmx->vmcs = alloc_vmcs();
3940 if (!vmx->vmcs)
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003941 goto free_msrs;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04003942
3943 vmcs_clear(vmx->vmcs);
3944
Avi Kivity15ad7142007-07-11 18:17:21 +03003945 cpu = get_cpu();
3946 vmx_vcpu_load(&vmx->vcpu, cpu);
Rusty Russell8b9cf982007-07-30 16:31:43 +10003947 err = vmx_vcpu_setup(vmx);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003948 vmx_vcpu_put(&vmx->vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03003949 put_cpu();
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003950 if (err)
3951 goto free_vmcs;
Marcelo Tosatti5e4a0b32008-02-14 21:21:43 -02003952 if (vm_need_virtualize_apic_accesses(kvm))
3953 if (alloc_apic_access_page(kvm) != 0)
3954 goto free_vmcs;
Ingo Molnar965b58a2007-01-05 16:36:23 -08003955
Sheng Yangb927a3c2009-07-21 10:42:48 +08003956 if (enable_ept) {
3957 if (!kvm->arch.ept_identity_map_addr)
3958 kvm->arch.ept_identity_map_addr =
3959 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
Sheng Yangb7ebfb02008-04-25 21:44:52 +08003960 if (alloc_identity_pagetable(kvm) != 0)
3961 goto free_vmcs;
Sheng Yangb927a3c2009-07-21 10:42:48 +08003962 }
Sheng Yangb7ebfb02008-04-25 21:44:52 +08003963
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003964 return &vmx->vcpu;
Ingo Molnar965b58a2007-01-05 16:36:23 -08003965
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003966free_vmcs:
3967 free_vmcs(vmx->vmcs);
3968free_msrs:
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003969 kfree(vmx->guest_msrs);
3970uninit_vcpu:
3971 kvm_vcpu_uninit(&vmx->vcpu);
3972free_vcpu:
Rusty Russella4770342007-08-01 14:46:11 +10003973 kmem_cache_free(kvm_vcpu_cache, vmx);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10003974 return ERR_PTR(err);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003975}
3976
Yang, Sheng002c7f72007-07-31 14:23:01 +03003977static void __init vmx_check_processor_compat(void *rtn)
3978{
3979 struct vmcs_config vmcs_conf;
3980
3981 *(int *)rtn = 0;
3982 if (setup_vmcs_config(&vmcs_conf) < 0)
3983 *(int *)rtn = -EIO;
3984 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
3985 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
3986 smp_processor_id());
3987 *(int *)rtn = -EIO;
3988 }
3989}
3990
Sheng Yang67253af2008-04-25 10:20:22 +08003991static int get_ept_level(void)
3992{
3993 return VMX_EPT_DEFAULT_GAW + 1;
3994}
3995
Sheng Yang4b12f0d2009-04-27 20:35:42 +08003996static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
Sheng Yang64d4d522008-10-09 16:01:57 +08003997{
Sheng Yang4b12f0d2009-04-27 20:35:42 +08003998 u64 ret;
3999
Sheng Yang522c68c2009-04-27 20:35:43 +08004000 /* For VT-d and EPT combination
4001 * 1. MMIO: always map as UC
4002 * 2. EPT with VT-d:
4003 * a. VT-d without snooping control feature: can't guarantee the
4004 * result, try to trust guest.
4005 * b. VT-d with snooping control feature: snooping control feature of
4006 * VT-d engine can guarantee the cache correctness. Just set it
4007 * to WB to keep consistent with host. So the same as item 3.
4008 * 3. EPT without VT-d: always map as WB and set IGMT=1 to keep
4009 * consistent with host MTRR
4010 */
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004011 if (is_mmio)
4012 ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
Sheng Yang522c68c2009-04-27 20:35:43 +08004013 else if (vcpu->kvm->arch.iommu_domain &&
4014 !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
4015 ret = kvm_get_guest_memory_type(vcpu, gfn) <<
4016 VMX_EPT_MT_EPTE_SHIFT;
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004017 else
Sheng Yang522c68c2009-04-27 20:35:43 +08004018 ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
4019 | VMX_EPT_IGMT_BIT;
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004020
4021 return ret;
Sheng Yang64d4d522008-10-09 16:01:57 +08004022}
4023
Avi Kivityf4c9e872009-12-28 16:06:35 +02004024#define _ER(x) { EXIT_REASON_##x, #x }
4025
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004026static const struct trace_print_flags vmx_exit_reasons_str[] = {
Avi Kivityf4c9e872009-12-28 16:06:35 +02004027 _ER(EXCEPTION_NMI),
4028 _ER(EXTERNAL_INTERRUPT),
4029 _ER(TRIPLE_FAULT),
4030 _ER(PENDING_INTERRUPT),
4031 _ER(NMI_WINDOW),
4032 _ER(TASK_SWITCH),
4033 _ER(CPUID),
4034 _ER(HLT),
4035 _ER(INVLPG),
4036 _ER(RDPMC),
4037 _ER(RDTSC),
4038 _ER(VMCALL),
4039 _ER(VMCLEAR),
4040 _ER(VMLAUNCH),
4041 _ER(VMPTRLD),
4042 _ER(VMPTRST),
4043 _ER(VMREAD),
4044 _ER(VMRESUME),
4045 _ER(VMWRITE),
4046 _ER(VMOFF),
4047 _ER(VMON),
4048 _ER(CR_ACCESS),
4049 _ER(DR_ACCESS),
4050 _ER(IO_INSTRUCTION),
4051 _ER(MSR_READ),
4052 _ER(MSR_WRITE),
4053 _ER(MWAIT_INSTRUCTION),
4054 _ER(MONITOR_INSTRUCTION),
4055 _ER(PAUSE_INSTRUCTION),
4056 _ER(MCE_DURING_VMENTRY),
4057 _ER(TPR_BELOW_THRESHOLD),
4058 _ER(APIC_ACCESS),
4059 _ER(EPT_VIOLATION),
4060 _ER(EPT_MISCONFIG),
4061 _ER(WBINVD),
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004062 { -1, NULL }
4063};
4064
Avi Kivityf4c9e872009-12-28 16:06:35 +02004065#undef _ER
4066
Sheng Yang17cc3932010-01-05 19:02:27 +08004067static int vmx_get_lpage_level(void)
Joerg Roedel344f4142009-07-27 16:30:48 +02004068{
Sheng Yang878403b2010-01-05 19:02:29 +08004069 if (enable_ept && !cpu_has_vmx_ept_1g_page())
4070 return PT_DIRECTORY_LEVEL;
4071 else
4072 /* For shadow and EPT supported 1GB page */
4073 return PT_PDPE_LEVEL;
Joerg Roedel344f4142009-07-27 16:30:48 +02004074}
4075
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004076static inline u32 bit(int bitno)
4077{
4078 return 1 << (bitno & 31);
4079}
4080
Sheng Yang0e851882009-12-18 16:48:46 +08004081static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
4082{
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004083 struct kvm_cpuid_entry2 *best;
4084 struct vcpu_vmx *vmx = to_vmx(vcpu);
4085 u32 exec_control;
4086
4087 vmx->rdtscp_enabled = false;
4088 if (vmx_rdtscp_supported()) {
4089 exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
4090 if (exec_control & SECONDARY_EXEC_RDTSCP) {
4091 best = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
4092 if (best && (best->edx & bit(X86_FEATURE_RDTSCP)))
4093 vmx->rdtscp_enabled = true;
4094 else {
4095 exec_control &= ~SECONDARY_EXEC_RDTSCP;
4096 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
4097 exec_control);
4098 }
4099 }
4100 }
Sheng Yang0e851882009-12-18 16:48:46 +08004101}
4102
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03004103static struct kvm_x86_ops vmx_x86_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08004104 .cpu_has_kvm_support = cpu_has_kvm_support,
4105 .disabled_by_bios = vmx_disabled_by_bios,
4106 .hardware_setup = hardware_setup,
4107 .hardware_unsetup = hardware_unsetup,
Yang, Sheng002c7f72007-07-31 14:23:01 +03004108 .check_processor_compatibility = vmx_check_processor_compat,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004109 .hardware_enable = hardware_enable,
4110 .hardware_disable = hardware_disable,
Sheng Yang04547152009-04-01 15:52:31 +08004111 .cpu_has_accelerated_tpr = report_flexpriority,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004112
4113 .vcpu_create = vmx_create_vcpu,
4114 .vcpu_free = vmx_free_vcpu,
Avi Kivity04d2cc72007-09-10 18:10:54 +03004115 .vcpu_reset = vmx_vcpu_reset,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004116
Avi Kivity04d2cc72007-09-10 18:10:54 +03004117 .prepare_guest_switch = vmx_save_host_state,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004118 .vcpu_load = vmx_vcpu_load,
4119 .vcpu_put = vmx_vcpu_put,
4120
4121 .set_guest_debug = set_guest_debug,
4122 .get_msr = vmx_get_msr,
4123 .set_msr = vmx_set_msr,
4124 .get_segment_base = vmx_get_segment_base,
4125 .get_segment = vmx_get_segment,
4126 .set_segment = vmx_set_segment,
Izik Eidus2e4d2652008-03-24 19:38:34 +02004127 .get_cpl = vmx_get_cpl,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004128 .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
Avi Kivitye8467fd2009-12-29 18:43:06 +02004129 .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
Anthony Liguori25c4c272007-04-27 09:29:21 +03004130 .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004131 .set_cr0 = vmx_set_cr0,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004132 .set_cr3 = vmx_set_cr3,
4133 .set_cr4 = vmx_set_cr4,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004134 .set_efer = vmx_set_efer,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004135 .get_idt = vmx_get_idt,
4136 .set_idt = vmx_set_idt,
4137 .get_gdt = vmx_get_gdt,
4138 .set_gdt = vmx_set_gdt,
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03004139 .cache_reg = vmx_cache_reg,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004140 .get_rflags = vmx_get_rflags,
4141 .set_rflags = vmx_set_rflags,
Avi Kivity02daab22009-12-30 12:40:26 +02004142 .fpu_deactivate = vmx_fpu_deactivate,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004143
4144 .tlb_flush = vmx_flush_tlb,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004145
Avi Kivity6aa8b732006-12-10 02:21:36 -08004146 .run = vmx_vcpu_run,
Avi Kivity6062d012009-03-23 17:35:17 +02004147 .handle_exit = vmx_handle_exit,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004148 .skip_emulated_instruction = skip_emulated_instruction,
Glauber Costa2809f5d2009-05-12 16:21:05 -04004149 .set_interrupt_shadow = vmx_set_interrupt_shadow,
4150 .get_interrupt_shadow = vmx_get_interrupt_shadow,
Ingo Molnar102d8322007-02-19 14:37:47 +02004151 .patch_hypercall = vmx_patch_hypercall,
Eddie Dong2a8067f2007-08-06 16:29:07 +03004152 .set_irq = vmx_inject_irq,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004153 .set_nmi = vmx_inject_nmi,
Avi Kivity298101d2007-11-25 13:41:11 +02004154 .queue_exception = vmx_queue_exception,
Gleb Natapov78646122009-03-23 12:12:11 +02004155 .interrupt_allowed = vmx_interrupt_allowed,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004156 .nmi_allowed = vmx_nmi_allowed,
Jan Kiszka3cfc3092009-11-12 01:04:25 +01004157 .get_nmi_mask = vmx_get_nmi_mask,
4158 .set_nmi_mask = vmx_set_nmi_mask,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004159 .enable_nmi_window = enable_nmi_window,
4160 .enable_irq_window = enable_irq_window,
4161 .update_cr8_intercept = update_cr8_intercept,
Gleb Natapov95ba8273132009-04-21 17:45:08 +03004162
Izik Eiduscbc94022007-10-25 00:29:55 +02004163 .set_tss_addr = vmx_set_tss_addr,
Sheng Yang67253af2008-04-25 10:20:22 +08004164 .get_tdp_level = get_ept_level,
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004165 .get_mt_mask = vmx_get_mt_mask,
Marcelo Tosatti229456f2009-06-17 09:22:14 -03004166
4167 .exit_reasons_str = vmx_exit_reasons_str,
Sheng Yang17cc3932010-01-05 19:02:27 +08004168 .get_lpage_level = vmx_get_lpage_level,
Sheng Yang0e851882009-12-18 16:48:46 +08004169
4170 .cpuid_update = vmx_cpuid_update,
Sheng Yang4e47c7a2009-12-18 16:48:47 +08004171
4172 .rdtscp_supported = vmx_rdtscp_supported,
Avi Kivity6aa8b732006-12-10 02:21:36 -08004173};
4174
4175static int __init vmx_init(void)
4176{
Avi Kivity26bb0982009-09-07 11:14:12 +03004177 int r, i;
4178
4179 rdmsrl_safe(MSR_EFER, &host_efer);
4180
4181 for (i = 0; i < NR_VMX_MSR; ++i)
4182 kvm_define_shared_msr(i, vmx_msr_index[i]);
He, Qingfdef3ad2007-04-30 09:45:24 +03004183
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004184 vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
He, Qingfdef3ad2007-04-30 09:45:24 +03004185 if (!vmx_io_bitmap_a)
4186 return -ENOMEM;
4187
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004188 vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
He, Qingfdef3ad2007-04-30 09:45:24 +03004189 if (!vmx_io_bitmap_b) {
4190 r = -ENOMEM;
4191 goto out;
4192 }
4193
Avi Kivity58972972009-02-24 22:26:47 +02004194 vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
4195 if (!vmx_msr_bitmap_legacy) {
Sheng Yang25c5f222008-03-28 13:18:56 +08004196 r = -ENOMEM;
4197 goto out1;
4198 }
4199
Avi Kivity58972972009-02-24 22:26:47 +02004200 vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
4201 if (!vmx_msr_bitmap_longmode) {
4202 r = -ENOMEM;
4203 goto out2;
4204 }
4205
He, Qingfdef3ad2007-04-30 09:45:24 +03004206 /*
4207 * Allow direct access to the PC debug port (it is often used for I/O
4208 * delays, but the vmexits simply slow things down).
4209 */
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004210 memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
4211 clear_bit(0x80, vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004212
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004213 memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
He, Qingfdef3ad2007-04-30 09:45:24 +03004214
Avi Kivity58972972009-02-24 22:26:47 +02004215 memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
4216 memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
Sheng Yang25c5f222008-03-28 13:18:56 +08004217
Sheng Yang2384d2b2008-01-17 15:14:33 +08004218 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
4219
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004220 r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE);
He, Qingfdef3ad2007-04-30 09:45:24 +03004221 if (r)
Avi Kivity58972972009-02-24 22:26:47 +02004222 goto out3;
Sheng Yang25c5f222008-03-28 13:18:56 +08004223
Avi Kivity58972972009-02-24 22:26:47 +02004224 vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
4225 vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
4226 vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
4227 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
4228 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
4229 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
He, Qingfdef3ad2007-04-30 09:45:24 +03004230
Avi Kivity089d0342009-03-23 18:26:32 +02004231 if (enable_ept) {
Sheng Yang14394422008-04-28 12:24:45 +08004232 bypass_guest_pf = 0;
Sheng Yang5fdbcb92008-07-16 09:25:40 +08004233 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
Sheng Yang2aaf69d2009-01-21 16:52:16 +08004234 VMX_EPT_WRITABLE_MASK);
Sheng Yang534e38b2008-09-08 15:12:30 +08004235 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
Sheng Yang4b12f0d2009-04-27 20:35:42 +08004236 VMX_EPT_EXECUTABLE_MASK);
Sheng Yang5fdbcb92008-07-16 09:25:40 +08004237 kvm_enable_tdp();
4238 } else
4239 kvm_disable_tdp();
Sheng Yang14394422008-04-28 12:24:45 +08004240
Avi Kivityc7addb92007-09-16 18:58:32 +02004241 if (bypass_guest_pf)
4242 kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);
4243
He, Qingfdef3ad2007-04-30 09:45:24 +03004244 return 0;
4245
Avi Kivity58972972009-02-24 22:26:47 +02004246out3:
4247 free_page((unsigned long)vmx_msr_bitmap_longmode);
Sheng Yang25c5f222008-03-28 13:18:56 +08004248out2:
Avi Kivity58972972009-02-24 22:26:47 +02004249 free_page((unsigned long)vmx_msr_bitmap_legacy);
He, Qingfdef3ad2007-04-30 09:45:24 +03004250out1:
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004251 free_page((unsigned long)vmx_io_bitmap_b);
He, Qingfdef3ad2007-04-30 09:45:24 +03004252out:
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004253 free_page((unsigned long)vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004254 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004255}
4256
4257static void __exit vmx_exit(void)
4258{
Avi Kivity58972972009-02-24 22:26:47 +02004259 free_page((unsigned long)vmx_msr_bitmap_legacy);
4260 free_page((unsigned long)vmx_msr_bitmap_longmode);
Avi Kivity3e7c73e2009-02-24 21:46:19 +02004261 free_page((unsigned long)vmx_io_bitmap_b);
4262 free_page((unsigned long)vmx_io_bitmap_a);
He, Qingfdef3ad2007-04-30 09:45:24 +03004263
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004264 kvm_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004265}
4266
4267module_init(vmx_init)
4268module_exit(vmx_exit)