blob: e2ee264740c7dbb6ea6717b3aa7617c393fd4831 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * AMD SVM support
5 *
6 * Copyright (C) 2006 Qumranet, Inc.
7 *
8 * Authors:
9 * Yaniv Kamay <yaniv@qumranet.com>
10 * Avi Kivity <avi@qumranet.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2. See
13 * the COPYING file in the top-level directory.
14 *
15 */
Avi Kivityedf88412007-12-16 11:02:48 +020016#include <linux/kvm_host.h>
17
Avi Kivitye4956062007-06-28 14:15:57 -040018#include "kvm_svm.h"
Eddie Dong85f455f2007-07-06 12:20:49 +030019#include "irq.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080020#include "mmu.h"
Avi Kivitye4956062007-06-28 14:15:57 -040021
Avi Kivity6aa8b732006-12-10 02:21:36 -080022#include <linux/module.h>
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +020023#include <linux/kernel.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080024#include <linux/vmalloc.h>
25#include <linux/highmem.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040026#include <linux/sched.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080027
Avi Kivitye4956062007-06-28 14:15:57 -040028#include <asm/desc.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080029
Avi Kivity4ecac3f2008-05-13 13:23:38 +030030#define __ex(x) __kvm_handle_fault_on_reboot(x)
31
Avi Kivity6aa8b732006-12-10 02:21:36 -080032MODULE_AUTHOR("Qumranet");
33MODULE_LICENSE("GPL");
34
35#define IOPM_ALLOC_ORDER 2
36#define MSRPM_ALLOC_ORDER 1
37
38#define DB_VECTOR 1
39#define UD_VECTOR 6
40#define GP_VECTOR 13
41
42#define DR7_GD_MASK (1 << 13)
43#define DR6_BD_MASK (1 << 13)
Avi Kivity6aa8b732006-12-10 02:21:36 -080044
45#define SEG_TYPE_LDT 2
46#define SEG_TYPE_BUSY_TSS16 3
47
Joerg Roedel80b77062007-03-30 17:02:14 +030048#define SVM_FEATURE_NPT (1 << 0)
49#define SVM_FEATURE_LBRV (1 << 1)
50#define SVM_DEATURE_SVML (1 << 2)
51
Joerg Roedel24e09cb2008-02-13 18:58:47 +010052#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
53
Joerg Roedel709ddeb2008-02-07 13:47:45 +010054/* enable NPT for AMD64 and X86 with PAE */
55#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
56static bool npt_enabled = true;
57#else
Joerg Roedele3da3ac2008-02-07 13:47:39 +010058static bool npt_enabled = false;
Joerg Roedel709ddeb2008-02-07 13:47:45 +010059#endif
Joerg Roedel6c7dac72008-02-07 13:47:40 +010060static int npt = 1;
61
62module_param(npt, int, S_IRUGO);
Joerg Roedele3da3ac2008-02-07 13:47:39 +010063
Avi Kivity04d2cc72007-09-10 18:10:54 +030064static void kvm_reput_irq(struct vcpu_svm *svm);
65
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -040066static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
67{
Rusty Russellfb3f0f52007-07-27 17:16:56 +100068 return container_of(vcpu, struct vcpu_svm, vcpu);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -040069}
70
Harvey Harrison4866d5e2008-02-19 10:32:02 -080071static unsigned long iopm_base;
Avi Kivity6aa8b732006-12-10 02:21:36 -080072
73struct kvm_ldttss_desc {
74 u16 limit0;
75 u16 base0;
76 unsigned base1 : 8, type : 5, dpl : 2, p : 1;
77 unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
78 u32 base3;
79 u32 zero1;
80} __attribute__((packed));
81
82struct svm_cpu_data {
83 int cpu;
84
Avi Kivity5008fdf2007-04-02 13:05:50 +030085 u64 asid_generation;
86 u32 max_asid;
87 u32 next_asid;
Avi Kivity6aa8b732006-12-10 02:21:36 -080088 struct kvm_ldttss_desc *tss_desc;
89
90 struct page *save_area;
91};
92
93static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
Joerg Roedel80b77062007-03-30 17:02:14 +030094static uint32_t svm_features;
Avi Kivity6aa8b732006-12-10 02:21:36 -080095
96struct svm_init_data {
97 int cpu;
98 int r;
99};
100
101static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
102
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +0200103#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800104#define MSRS_RANGE_SIZE 2048
105#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
106
107#define MAX_INST_SIZE 15
108
Joerg Roedel80b77062007-03-30 17:02:14 +0300109static inline u32 svm_has(u32 feat)
110{
111 return svm_features & feat;
112}
113
Avi Kivity6aa8b732006-12-10 02:21:36 -0800114static inline u8 pop_irq(struct kvm_vcpu *vcpu)
115{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800116 int word_index = __ffs(vcpu->arch.irq_summary);
117 int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800118 int irq = word_index * BITS_PER_LONG + bit_index;
119
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800120 clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
121 if (!vcpu->arch.irq_pending[word_index])
122 clear_bit(word_index, &vcpu->arch.irq_summary);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800123 return irq;
124}
125
126static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq)
127{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800128 set_bit(irq, vcpu->arch.irq_pending);
129 set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800130}
131
132static inline void clgi(void)
133{
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300134 asm volatile (__ex(SVM_CLGI));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800135}
136
137static inline void stgi(void)
138{
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300139 asm volatile (__ex(SVM_STGI));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800140}
141
142static inline void invlpga(unsigned long addr, u32 asid)
143{
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300144 asm volatile (__ex(SVM_INVLPGA) :: "a"(addr), "c"(asid));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800145}
146
147static inline unsigned long kvm_read_cr2(void)
148{
149 unsigned long cr2;
150
151 asm volatile ("mov %%cr2, %0" : "=r" (cr2));
152 return cr2;
153}
154
155static inline void kvm_write_cr2(unsigned long val)
156{
157 asm volatile ("mov %0, %%cr2" :: "r" (val));
158}
159
160static inline unsigned long read_dr6(void)
161{
162 unsigned long dr6;
163
164 asm volatile ("mov %%dr6, %0" : "=r" (dr6));
165 return dr6;
166}
167
168static inline void write_dr6(unsigned long val)
169{
170 asm volatile ("mov %0, %%dr6" :: "r" (val));
171}
172
173static inline unsigned long read_dr7(void)
174{
175 unsigned long dr7;
176
177 asm volatile ("mov %%dr7, %0" : "=r" (dr7));
178 return dr7;
179}
180
181static inline void write_dr7(unsigned long val)
182{
183 asm volatile ("mov %0, %%dr7" :: "r" (val));
184}
185
Avi Kivity6aa8b732006-12-10 02:21:36 -0800186static inline void force_new_asid(struct kvm_vcpu *vcpu)
187{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400188 to_svm(vcpu)->asid_generation--;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800189}
190
191static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
192{
193 force_new_asid(vcpu);
194}
195
196static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
197{
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100198 if (!npt_enabled && !(efer & EFER_LMA))
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -0600199 efer &= ~EFER_LME;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800200
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400201 to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800202 vcpu->arch.shadow_efer = efer;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800203}
204
Avi Kivity298101d2007-11-25 13:41:11 +0200205static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
206 bool has_error_code, u32 error_code)
207{
208 struct vcpu_svm *svm = to_svm(vcpu);
209
210 svm->vmcb->control.event_inj = nr
211 | SVM_EVTINJ_VALID
212 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
213 | SVM_EVTINJ_TYPE_EXEPT;
214 svm->vmcb->control.event_inj_err = error_code;
215}
216
217static bool svm_exception_injected(struct kvm_vcpu *vcpu)
218{
219 struct vcpu_svm *svm = to_svm(vcpu);
220
221 return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID);
222}
223
Avi Kivity6aa8b732006-12-10 02:21:36 -0800224static int is_external_interrupt(u32 info)
225{
226 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
227 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
228}
229
230static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
231{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400232 struct vcpu_svm *svm = to_svm(vcpu);
233
234 if (!svm->next_rip) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800235 printk(KERN_DEBUG "%s: NOP\n", __func__);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800236 return;
237 }
Mike Dayd77c26f2007-10-08 09:02:08 -0400238 if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800239 printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800240 __func__,
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400241 svm->vmcb->save.rip,
242 svm->next_rip);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800243
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800244 vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400245 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
Dor Laorc1150d82007-01-05 16:36:24 -0800246
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800247 vcpu->arch.interrupt_window_open = 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800248}
249
250static int has_svm(void)
251{
252 uint32_t eax, ebx, ecx, edx;
253
Avi Kivity1e885462006-12-29 16:49:34 -0800254 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800255 printk(KERN_INFO "has_svm: not amd\n");
256 return 0;
257 }
258
259 cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
260 if (eax < SVM_CPUID_FUNC) {
261 printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n");
262 return 0;
263 }
264
265 cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
266 if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) {
267 printk(KERN_DEBUG "has_svm: svm not available\n");
268 return 0;
269 }
270 return 1;
271}
272
273static void svm_hardware_disable(void *garbage)
274{
Joerg Roedel0da1db752008-07-02 16:02:11 +0200275 uint64_t efer;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800276
Joerg Roedel0da1db752008-07-02 16:02:11 +0200277 wrmsrl(MSR_VM_HSAVE_PA, 0);
278 rdmsrl(MSR_EFER, efer);
279 wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800280}
281
282static void svm_hardware_enable(void *garbage)
283{
284
285 struct svm_cpu_data *svm_data;
286 uint64_t efer;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800287 struct desc_ptr gdt_descr;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800288 struct desc_struct *gdt;
289 int me = raw_smp_processor_id();
290
291 if (!has_svm()) {
292 printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
293 return;
294 }
295 svm_data = per_cpu(svm_data, me);
296
297 if (!svm_data) {
298 printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
299 me);
300 return;
301 }
302
303 svm_data->asid_generation = 1;
304 svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
305 svm_data->next_asid = svm_data->max_asid + 1;
306
Mike Dayd77c26f2007-10-08 09:02:08 -0400307 asm volatile ("sgdt %0" : "=m"(gdt_descr));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800308 gdt = (struct desc_struct *)gdt_descr.address;
309 svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
310
311 rdmsrl(MSR_EFER, efer);
312 wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK);
313
314 wrmsrl(MSR_VM_HSAVE_PA,
315 page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
316}
317
Joerg Roedel0da1db752008-07-02 16:02:11 +0200318static void svm_cpu_uninit(int cpu)
319{
320 struct svm_cpu_data *svm_data
321 = per_cpu(svm_data, raw_smp_processor_id());
322
323 if (!svm_data)
324 return;
325
326 per_cpu(svm_data, raw_smp_processor_id()) = NULL;
327 __free_page(svm_data->save_area);
328 kfree(svm_data);
329}
330
Avi Kivity6aa8b732006-12-10 02:21:36 -0800331static int svm_cpu_init(int cpu)
332{
333 struct svm_cpu_data *svm_data;
334 int r;
335
336 svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
337 if (!svm_data)
338 return -ENOMEM;
339 svm_data->cpu = cpu;
340 svm_data->save_area = alloc_page(GFP_KERNEL);
341 r = -ENOMEM;
342 if (!svm_data->save_area)
343 goto err_1;
344
345 per_cpu(svm_data, cpu) = svm_data;
346
347 return 0;
348
349err_1:
350 kfree(svm_data);
351 return r;
352
353}
354
Rusty Russellbfc733a2007-07-31 20:42:42 +1000355static void set_msr_interception(u32 *msrpm, unsigned msr,
356 int read, int write)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800357{
358 int i;
359
360 for (i = 0; i < NUM_MSR_MAPS; i++) {
361 if (msr >= msrpm_ranges[i] &&
362 msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
363 u32 msr_offset = (i * MSRS_IN_RANGE + msr -
364 msrpm_ranges[i]) * 2;
365
366 u32 *base = msrpm + (msr_offset / 32);
367 u32 msr_shift = msr_offset % 32;
368 u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
369 *base = (*base & ~(0x3 << msr_shift)) |
370 (mask << msr_shift);
Rusty Russellbfc733a2007-07-31 20:42:42 +1000371 return;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800372 }
373 }
Rusty Russellbfc733a2007-07-31 20:42:42 +1000374 BUG();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800375}
376
Joerg Roedelf65c2292008-02-13 18:58:46 +0100377static void svm_vcpu_init_msrpm(u32 *msrpm)
378{
379 memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
380
381#ifdef CONFIG_X86_64
382 set_msr_interception(msrpm, MSR_GS_BASE, 1, 1);
383 set_msr_interception(msrpm, MSR_FS_BASE, 1, 1);
384 set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1);
385 set_msr_interception(msrpm, MSR_LSTAR, 1, 1);
386 set_msr_interception(msrpm, MSR_CSTAR, 1, 1);
387 set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1);
388#endif
389 set_msr_interception(msrpm, MSR_K6_STAR, 1, 1);
390 set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1);
391 set_msr_interception(msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
392 set_msr_interception(msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
393}
394
Joerg Roedel24e09cb2008-02-13 18:58:47 +0100395static void svm_enable_lbrv(struct vcpu_svm *svm)
396{
397 u32 *msrpm = svm->msrpm;
398
399 svm->vmcb->control.lbr_ctl = 1;
400 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
401 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
402 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
403 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
404}
405
406static void svm_disable_lbrv(struct vcpu_svm *svm)
407{
408 u32 *msrpm = svm->msrpm;
409
410 svm->vmcb->control.lbr_ctl = 0;
411 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
412 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
413 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
414 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
415}
416
Avi Kivity6aa8b732006-12-10 02:21:36 -0800417static __init int svm_hardware_setup(void)
418{
419 int cpu;
420 struct page *iopm_pages;
Joerg Roedelf65c2292008-02-13 18:58:46 +0100421 void *iopm_va;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800422 int r;
423
Avi Kivity6aa8b732006-12-10 02:21:36 -0800424 iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
425
426 if (!iopm_pages)
427 return -ENOMEM;
Anthony Liguoric8681332007-04-30 09:48:11 +0300428
429 iopm_va = page_address(iopm_pages);
430 memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
431 clear_bit(0x80, iopm_va); /* allow direct access to PC debug port */
Avi Kivity6aa8b732006-12-10 02:21:36 -0800432 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
433
Joerg Roedel50a37eb2008-01-31 14:57:38 +0100434 if (boot_cpu_has(X86_FEATURE_NX))
435 kvm_enable_efer_bits(EFER_NX);
436
Avi Kivity6aa8b732006-12-10 02:21:36 -0800437 for_each_online_cpu(cpu) {
438 r = svm_cpu_init(cpu);
439 if (r)
Joerg Roedelf65c2292008-02-13 18:58:46 +0100440 goto err;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800441 }
Joerg Roedel33bd6a02008-02-07 13:47:38 +0100442
443 svm_features = cpuid_edx(SVM_CPUID_FUNC);
444
Joerg Roedele3da3ac2008-02-07 13:47:39 +0100445 if (!svm_has(SVM_FEATURE_NPT))
446 npt_enabled = false;
447
Joerg Roedel6c7dac72008-02-07 13:47:40 +0100448 if (npt_enabled && !npt) {
449 printk(KERN_INFO "kvm: Nested Paging disabled\n");
450 npt_enabled = false;
451 }
452
Joerg Roedel18552672008-02-07 13:47:41 +0100453 if (npt_enabled) {
Joerg Roedele3da3ac2008-02-07 13:47:39 +0100454 printk(KERN_INFO "kvm: Nested Paging enabled\n");
Joerg Roedel18552672008-02-07 13:47:41 +0100455 kvm_enable_tdp();
Joerg Roedel5f4cb662008-07-14 20:36:36 +0200456 } else
457 kvm_disable_tdp();
Joerg Roedele3da3ac2008-02-07 13:47:39 +0100458
Avi Kivity6aa8b732006-12-10 02:21:36 -0800459 return 0;
460
Joerg Roedelf65c2292008-02-13 18:58:46 +0100461err:
Avi Kivity6aa8b732006-12-10 02:21:36 -0800462 __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
463 iopm_base = 0;
464 return r;
465}
466
467static __exit void svm_hardware_unsetup(void)
468{
Joerg Roedel0da1db752008-07-02 16:02:11 +0200469 int cpu;
470
471 for_each_online_cpu(cpu)
472 svm_cpu_uninit(cpu);
473
Avi Kivity6aa8b732006-12-10 02:21:36 -0800474 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
Joerg Roedelf65c2292008-02-13 18:58:46 +0100475 iopm_base = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800476}
477
478static void init_seg(struct vmcb_seg *seg)
479{
480 seg->selector = 0;
481 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
482 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
483 seg->limit = 0xffff;
484 seg->base = 0;
485}
486
487static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
488{
489 seg->selector = 0;
490 seg->attrib = SVM_SELECTOR_P_MASK | type;
491 seg->limit = 0xffff;
492 seg->base = 0;
493}
494
Joerg Roedele6101a92008-02-13 18:58:45 +0100495static void init_vmcb(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800496{
Joerg Roedele6101a92008-02-13 18:58:45 +0100497 struct vmcb_control_area *control = &svm->vmcb->control;
498 struct vmcb_save_area *save = &svm->vmcb->save;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800499
500 control->intercept_cr_read = INTERCEPT_CR0_MASK |
501 INTERCEPT_CR3_MASK |
Joerg Roedel649d6862008-04-16 16:51:15 +0200502 INTERCEPT_CR4_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800503
504 control->intercept_cr_write = INTERCEPT_CR0_MASK |
505 INTERCEPT_CR3_MASK |
Avi Kivity80a81192007-12-06 19:50:00 +0200506 INTERCEPT_CR4_MASK |
507 INTERCEPT_CR8_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800508
509 control->intercept_dr_read = INTERCEPT_DR0_MASK |
510 INTERCEPT_DR1_MASK |
511 INTERCEPT_DR2_MASK |
512 INTERCEPT_DR3_MASK;
513
514 control->intercept_dr_write = INTERCEPT_DR0_MASK |
515 INTERCEPT_DR1_MASK |
516 INTERCEPT_DR2_MASK |
517 INTERCEPT_DR3_MASK |
518 INTERCEPT_DR5_MASK |
519 INTERCEPT_DR7_MASK;
520
Anthony Liguori7aa81cc2007-09-17 14:57:50 -0500521 control->intercept_exceptions = (1 << PF_VECTOR) |
Joerg Roedel53371b52008-04-09 14:15:30 +0200522 (1 << UD_VECTOR) |
523 (1 << MC_VECTOR);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800524
525
526 control->intercept = (1ULL << INTERCEPT_INTR) |
527 (1ULL << INTERCEPT_NMI) |
Joerg Roedel01525272007-02-19 14:37:47 +0200528 (1ULL << INTERCEPT_SMI) |
Avi Kivity6aa8b732006-12-10 02:21:36 -0800529 (1ULL << INTERCEPT_CPUID) |
Avi Kivitycf5a94d2007-10-28 16:11:58 +0200530 (1ULL << INTERCEPT_INVD) |
Avi Kivity6aa8b732006-12-10 02:21:36 -0800531 (1ULL << INTERCEPT_HLT) |
Avi Kivity6aa8b732006-12-10 02:21:36 -0800532 (1ULL << INTERCEPT_INVLPGA) |
533 (1ULL << INTERCEPT_IOIO_PROT) |
534 (1ULL << INTERCEPT_MSR_PROT) |
535 (1ULL << INTERCEPT_TASK_SWITCH) |
Joerg Roedel46fe4dd2007-01-26 00:56:42 -0800536 (1ULL << INTERCEPT_SHUTDOWN) |
Avi Kivity6aa8b732006-12-10 02:21:36 -0800537 (1ULL << INTERCEPT_VMRUN) |
538 (1ULL << INTERCEPT_VMMCALL) |
539 (1ULL << INTERCEPT_VMLOAD) |
540 (1ULL << INTERCEPT_VMSAVE) |
541 (1ULL << INTERCEPT_STGI) |
542 (1ULL << INTERCEPT_CLGI) |
Joerg Roedel916ce232007-03-21 19:47:00 +0100543 (1ULL << INTERCEPT_SKINIT) |
Avi Kivitycf5a94d2007-10-28 16:11:58 +0200544 (1ULL << INTERCEPT_WBINVD) |
Joerg Roedel916ce232007-03-21 19:47:00 +0100545 (1ULL << INTERCEPT_MONITOR) |
546 (1ULL << INTERCEPT_MWAIT);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800547
548 control->iopm_base_pa = iopm_base;
Joerg Roedelf65c2292008-02-13 18:58:46 +0100549 control->msrpm_base_pa = __pa(svm->msrpm);
Avi Kivity0cc50642007-03-25 12:07:27 +0200550 control->tsc_offset = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800551 control->int_ctl = V_INTR_MASKING_MASK;
552
553 init_seg(&save->es);
554 init_seg(&save->ss);
555 init_seg(&save->ds);
556 init_seg(&save->fs);
557 init_seg(&save->gs);
558
559 save->cs.selector = 0xf000;
560 /* Executable/Readable Code Segment */
561 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
562 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
563 save->cs.limit = 0xffff;
Avi Kivityd92899a2007-02-12 00:54:38 -0800564 /*
565 * cs.base should really be 0xffff0000, but vmx can't handle that, so
566 * be consistent with it.
567 *
568 * Replace when we have real mode working for vmx.
569 */
570 save->cs.base = 0xf0000;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800571
572 save->gdtr.limit = 0xffff;
573 save->idtr.limit = 0xffff;
574
575 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
576 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
577
578 save->efer = MSR_EFER_SVME_MASK;
Mike Dayd77c26f2007-10-08 09:02:08 -0400579 save->dr6 = 0xffff0ff0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800580 save->dr7 = 0x400;
581 save->rflags = 2;
582 save->rip = 0x0000fff0;
583
584 /*
585 * cr0 val on cpu init should be 0x60000010, we enable cpu
586 * cache by default. the orderly way is to enable cache in bios.
587 */
Rusty Russell707d92fa2007-07-17 23:19:08 +1000588 save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP;
Rusty Russell66aee912007-07-17 23:34:16 +1000589 save->cr4 = X86_CR4_PAE;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800590 /* rdx = ?? */
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100591
592 if (npt_enabled) {
593 /* Setup VMCB for Nested Paging */
594 control->nested_ctl = 1;
Joerg Roedel35649902008-04-09 16:04:32 +0200595 control->intercept &= ~(1ULL << INTERCEPT_TASK_SWITCH);
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100596 control->intercept_exceptions &= ~(1 << PF_VECTOR);
597 control->intercept_cr_read &= ~(INTERCEPT_CR0_MASK|
598 INTERCEPT_CR3_MASK);
599 control->intercept_cr_write &= ~(INTERCEPT_CR0_MASK|
600 INTERCEPT_CR3_MASK);
601 save->g_pat = 0x0007040600070406ULL;
602 /* enable caching because the QEMU Bios doesn't enable it */
603 save->cr0 = X86_CR0_ET;
604 save->cr3 = 0;
605 save->cr4 = 0;
606 }
Avi Kivitya79d2f12008-04-14 13:10:21 +0300607 force_new_asid(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800608}
609
Avi Kivitye00c8cf2007-10-21 11:00:39 +0200610static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
Avi Kivity04d2cc72007-09-10 18:10:54 +0300611{
612 struct vcpu_svm *svm = to_svm(vcpu);
613
Joerg Roedele6101a92008-02-13 18:58:45 +0100614 init_vmcb(svm);
Avi Kivity70433382007-11-07 12:57:23 +0200615
616 if (vcpu->vcpu_id != 0) {
617 svm->vmcb->save.rip = 0;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800618 svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
619 svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
Avi Kivity70433382007-11-07 12:57:23 +0200620 }
Avi Kivitye00c8cf2007-10-21 11:00:39 +0200621
622 return 0;
Avi Kivity04d2cc72007-09-10 18:10:54 +0300623}
624
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000625static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800626{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400627 struct vcpu_svm *svm;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800628 struct page *page;
Joerg Roedelf65c2292008-02-13 18:58:46 +0100629 struct page *msrpm_pages;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000630 int err;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800631
Rusty Russellc16f8622007-07-30 21:12:19 +1000632 svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000633 if (!svm) {
634 err = -ENOMEM;
635 goto out;
636 }
637
638 err = kvm_vcpu_init(&svm->vcpu, kvm, id);
639 if (err)
640 goto free_svm;
641
Avi Kivity6aa8b732006-12-10 02:21:36 -0800642 page = alloc_page(GFP_KERNEL);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000643 if (!page) {
644 err = -ENOMEM;
645 goto uninit;
646 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800647
Joerg Roedelf65c2292008-02-13 18:58:46 +0100648 err = -ENOMEM;
649 msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
650 if (!msrpm_pages)
651 goto uninit;
652 svm->msrpm = page_address(msrpm_pages);
653 svm_vcpu_init_msrpm(svm->msrpm);
654
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400655 svm->vmcb = page_address(page);
656 clear_page(svm->vmcb);
657 svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
658 svm->asid_generation = 0;
659 memset(svm->db_regs, 0, sizeof(svm->db_regs));
Joerg Roedele6101a92008-02-13 18:58:45 +0100660 init_vmcb(svm);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400661
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000662 fx_init(&svm->vcpu);
663 svm->vcpu.fpu_active = 1;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800664 svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000665 if (svm->vcpu.vcpu_id == 0)
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800666 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800667
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000668 return &svm->vcpu;
Avi Kivity36241b82006-12-22 01:05:20 -0800669
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000670uninit:
671 kvm_vcpu_uninit(&svm->vcpu);
672free_svm:
Rusty Russella4770342007-08-01 14:46:11 +1000673 kmem_cache_free(kvm_vcpu_cache, svm);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000674out:
675 return ERR_PTR(err);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800676}
677
678static void svm_free_vcpu(struct kvm_vcpu *vcpu)
679{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400680 struct vcpu_svm *svm = to_svm(vcpu);
681
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000682 __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
Joerg Roedelf65c2292008-02-13 18:58:46 +0100683 __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000684 kvm_vcpu_uninit(vcpu);
Rusty Russella4770342007-08-01 14:46:11 +1000685 kmem_cache_free(kvm_vcpu_cache, svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800686}
687
Avi Kivity15ad7142007-07-11 18:17:21 +0300688static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800689{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400690 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300691 int i;
Avi Kivity0cc50642007-03-25 12:07:27 +0200692
Avi Kivity0cc50642007-03-25 12:07:27 +0200693 if (unlikely(cpu != vcpu->cpu)) {
694 u64 tsc_this, delta;
695
696 /*
697 * Make sure that the guest sees a monotonically
698 * increasing TSC.
699 */
700 rdtscll(tsc_this);
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800701 delta = vcpu->arch.host_tsc - tsc_this;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400702 svm->vmcb->control.tsc_offset += delta;
Avi Kivity0cc50642007-03-25 12:07:27 +0200703 vcpu->cpu = cpu;
Marcelo Tosatti2f599712008-05-27 12:10:20 -0300704 kvm_migrate_timers(vcpu);
Avi Kivity0cc50642007-03-25 12:07:27 +0200705 }
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300706
707 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400708 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800709}
710
711static void svm_vcpu_put(struct kvm_vcpu *vcpu)
712{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400713 struct vcpu_svm *svm = to_svm(vcpu);
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300714 int i;
715
Avi Kivitye1beb1d2007-11-18 13:50:24 +0200716 ++vcpu->stat.host_state_reload;
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300717 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400718 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300719
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800720 rdtscll(vcpu->arch.host_tsc);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800721}
722
723static void svm_cache_regs(struct kvm_vcpu *vcpu)
724{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400725 struct vcpu_svm *svm = to_svm(vcpu);
726
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800727 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
728 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
729 vcpu->arch.rip = svm->vmcb->save.rip;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800730}
731
732static void svm_decache_regs(struct kvm_vcpu *vcpu)
733{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400734 struct vcpu_svm *svm = to_svm(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800735 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
736 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
737 svm->vmcb->save.rip = vcpu->arch.rip;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800738}
739
740static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
741{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400742 return to_svm(vcpu)->vmcb->save.rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800743}
744
745static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
746{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400747 to_svm(vcpu)->vmcb->save.rflags = rflags;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800748}
749
750static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
751{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400752 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800753
754 switch (seg) {
755 case VCPU_SREG_CS: return &save->cs;
756 case VCPU_SREG_DS: return &save->ds;
757 case VCPU_SREG_ES: return &save->es;
758 case VCPU_SREG_FS: return &save->fs;
759 case VCPU_SREG_GS: return &save->gs;
760 case VCPU_SREG_SS: return &save->ss;
761 case VCPU_SREG_TR: return &save->tr;
762 case VCPU_SREG_LDTR: return &save->ldtr;
763 }
764 BUG();
Al Viro8b6d44c2007-02-09 16:38:40 +0000765 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800766}
767
768static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
769{
770 struct vmcb_seg *s = svm_seg(vcpu, seg);
771
772 return s->base;
773}
774
775static void svm_get_segment(struct kvm_vcpu *vcpu,
776 struct kvm_segment *var, int seg)
777{
778 struct vmcb_seg *s = svm_seg(vcpu, seg);
779
780 var->base = s->base;
781 var->limit = s->limit;
782 var->selector = s->selector;
783 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
784 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
785 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
786 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
787 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
788 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
789 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
790 var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
791 var->unusable = !var->present;
792}
793
Izik Eidus2e4d2652008-03-24 19:38:34 +0200794static int svm_get_cpl(struct kvm_vcpu *vcpu)
795{
796 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
797
798 return save->cpl;
799}
800
Avi Kivity6aa8b732006-12-10 02:21:36 -0800801static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
802{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400803 struct vcpu_svm *svm = to_svm(vcpu);
804
805 dt->limit = svm->vmcb->save.idtr.limit;
806 dt->base = svm->vmcb->save.idtr.base;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800807}
808
809static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
810{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400811 struct vcpu_svm *svm = to_svm(vcpu);
812
813 svm->vmcb->save.idtr.limit = dt->limit;
814 svm->vmcb->save.idtr.base = dt->base ;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800815}
816
817static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
818{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400819 struct vcpu_svm *svm = to_svm(vcpu);
820
821 dt->limit = svm->vmcb->save.gdtr.limit;
822 dt->base = svm->vmcb->save.gdtr.base;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800823}
824
825static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
826{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400827 struct vcpu_svm *svm = to_svm(vcpu);
828
829 svm->vmcb->save.gdtr.limit = dt->limit;
830 svm->vmcb->save.gdtr.base = dt->base ;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800831}
832
Anthony Liguori25c4c272007-04-27 09:29:21 +0300833static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
Avi Kivity399badf2007-01-05 16:36:38 -0800834{
835}
836
Avi Kivity6aa8b732006-12-10 02:21:36 -0800837static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
838{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400839 struct vcpu_svm *svm = to_svm(vcpu);
840
Avi Kivity05b3e0c2006-12-13 00:33:45 -0800841#ifdef CONFIG_X86_64
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800842 if (vcpu->arch.shadow_efer & EFER_LME) {
Rusty Russell707d92fa2007-07-17 23:19:08 +1000843 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800844 vcpu->arch.shadow_efer |= EFER_LMA;
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -0600845 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800846 }
847
Mike Dayd77c26f2007-10-08 09:02:08 -0400848 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800849 vcpu->arch.shadow_efer &= ~EFER_LMA;
Carlo Marcelo Arenas Belon2b5203e2007-12-01 06:17:11 -0600850 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800851 }
852 }
853#endif
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100854 if (npt_enabled)
855 goto set;
856
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800857 if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) {
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400858 svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
Anthony Liguori7807fa62007-04-23 09:17:21 -0500859 vcpu->fpu_active = 1;
860 }
861
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800862 vcpu->arch.cr0 = cr0;
Rusty Russell707d92fa2007-07-17 23:19:08 +1000863 cr0 |= X86_CR0_PG | X86_CR0_WP;
Joerg Roedel6b390b62008-01-29 13:01:27 +0100864 if (!vcpu->fpu_active) {
865 svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
Joerg Roedel334df502008-01-21 13:09:33 +0100866 cr0 |= X86_CR0_TS;
Joerg Roedel6b390b62008-01-29 13:01:27 +0100867 }
Joerg Roedel709ddeb2008-02-07 13:47:45 +0100868set:
869 /*
870 * re-enable caching here because the QEMU bios
871 * does not do it - this results in some delay at
872 * reboot
873 */
874 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400875 svm->vmcb->save.cr0 = cr0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800876}
877
878static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
879{
Joerg Roedel6394b642008-04-09 14:15:29 +0200880 unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
881
Joerg Roedelec077262008-04-09 14:15:28 +0200882 vcpu->arch.cr4 = cr4;
883 if (!npt_enabled)
884 cr4 |= X86_CR4_PAE;
Joerg Roedel6394b642008-04-09 14:15:29 +0200885 cr4 |= host_cr4_mce;
Joerg Roedelec077262008-04-09 14:15:28 +0200886 to_svm(vcpu)->vmcb->save.cr4 = cr4;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800887}
888
889static void svm_set_segment(struct kvm_vcpu *vcpu,
890 struct kvm_segment *var, int seg)
891{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400892 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800893 struct vmcb_seg *s = svm_seg(vcpu, seg);
894
895 s->base = var->base;
896 s->limit = var->limit;
897 s->selector = var->selector;
898 if (var->unusable)
899 s->attrib = 0;
900 else {
901 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
902 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
903 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
904 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
905 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
906 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
907 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
908 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
909 }
910 if (seg == VCPU_SREG_CS)
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400911 svm->vmcb->save.cpl
912 = (svm->vmcb->save.cs.attrib
Avi Kivity6aa8b732006-12-10 02:21:36 -0800913 >> SVM_SELECTOR_DPL_SHIFT) & 3;
914
915}
916
Avi Kivity6aa8b732006-12-10 02:21:36 -0800917static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
918{
919 return -EOPNOTSUPP;
920}
921
Eddie Dong2a8067f2007-08-06 16:29:07 +0300922static int svm_get_irq(struct kvm_vcpu *vcpu)
923{
924 struct vcpu_svm *svm = to_svm(vcpu);
925 u32 exit_int_info = svm->vmcb->control.exit_int_info;
926
927 if (is_external_interrupt(exit_int_info))
928 return exit_int_info & SVM_EVTINJ_VEC_MASK;
929 return -1;
930}
931
Avi Kivity6aa8b732006-12-10 02:21:36 -0800932static void load_host_msrs(struct kvm_vcpu *vcpu)
933{
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300934#ifdef CONFIG_X86_64
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400935 wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300936#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800937}
938
939static void save_host_msrs(struct kvm_vcpu *vcpu)
940{
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300941#ifdef CONFIG_X86_64
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400942 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base);
Anthony Liguori94dfbdb2007-04-29 11:56:06 +0300943#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800944}
945
Rusty Russelle756fc62007-07-30 20:07:08 +1000946static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800947{
948 if (svm_data->next_asid > svm_data->max_asid) {
949 ++svm_data->asid_generation;
950 svm_data->next_asid = 1;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400951 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800952 }
953
Rusty Russelle756fc62007-07-30 20:07:08 +1000954 svm->vcpu.cpu = svm_data->cpu;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400955 svm->asid_generation = svm_data->asid_generation;
956 svm->vmcb->control.asid = svm_data->next_asid++;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800957}
958
Avi Kivity6aa8b732006-12-10 02:21:36 -0800959static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
960{
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +0200961 unsigned long val = to_svm(vcpu)->db_regs[dr];
962 KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);
963 return val;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800964}
965
966static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
967 int *exception)
968{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400969 struct vcpu_svm *svm = to_svm(vcpu);
970
Avi Kivity6aa8b732006-12-10 02:21:36 -0800971 *exception = 0;
972
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400973 if (svm->vmcb->save.dr7 & DR7_GD_MASK) {
974 svm->vmcb->save.dr7 &= ~DR7_GD_MASK;
975 svm->vmcb->save.dr6 |= DR6_BD_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800976 *exception = DB_VECTOR;
977 return;
978 }
979
980 switch (dr) {
981 case 0 ... 3:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400982 svm->db_regs[dr] = value;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800983 return;
984 case 4 ... 5:
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800985 if (vcpu->arch.cr4 & X86_CR4_DE) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800986 *exception = UD_VECTOR;
987 return;
988 }
989 case 7: {
990 if (value & ~((1ULL << 32) - 1)) {
991 *exception = GP_VECTOR;
992 return;
993 }
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -0400994 svm->vmcb->save.dr7 = value;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800995 return;
996 }
997 default:
998 printk(KERN_DEBUG "%s: unexpected dr %u\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800999 __func__, dr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001000 *exception = UD_VECTOR;
1001 return;
1002 }
1003}
1004
Rusty Russelle756fc62007-07-30 20:07:08 +10001005static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001006{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001007 u32 exit_int_info = svm->vmcb->control.exit_int_info;
Rusty Russelle756fc62007-07-30 20:07:08 +10001008 struct kvm *kvm = svm->vcpu.kvm;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001009 u64 fault_address;
1010 u32 error_code;
Avi Kivity577bdc42008-07-19 08:57:05 +03001011 bool event_injection = false;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001012
Eddie Dong85f455f2007-07-06 12:20:49 +03001013 if (!irqchip_in_kernel(kvm) &&
Avi Kivity577bdc42008-07-19 08:57:05 +03001014 is_external_interrupt(exit_int_info)) {
1015 event_injection = true;
Rusty Russelle756fc62007-07-30 20:07:08 +10001016 push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK);
Avi Kivity577bdc42008-07-19 08:57:05 +03001017 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001018
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001019 fault_address = svm->vmcb->control.exit_info_2;
1020 error_code = svm->vmcb->control.exit_info_1;
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001021
1022 if (!npt_enabled)
1023 KVMTRACE_3D(PAGE_FAULT, &svm->vcpu, error_code,
1024 (u32)fault_address, (u32)(fault_address >> 32),
1025 handler);
Joerg Roedeld2ebb412008-04-30 17:56:04 +02001026 else
1027 KVMTRACE_3D(TDP_FAULT, &svm->vcpu, error_code,
1028 (u32)fault_address, (u32)(fault_address >> 32),
1029 handler);
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001030
Avi Kivity577bdc42008-07-19 08:57:05 +03001031 if (event_injection)
1032 kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
Avi Kivity30677142007-10-28 18:48:59 +02001033 return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001034}
1035
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05001036static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1037{
1038 int er;
1039
Sheng Yang571008d2008-01-02 14:49:22 +08001040 er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05001041 if (er != EMULATE_DONE)
Avi Kivity7ee5d9402007-11-25 15:22:50 +02001042 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05001043 return 1;
1044}
1045
Rusty Russelle756fc62007-07-30 20:07:08 +10001046static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Anthony Liguori7807fa62007-04-23 09:17:21 -05001047{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001048 svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001049 if (!(svm->vcpu.arch.cr0 & X86_CR0_TS))
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001050 svm->vmcb->save.cr0 &= ~X86_CR0_TS;
Rusty Russelle756fc62007-07-30 20:07:08 +10001051 svm->vcpu.fpu_active = 1;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001052
1053 return 1;
Anthony Liguori7807fa62007-04-23 09:17:21 -05001054}
1055
Joerg Roedel53371b52008-04-09 14:15:30 +02001056static int mc_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1057{
1058 /*
1059 * On an #MC intercept the MCE handler is not called automatically in
1060 * the host. So do it by hand here.
1061 */
1062 asm volatile (
1063 "int $0x12\n");
1064 /* not sure if we ever come back to this point */
1065
1066 return 1;
1067}
1068
Rusty Russelle756fc62007-07-30 20:07:08 +10001069static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08001070{
1071 /*
1072 * VMCB is undefined after a SHUTDOWN intercept
1073 * so reinitialize it.
1074 */
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001075 clear_page(svm->vmcb);
Joerg Roedele6101a92008-02-13 18:58:45 +01001076 init_vmcb(svm);
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08001077
1078 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1079 return 0;
1080}
1081
Rusty Russelle756fc62007-07-30 20:07:08 +10001082static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001083{
Mike Dayd77c26f2007-10-08 09:02:08 -04001084 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
Avi Kivity039576c2007-03-20 12:46:50 +02001085 int size, down, in, string, rep;
1086 unsigned port;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001087
Rusty Russelle756fc62007-07-30 20:07:08 +10001088 ++svm->vcpu.stat.io_exits;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001089
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001090 svm->next_rip = svm->vmcb->control.exit_info_2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001091
Laurent Viviere70669a2007-08-05 10:36:40 +03001092 string = (io_info & SVM_IOIO_STR_MASK) != 0;
1093
1094 if (string) {
Laurent Vivier34273182007-09-18 11:27:37 +02001095 if (emulate_instruction(&svm->vcpu,
1096 kvm_run, 0, 0, 0) == EMULATE_DO_MMIO)
Laurent Viviere70669a2007-08-05 10:36:40 +03001097 return 0;
1098 return 1;
1099 }
1100
Avi Kivity039576c2007-03-20 12:46:50 +02001101 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1102 port = io_info >> 16;
1103 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
Avi Kivity039576c2007-03-20 12:46:50 +02001104 rep = (io_info & SVM_IOIO_REP_MASK) != 0;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001105 down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001106
Laurent Vivier3090dd72007-08-05 10:43:32 +03001107 return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001108}
1109
Joerg Roedelc47f0982008-04-30 17:56:00 +02001110static int nmi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1111{
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001112 KVMTRACE_0D(NMI, &svm->vcpu, handler);
Joerg Roedelc47f0982008-04-30 17:56:00 +02001113 return 1;
1114}
1115
Joerg Roedela0698052008-04-30 17:56:01 +02001116static int intr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1117{
1118 ++svm->vcpu.stat.irq_exits;
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001119 KVMTRACE_0D(INTR, &svm->vcpu, handler);
Joerg Roedela0698052008-04-30 17:56:01 +02001120 return 1;
1121}
1122
Rusty Russelle756fc62007-07-30 20:07:08 +10001123static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001124{
1125 return 1;
1126}
1127
Rusty Russelle756fc62007-07-30 20:07:08 +10001128static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001129{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001130 svm->next_rip = svm->vmcb->save.rip + 1;
Rusty Russelle756fc62007-07-30 20:07:08 +10001131 skip_emulated_instruction(&svm->vcpu);
1132 return kvm_emulate_halt(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001133}
1134
Rusty Russelle756fc62007-07-30 20:07:08 +10001135static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity02e235b2007-02-19 14:37:47 +02001136{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001137 svm->next_rip = svm->vmcb->save.rip + 3;
Rusty Russelle756fc62007-07-30 20:07:08 +10001138 skip_emulated_instruction(&svm->vcpu);
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05001139 kvm_emulate_hypercall(&svm->vcpu);
1140 return 1;
Avi Kivity02e235b2007-02-19 14:37:47 +02001141}
1142
Rusty Russelle756fc62007-07-30 20:07:08 +10001143static int invalid_op_interception(struct vcpu_svm *svm,
1144 struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001145{
Avi Kivity7ee5d9402007-11-25 15:22:50 +02001146 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001147 return 1;
1148}
1149
Rusty Russelle756fc62007-07-30 20:07:08 +10001150static int task_switch_interception(struct vcpu_svm *svm,
1151 struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001152{
Izik Eidus37817f22008-03-24 23:14:53 +02001153 u16 tss_selector;
1154
1155 tss_selector = (u16)svm->vmcb->control.exit_info_1;
1156 if (svm->vmcb->control.exit_info_2 &
1157 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
1158 return kvm_task_switch(&svm->vcpu, tss_selector,
1159 TASK_SWITCH_IRET);
1160 if (svm->vmcb->control.exit_info_2 &
1161 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
1162 return kvm_task_switch(&svm->vcpu, tss_selector,
1163 TASK_SWITCH_JMP);
1164 return kvm_task_switch(&svm->vcpu, tss_selector, TASK_SWITCH_CALL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001165}
1166
Rusty Russelle756fc62007-07-30 20:07:08 +10001167static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001168{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001169 svm->next_rip = svm->vmcb->save.rip + 2;
Rusty Russelle756fc62007-07-30 20:07:08 +10001170 kvm_emulate_cpuid(&svm->vcpu);
Avi Kivity06465c52007-02-28 20:46:53 +02001171 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001172}
1173
Rusty Russelle756fc62007-07-30 20:07:08 +10001174static int emulate_on_interception(struct vcpu_svm *svm,
1175 struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001176{
Laurent Vivier34273182007-09-18 11:27:37 +02001177 if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE)
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001178 pr_unimpl(&svm->vcpu, "%s: failed\n", __func__);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001179 return 1;
1180}
1181
Joerg Roedel1d075432007-12-06 21:02:25 +01001182static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
1183{
1184 emulate_instruction(&svm->vcpu, NULL, 0, 0, 0);
1185 if (irqchip_in_kernel(svm->vcpu.kvm))
1186 return 1;
1187 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
1188 return 0;
1189}
1190
Avi Kivity6aa8b732006-12-10 02:21:36 -08001191static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
1192{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001193 struct vcpu_svm *svm = to_svm(vcpu);
1194
Avi Kivity6aa8b732006-12-10 02:21:36 -08001195 switch (ecx) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001196 case MSR_IA32_TIME_STAMP_COUNTER: {
1197 u64 tsc;
1198
1199 rdtscll(tsc);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001200 *data = svm->vmcb->control.tsc_offset + tsc;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001201 break;
1202 }
Avi Kivity0e859ca2006-12-22 01:05:08 -08001203 case MSR_K6_STAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001204 *data = svm->vmcb->save.star;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001205 break;
Avi Kivity0e859ca2006-12-22 01:05:08 -08001206#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001207 case MSR_LSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001208 *data = svm->vmcb->save.lstar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001209 break;
1210 case MSR_CSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001211 *data = svm->vmcb->save.cstar;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001212 break;
1213 case MSR_KERNEL_GS_BASE:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001214 *data = svm->vmcb->save.kernel_gs_base;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001215 break;
1216 case MSR_SYSCALL_MASK:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001217 *data = svm->vmcb->save.sfmask;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001218 break;
1219#endif
1220 case MSR_IA32_SYSENTER_CS:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001221 *data = svm->vmcb->save.sysenter_cs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001222 break;
1223 case MSR_IA32_SYSENTER_EIP:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001224 *data = svm->vmcb->save.sysenter_eip;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001225 break;
1226 case MSR_IA32_SYSENTER_ESP:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001227 *data = svm->vmcb->save.sysenter_esp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001228 break;
Joerg Roedela2938c82008-02-13 16:30:28 +01001229 /* Nobody will change the following 5 values in the VMCB so
1230 we can safely return them on rdmsr. They will always be 0
1231 until LBRV is implemented. */
1232 case MSR_IA32_DEBUGCTLMSR:
1233 *data = svm->vmcb->save.dbgctl;
1234 break;
1235 case MSR_IA32_LASTBRANCHFROMIP:
1236 *data = svm->vmcb->save.br_from;
1237 break;
1238 case MSR_IA32_LASTBRANCHTOIP:
1239 *data = svm->vmcb->save.br_to;
1240 break;
1241 case MSR_IA32_LASTINTFROMIP:
1242 *data = svm->vmcb->save.last_excp_from;
1243 break;
1244 case MSR_IA32_LASTINTTOIP:
1245 *data = svm->vmcb->save.last_excp_to;
1246 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001247 default:
Avi Kivity3bab1f52006-12-29 16:49:48 -08001248 return kvm_get_msr_common(vcpu, ecx, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001249 }
1250 return 0;
1251}
1252
Rusty Russelle756fc62007-07-30 20:07:08 +10001253static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001254{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001255 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
Avi Kivity6aa8b732006-12-10 02:21:36 -08001256 u64 data;
1257
Rusty Russelle756fc62007-07-30 20:07:08 +10001258 if (svm_get_msr(&svm->vcpu, ecx, &data))
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02001259 kvm_inject_gp(&svm->vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001260 else {
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001261 KVMTRACE_3D(MSR_READ, &svm->vcpu, ecx, (u32)data,
1262 (u32)(data >> 32), handler);
1263
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001264 svm->vmcb->save.rax = data & 0xffffffff;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001265 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001266 svm->next_rip = svm->vmcb->save.rip + 2;
Rusty Russelle756fc62007-07-30 20:07:08 +10001267 skip_emulated_instruction(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001268 }
1269 return 1;
1270}
1271
1272static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
1273{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001274 struct vcpu_svm *svm = to_svm(vcpu);
1275
Avi Kivity6aa8b732006-12-10 02:21:36 -08001276 switch (ecx) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001277 case MSR_IA32_TIME_STAMP_COUNTER: {
1278 u64 tsc;
1279
1280 rdtscll(tsc);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001281 svm->vmcb->control.tsc_offset = data - tsc;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001282 break;
1283 }
Avi Kivity0e859ca2006-12-22 01:05:08 -08001284 case MSR_K6_STAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001285 svm->vmcb->save.star = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001286 break;
Robert P. J. Day49b14f22007-01-29 13:19:50 -08001287#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001288 case MSR_LSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001289 svm->vmcb->save.lstar = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001290 break;
1291 case MSR_CSTAR:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001292 svm->vmcb->save.cstar = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001293 break;
1294 case MSR_KERNEL_GS_BASE:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001295 svm->vmcb->save.kernel_gs_base = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001296 break;
1297 case MSR_SYSCALL_MASK:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001298 svm->vmcb->save.sfmask = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001299 break;
1300#endif
1301 case MSR_IA32_SYSENTER_CS:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001302 svm->vmcb->save.sysenter_cs = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001303 break;
1304 case MSR_IA32_SYSENTER_EIP:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001305 svm->vmcb->save.sysenter_eip = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001306 break;
1307 case MSR_IA32_SYSENTER_ESP:
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001308 svm->vmcb->save.sysenter_esp = data;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001309 break;
Joerg Roedela2938c82008-02-13 16:30:28 +01001310 case MSR_IA32_DEBUGCTLMSR:
Joerg Roedel24e09cb2008-02-13 18:58:47 +01001311 if (!svm_has(SVM_FEATURE_LBRV)) {
1312 pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001313 __func__, data);
Joerg Roedel24e09cb2008-02-13 18:58:47 +01001314 break;
1315 }
1316 if (data & DEBUGCTL_RESERVED_BITS)
1317 return 1;
1318
1319 svm->vmcb->save.dbgctl = data;
1320 if (data & (1ULL<<0))
1321 svm_enable_lbrv(svm);
1322 else
1323 svm_disable_lbrv(svm);
Joerg Roedela2938c82008-02-13 16:30:28 +01001324 break;
Joerg Roedel62b9aba2007-12-11 15:36:57 +01001325 case MSR_K7_EVNTSEL0:
1326 case MSR_K7_EVNTSEL1:
1327 case MSR_K7_EVNTSEL2:
1328 case MSR_K7_EVNTSEL3:
Chris Lalancette14ae51b2008-05-05 13:05:16 -04001329 case MSR_K7_PERFCTR0:
1330 case MSR_K7_PERFCTR1:
1331 case MSR_K7_PERFCTR2:
1332 case MSR_K7_PERFCTR3:
Joerg Roedel62b9aba2007-12-11 15:36:57 +01001333 /*
Chris Lalancette14ae51b2008-05-05 13:05:16 -04001334 * Just discard all writes to the performance counters; this
1335 * should keep both older linux and windows 64-bit guests
1336 * happy
Joerg Roedel62b9aba2007-12-11 15:36:57 +01001337 */
Chris Lalancette14ae51b2008-05-05 13:05:16 -04001338 pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", ecx, data);
1339
Joerg Roedel62b9aba2007-12-11 15:36:57 +01001340 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001341 default:
Avi Kivity3bab1f52006-12-29 16:49:48 -08001342 return kvm_set_msr_common(vcpu, ecx, data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001343 }
1344 return 0;
1345}
1346
Rusty Russelle756fc62007-07-30 20:07:08 +10001347static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001348{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001349 u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001350 u64 data = (svm->vmcb->save.rax & -1u)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001351 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001352
1353 KVMTRACE_3D(MSR_WRITE, &svm->vcpu, ecx, (u32)data, (u32)(data >> 32),
1354 handler);
1355
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001356 svm->next_rip = svm->vmcb->save.rip + 2;
Rusty Russelle756fc62007-07-30 20:07:08 +10001357 if (svm_set_msr(&svm->vcpu, ecx, data))
Avi Kivityc1a5d4f2007-11-25 14:12:03 +02001358 kvm_inject_gp(&svm->vcpu, 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001359 else
Rusty Russelle756fc62007-07-30 20:07:08 +10001360 skip_emulated_instruction(&svm->vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001361 return 1;
1362}
1363
Rusty Russelle756fc62007-07-30 20:07:08 +10001364static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001365{
Rusty Russelle756fc62007-07-30 20:07:08 +10001366 if (svm->vmcb->control.exit_info_1)
1367 return wrmsr_interception(svm, kvm_run);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001368 else
Rusty Russelle756fc62007-07-30 20:07:08 +10001369 return rdmsr_interception(svm, kvm_run);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001370}
1371
Rusty Russelle756fc62007-07-30 20:07:08 +10001372static int interrupt_window_interception(struct vcpu_svm *svm,
Dor Laorc1150d82007-01-05 16:36:24 -08001373 struct kvm_run *kvm_run)
1374{
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001375 KVMTRACE_0D(PEND_INTR, &svm->vcpu, handler);
1376
Eddie Dong85f455f2007-07-06 12:20:49 +03001377 svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
1378 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
Dor Laorc1150d82007-01-05 16:36:24 -08001379 /*
1380 * If the user space waits to inject interrupts, exit as soon as
1381 * possible
1382 */
1383 if (kvm_run->request_interrupt_window &&
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001384 !svm->vcpu.arch.irq_summary) {
Rusty Russelle756fc62007-07-30 20:07:08 +10001385 ++svm->vcpu.stat.irq_window_exits;
Dor Laorc1150d82007-01-05 16:36:24 -08001386 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
1387 return 0;
1388 }
1389
1390 return 1;
1391}
1392
Rusty Russelle756fc62007-07-30 20:07:08 +10001393static int (*svm_exit_handlers[])(struct vcpu_svm *svm,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001394 struct kvm_run *kvm_run) = {
1395 [SVM_EXIT_READ_CR0] = emulate_on_interception,
1396 [SVM_EXIT_READ_CR3] = emulate_on_interception,
1397 [SVM_EXIT_READ_CR4] = emulate_on_interception,
Avi Kivity80a81192007-12-06 19:50:00 +02001398 [SVM_EXIT_READ_CR8] = emulate_on_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001399 /* for now: */
1400 [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
1401 [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
1402 [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
Joerg Roedel1d075432007-12-06 21:02:25 +01001403 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001404 [SVM_EXIT_READ_DR0] = emulate_on_interception,
1405 [SVM_EXIT_READ_DR1] = emulate_on_interception,
1406 [SVM_EXIT_READ_DR2] = emulate_on_interception,
1407 [SVM_EXIT_READ_DR3] = emulate_on_interception,
1408 [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
1409 [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
1410 [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
1411 [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
1412 [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
1413 [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
Anthony Liguori7aa81cc2007-09-17 14:57:50 -05001414 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001415 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
Anthony Liguori7807fa62007-04-23 09:17:21 -05001416 [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception,
Joerg Roedel53371b52008-04-09 14:15:30 +02001417 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
Joerg Roedela0698052008-04-30 17:56:01 +02001418 [SVM_EXIT_INTR] = intr_interception,
Joerg Roedelc47f0982008-04-30 17:56:00 +02001419 [SVM_EXIT_NMI] = nmi_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001420 [SVM_EXIT_SMI] = nop_on_interception,
1421 [SVM_EXIT_INIT] = nop_on_interception,
Dor Laorc1150d82007-01-05 16:36:24 -08001422 [SVM_EXIT_VINTR] = interrupt_window_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001423 /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */
1424 [SVM_EXIT_CPUID] = cpuid_interception,
Avi Kivitycf5a94d2007-10-28 16:11:58 +02001425 [SVM_EXIT_INVD] = emulate_on_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001426 [SVM_EXIT_HLT] = halt_interception,
1427 [SVM_EXIT_INVLPG] = emulate_on_interception,
1428 [SVM_EXIT_INVLPGA] = invalid_op_interception,
1429 [SVM_EXIT_IOIO] = io_interception,
1430 [SVM_EXIT_MSR] = msr_interception,
1431 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
Joerg Roedel46fe4dd2007-01-26 00:56:42 -08001432 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001433 [SVM_EXIT_VMRUN] = invalid_op_interception,
Avi Kivity02e235b2007-02-19 14:37:47 +02001434 [SVM_EXIT_VMMCALL] = vmmcall_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001435 [SVM_EXIT_VMLOAD] = invalid_op_interception,
1436 [SVM_EXIT_VMSAVE] = invalid_op_interception,
1437 [SVM_EXIT_STGI] = invalid_op_interception,
1438 [SVM_EXIT_CLGI] = invalid_op_interception,
1439 [SVM_EXIT_SKINIT] = invalid_op_interception,
Avi Kivitycf5a94d2007-10-28 16:11:58 +02001440 [SVM_EXIT_WBINVD] = emulate_on_interception,
Joerg Roedel916ce232007-03-21 19:47:00 +01001441 [SVM_EXIT_MONITOR] = invalid_op_interception,
1442 [SVM_EXIT_MWAIT] = invalid_op_interception,
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001443 [SVM_EXIT_NPF] = pf_interception,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001444};
1445
Avi Kivity04d2cc72007-09-10 18:10:54 +03001446static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001447{
Avi Kivity04d2cc72007-09-10 18:10:54 +03001448 struct vcpu_svm *svm = to_svm(vcpu);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001449 u32 exit_code = svm->vmcb->control.exit_code;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001450
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001451 KVMTRACE_3D(VMEXIT, vcpu, exit_code, (u32)svm->vmcb->save.rip,
1452 (u32)((u64)svm->vmcb->save.rip >> 32), entryexit);
1453
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001454 if (npt_enabled) {
1455 int mmu_reload = 0;
1456 if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) {
1457 svm_set_cr0(vcpu, svm->vmcb->save.cr0);
1458 mmu_reload = 1;
1459 }
1460 vcpu->arch.cr0 = svm->vmcb->save.cr0;
1461 vcpu->arch.cr3 = svm->vmcb->save.cr3;
1462 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1463 if (!load_pdptrs(vcpu, vcpu->arch.cr3)) {
1464 kvm_inject_gp(vcpu, 0);
1465 return 1;
1466 }
1467 }
1468 if (mmu_reload) {
1469 kvm_mmu_reset_context(vcpu);
1470 kvm_mmu_load(vcpu);
1471 }
1472 }
1473
Avi Kivity04d2cc72007-09-10 18:10:54 +03001474 kvm_reput_irq(svm);
1475
1476 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
1477 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1478 kvm_run->fail_entry.hardware_entry_failure_reason
1479 = svm->vmcb->control.exit_code;
1480 return 0;
1481 }
1482
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001483 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001484 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
1485 exit_code != SVM_EXIT_NPF)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001486 printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
1487 "exit_code 0x%x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001488 __func__, svm->vmcb->control.exit_int_info,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001489 exit_code);
1490
Ahmed S. Darwish9d8f5492007-02-19 14:37:46 +02001491 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
Joe Perches56919c52007-11-12 20:06:51 -08001492 || !svm_exit_handlers[exit_code]) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001493 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
Avi Kivity364b6252007-04-16 14:28:40 +03001494 kvm_run->hw.hardware_exit_reason = exit_code;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001495 return 0;
1496 }
1497
Rusty Russelle756fc62007-07-30 20:07:08 +10001498 return svm_exit_handlers[exit_code](svm, kvm_run);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001499}
1500
1501static void reload_tss(struct kvm_vcpu *vcpu)
1502{
1503 int cpu = raw_smp_processor_id();
1504
1505 struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
Mike Dayd77c26f2007-10-08 09:02:08 -04001506 svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */
Avi Kivity6aa8b732006-12-10 02:21:36 -08001507 load_TR_desc();
1508}
1509
Rusty Russelle756fc62007-07-30 20:07:08 +10001510static void pre_svm_run(struct vcpu_svm *svm)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001511{
1512 int cpu = raw_smp_processor_id();
1513
1514 struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
1515
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001516 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
Rusty Russelle756fc62007-07-30 20:07:08 +10001517 if (svm->vcpu.cpu != cpu ||
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001518 svm->asid_generation != svm_data->asid_generation)
Rusty Russelle756fc62007-07-30 20:07:08 +10001519 new_asid(svm, svm_data);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001520}
1521
1522
Eddie Dong85f455f2007-07-06 12:20:49 +03001523static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001524{
1525 struct vmcb_control_area *control;
1526
Joerg Roedelaf9ca2d2008-04-30 17:56:03 +02001527 KVMTRACE_1D(INJ_VIRQ, &svm->vcpu, (u32)irq, handler);
1528
Rusty Russelle756fc62007-07-30 20:07:08 +10001529 control = &svm->vmcb->control;
Eddie Dong85f455f2007-07-06 12:20:49 +03001530 control->int_vector = irq;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001531 control->int_ctl &= ~V_INTR_PRIO_MASK;
1532 control->int_ctl |= V_IRQ_MASK |
1533 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1534}
1535
Eddie Dong2a8067f2007-08-06 16:29:07 +03001536static void svm_set_irq(struct kvm_vcpu *vcpu, int irq)
1537{
1538 struct vcpu_svm *svm = to_svm(vcpu);
1539
1540 svm_inject_irq(svm, irq);
1541}
1542
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001543static void update_cr8_intercept(struct kvm_vcpu *vcpu)
1544{
1545 struct vcpu_svm *svm = to_svm(vcpu);
1546 struct vmcb *vmcb = svm->vmcb;
1547 int max_irr, tpr;
1548
1549 if (!irqchip_in_kernel(vcpu->kvm) || vcpu->arch.apic->vapic_addr)
1550 return;
1551
1552 vmcb->control.intercept_cr_write &= ~INTERCEPT_CR8_MASK;
1553
1554 max_irr = kvm_lapic_find_highest_irr(vcpu);
1555 if (max_irr == -1)
1556 return;
1557
1558 tpr = kvm_lapic_get_cr8(vcpu) << 4;
1559
1560 if (tpr >= (max_irr & 0xf0))
1561 vmcb->control.intercept_cr_write |= INTERCEPT_CR8_MASK;
1562}
1563
Avi Kivity04d2cc72007-09-10 18:10:54 +03001564static void svm_intr_assist(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001565{
Avi Kivity04d2cc72007-09-10 18:10:54 +03001566 struct vcpu_svm *svm = to_svm(vcpu);
Eddie Dong85f455f2007-07-06 12:20:49 +03001567 struct vmcb *vmcb = svm->vmcb;
1568 int intr_vector = -1;
1569
1570 if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) &&
1571 ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) {
1572 intr_vector = vmcb->control.exit_int_info &
1573 SVM_EVTINJ_VEC_MASK;
1574 vmcb->control.exit_int_info = 0;
1575 svm_inject_irq(svm, intr_vector);
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001576 goto out;
Eddie Dong85f455f2007-07-06 12:20:49 +03001577 }
1578
1579 if (vmcb->control.int_ctl & V_IRQ_MASK)
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001580 goto out;
Eddie Dong85f455f2007-07-06 12:20:49 +03001581
Eddie Dong1b9778d2007-09-03 16:56:58 +03001582 if (!kvm_cpu_has_interrupt(vcpu))
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001583 goto out;
Eddie Dong85f455f2007-07-06 12:20:49 +03001584
1585 if (!(vmcb->save.rflags & X86_EFLAGS_IF) ||
1586 (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
1587 (vmcb->control.event_inj & SVM_EVTINJ_VALID)) {
1588 /* unable to deliver irq, set pending irq */
1589 vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR);
1590 svm_inject_irq(svm, 0x0);
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001591 goto out;
Eddie Dong85f455f2007-07-06 12:20:49 +03001592 }
1593 /* Okay, we can deliver the interrupt: grab it and update PIC state. */
Eddie Dong1b9778d2007-09-03 16:56:58 +03001594 intr_vector = kvm_cpu_get_interrupt(vcpu);
Eddie Dong85f455f2007-07-06 12:20:49 +03001595 svm_inject_irq(svm, intr_vector);
Eddie Dong1b9778d2007-09-03 16:56:58 +03001596 kvm_timer_intr_post(vcpu, intr_vector);
Joerg Roedelaaacfc92008-04-16 16:51:18 +02001597out:
1598 update_cr8_intercept(vcpu);
Eddie Dong85f455f2007-07-06 12:20:49 +03001599}
1600
1601static void kvm_reput_irq(struct vcpu_svm *svm)
1602{
Rusty Russelle756fc62007-07-30 20:07:08 +10001603 struct vmcb_control_area *control = &svm->vmcb->control;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001604
Eddie Dong7017fc32007-07-18 11:34:57 +03001605 if ((control->int_ctl & V_IRQ_MASK)
1606 && !irqchip_in_kernel(svm->vcpu.kvm)) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001607 control->int_ctl &= ~V_IRQ_MASK;
Rusty Russelle756fc62007-07-30 20:07:08 +10001608 push_irq(&svm->vcpu, control->int_vector);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001609 }
Dor Laorc1150d82007-01-05 16:36:24 -08001610
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001611 svm->vcpu.arch.interrupt_window_open =
Dor Laorc1150d82007-01-05 16:36:24 -08001612 !(control->int_state & SVM_INTERRUPT_SHADOW_MASK);
1613}
1614
Eddie Dong85f455f2007-07-06 12:20:49 +03001615static void svm_do_inject_vector(struct vcpu_svm *svm)
1616{
1617 struct kvm_vcpu *vcpu = &svm->vcpu;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001618 int word_index = __ffs(vcpu->arch.irq_summary);
1619 int bit_index = __ffs(vcpu->arch.irq_pending[word_index]);
Eddie Dong85f455f2007-07-06 12:20:49 +03001620 int irq = word_index * BITS_PER_LONG + bit_index;
1621
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001622 clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]);
1623 if (!vcpu->arch.irq_pending[word_index])
1624 clear_bit(word_index, &vcpu->arch.irq_summary);
Eddie Dong85f455f2007-07-06 12:20:49 +03001625 svm_inject_irq(svm, irq);
1626}
1627
Avi Kivity04d2cc72007-09-10 18:10:54 +03001628static void do_interrupt_requests(struct kvm_vcpu *vcpu,
Dor Laorc1150d82007-01-05 16:36:24 -08001629 struct kvm_run *kvm_run)
1630{
Avi Kivity04d2cc72007-09-10 18:10:54 +03001631 struct vcpu_svm *svm = to_svm(vcpu);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001632 struct vmcb_control_area *control = &svm->vmcb->control;
Dor Laorc1150d82007-01-05 16:36:24 -08001633
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001634 svm->vcpu.arch.interrupt_window_open =
Dor Laorc1150d82007-01-05 16:36:24 -08001635 (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001636 (svm->vmcb->save.rflags & X86_EFLAGS_IF));
Dor Laorc1150d82007-01-05 16:36:24 -08001637
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001638 if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary)
Dor Laorc1150d82007-01-05 16:36:24 -08001639 /*
1640 * If interrupts enabled, and not blocked by sti or mov ss. Good.
1641 */
Eddie Dong85f455f2007-07-06 12:20:49 +03001642 svm_do_inject_vector(svm);
Dor Laorc1150d82007-01-05 16:36:24 -08001643
1644 /*
1645 * Interrupts blocked. Wait for unblock.
1646 */
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001647 if (!svm->vcpu.arch.interrupt_window_open &&
1648 (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window))
Dor Laorc1150d82007-01-05 16:36:24 -08001649 control->intercept |= 1ULL << INTERCEPT_VINTR;
Mike Dayd77c26f2007-10-08 09:02:08 -04001650 else
Dor Laorc1150d82007-01-05 16:36:24 -08001651 control->intercept &= ~(1ULL << INTERCEPT_VINTR);
1652}
1653
Izik Eiduscbc94022007-10-25 00:29:55 +02001654static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
1655{
1656 return 0;
1657}
1658
Avi Kivity6aa8b732006-12-10 02:21:36 -08001659static void save_db_regs(unsigned long *db_regs)
1660{
Avi Kivity5aff4582006-12-13 00:33:45 -08001661 asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0]));
1662 asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1]));
1663 asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2]));
1664 asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3]));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001665}
1666
1667static void load_db_regs(unsigned long *db_regs)
1668{
Avi Kivity5aff4582006-12-13 00:33:45 -08001669 asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0]));
1670 asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1]));
1671 asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2]));
1672 asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3]));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001673}
1674
Avi Kivityd9e368d2007-06-07 19:18:30 +03001675static void svm_flush_tlb(struct kvm_vcpu *vcpu)
1676{
1677 force_new_asid(vcpu);
1678}
1679
Avi Kivity04d2cc72007-09-10 18:10:54 +03001680static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
1681{
1682}
1683
Joerg Roedeld7bf8222008-04-16 16:51:17 +02001684static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
1685{
1686 struct vcpu_svm *svm = to_svm(vcpu);
1687
1688 if (!(svm->vmcb->control.intercept_cr_write & INTERCEPT_CR8_MASK)) {
1689 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
1690 kvm_lapic_set_tpr(vcpu, cr8);
1691 }
1692}
1693
Joerg Roedel649d6862008-04-16 16:51:15 +02001694static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
1695{
1696 struct vcpu_svm *svm = to_svm(vcpu);
1697 u64 cr8;
1698
1699 if (!irqchip_in_kernel(vcpu->kvm))
1700 return;
1701
1702 cr8 = kvm_get_cr8(vcpu);
1703 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
1704 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
1705}
1706
Avi Kivity04d2cc72007-09-10 18:10:54 +03001707static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001708{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001709 struct vcpu_svm *svm = to_svm(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001710 u16 fs_selector;
1711 u16 gs_selector;
1712 u16 ldt_selector;
Avi Kivityd9e368d2007-06-07 19:18:30 +03001713
Rusty Russelle756fc62007-07-30 20:07:08 +10001714 pre_svm_run(svm);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001715
Joerg Roedel649d6862008-04-16 16:51:15 +02001716 sync_lapic_to_cr8(vcpu);
1717
Avi Kivity6aa8b732006-12-10 02:21:36 -08001718 save_host_msrs(vcpu);
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001719 fs_selector = kvm_read_fs();
1720 gs_selector = kvm_read_gs();
1721 ldt_selector = kvm_read_ldt();
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001722 svm->host_cr2 = kvm_read_cr2();
1723 svm->host_dr6 = read_dr6();
1724 svm->host_dr7 = read_dr7();
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001725 svm->vmcb->save.cr2 = vcpu->arch.cr2;
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001726 /* required for live migration with NPT */
1727 if (npt_enabled)
1728 svm->vmcb->save.cr3 = vcpu->arch.cr3;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001729
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001730 if (svm->vmcb->save.dr7 & 0xff) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001731 write_dr7(0);
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001732 save_db_regs(svm->host_db_regs);
1733 load_db_regs(svm->db_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001734 }
Avi Kivity36241b82006-12-22 01:05:20 -08001735
Avi Kivity04d2cc72007-09-10 18:10:54 +03001736 clgi();
1737
1738 local_irq_enable();
Avi Kivity36241b82006-12-22 01:05:20 -08001739
Avi Kivity6aa8b732006-12-10 02:21:36 -08001740 asm volatile (
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001741#ifdef CONFIG_X86_64
Laurent Vivier54a08c02007-10-25 14:18:53 +02001742 "push %%rbp; \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001743#else
Laurent Vivierfe7935d2007-10-25 14:18:54 +02001744 "push %%ebp; \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001745#endif
1746
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001747#ifdef CONFIG_X86_64
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001748 "mov %c[rbx](%[svm]), %%rbx \n\t"
1749 "mov %c[rcx](%[svm]), %%rcx \n\t"
1750 "mov %c[rdx](%[svm]), %%rdx \n\t"
1751 "mov %c[rsi](%[svm]), %%rsi \n\t"
1752 "mov %c[rdi](%[svm]), %%rdi \n\t"
1753 "mov %c[rbp](%[svm]), %%rbp \n\t"
1754 "mov %c[r8](%[svm]), %%r8 \n\t"
1755 "mov %c[r9](%[svm]), %%r9 \n\t"
1756 "mov %c[r10](%[svm]), %%r10 \n\t"
1757 "mov %c[r11](%[svm]), %%r11 \n\t"
1758 "mov %c[r12](%[svm]), %%r12 \n\t"
1759 "mov %c[r13](%[svm]), %%r13 \n\t"
1760 "mov %c[r14](%[svm]), %%r14 \n\t"
1761 "mov %c[r15](%[svm]), %%r15 \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001762#else
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001763 "mov %c[rbx](%[svm]), %%ebx \n\t"
1764 "mov %c[rcx](%[svm]), %%ecx \n\t"
1765 "mov %c[rdx](%[svm]), %%edx \n\t"
1766 "mov %c[rsi](%[svm]), %%esi \n\t"
1767 "mov %c[rdi](%[svm]), %%edi \n\t"
1768 "mov %c[rbp](%[svm]), %%ebp \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001769#endif
1770
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001771#ifdef CONFIG_X86_64
Avi Kivity6aa8b732006-12-10 02:21:36 -08001772 /* Enter guest mode */
1773 "push %%rax \n\t"
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001774 "mov %c[vmcb](%[svm]), %%rax \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001775 __ex(SVM_VMLOAD) "\n\t"
1776 __ex(SVM_VMRUN) "\n\t"
1777 __ex(SVM_VMSAVE) "\n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001778 "pop %%rax \n\t"
1779#else
1780 /* Enter guest mode */
1781 "push %%eax \n\t"
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001782 "mov %c[vmcb](%[svm]), %%eax \n\t"
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001783 __ex(SVM_VMLOAD) "\n\t"
1784 __ex(SVM_VMRUN) "\n\t"
1785 __ex(SVM_VMSAVE) "\n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001786 "pop %%eax \n\t"
1787#endif
1788
1789 /* Save guest registers, load host registers */
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001790#ifdef CONFIG_X86_64
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001791 "mov %%rbx, %c[rbx](%[svm]) \n\t"
1792 "mov %%rcx, %c[rcx](%[svm]) \n\t"
1793 "mov %%rdx, %c[rdx](%[svm]) \n\t"
1794 "mov %%rsi, %c[rsi](%[svm]) \n\t"
1795 "mov %%rdi, %c[rdi](%[svm]) \n\t"
1796 "mov %%rbp, %c[rbp](%[svm]) \n\t"
1797 "mov %%r8, %c[r8](%[svm]) \n\t"
1798 "mov %%r9, %c[r9](%[svm]) \n\t"
1799 "mov %%r10, %c[r10](%[svm]) \n\t"
1800 "mov %%r11, %c[r11](%[svm]) \n\t"
1801 "mov %%r12, %c[r12](%[svm]) \n\t"
1802 "mov %%r13, %c[r13](%[svm]) \n\t"
1803 "mov %%r14, %c[r14](%[svm]) \n\t"
1804 "mov %%r15, %c[r15](%[svm]) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001805
Laurent Vivier54a08c02007-10-25 14:18:53 +02001806 "pop %%rbp; \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001807#else
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001808 "mov %%ebx, %c[rbx](%[svm]) \n\t"
1809 "mov %%ecx, %c[rcx](%[svm]) \n\t"
1810 "mov %%edx, %c[rdx](%[svm]) \n\t"
1811 "mov %%esi, %c[rsi](%[svm]) \n\t"
1812 "mov %%edi, %c[rdi](%[svm]) \n\t"
1813 "mov %%ebp, %c[rbp](%[svm]) \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001814
Laurent Vivierfe7935d2007-10-25 14:18:54 +02001815 "pop %%ebp; \n\t"
Avi Kivity6aa8b732006-12-10 02:21:36 -08001816#endif
1817 :
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001818 : [svm]"a"(svm),
Avi Kivity6aa8b732006-12-10 02:21:36 -08001819 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001820 [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
1821 [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
1822 [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
1823 [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
1824 [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
1825 [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
Avi Kivity05b3e0c2006-12-13 00:33:45 -08001826#ifdef CONFIG_X86_64
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001827 , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
1828 [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
1829 [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
1830 [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
1831 [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
1832 [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
1833 [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
1834 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001835#endif
Laurent Vivier54a08c02007-10-25 14:18:53 +02001836 : "cc", "memory"
1837#ifdef CONFIG_X86_64
1838 , "rbx", "rcx", "rdx", "rsi", "rdi"
1839 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
Laurent Vivierfe7935d2007-10-25 14:18:54 +02001840#else
1841 , "ebx", "ecx", "edx" , "esi", "edi"
Laurent Vivier54a08c02007-10-25 14:18:53 +02001842#endif
1843 );
Avi Kivity6aa8b732006-12-10 02:21:36 -08001844
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001845 if ((svm->vmcb->save.dr7 & 0xff))
1846 load_db_regs(svm->host_db_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001847
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001848 vcpu->arch.cr2 = svm->vmcb->save.cr2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001849
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001850 write_dr6(svm->host_dr6);
1851 write_dr7(svm->host_dr7);
1852 kvm_write_cr2(svm->host_cr2);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001853
Avi Kivityd6e88ae2008-07-10 16:53:33 +03001854 kvm_load_fs(fs_selector);
1855 kvm_load_gs(gs_selector);
1856 kvm_load_ldt(ldt_selector);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001857 load_host_msrs(vcpu);
1858
1859 reload_tss(vcpu);
1860
Avi Kivity56ba47d2007-11-07 17:14:18 +02001861 local_irq_disable();
1862
1863 stgi();
1864
Joerg Roedeld7bf8222008-04-16 16:51:17 +02001865 sync_cr8_to_lapic(vcpu);
1866
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001867 svm->next_rip = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001868}
1869
Avi Kivity6aa8b732006-12-10 02:21:36 -08001870static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
1871{
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001872 struct vcpu_svm *svm = to_svm(vcpu);
1873
Joerg Roedel709ddeb2008-02-07 13:47:45 +01001874 if (npt_enabled) {
1875 svm->vmcb->control.nested_cr3 = root;
1876 force_new_asid(vcpu);
1877 return;
1878 }
1879
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001880 svm->vmcb->save.cr3 = root;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001881 force_new_asid(vcpu);
Anthony Liguori7807fa62007-04-23 09:17:21 -05001882
1883 if (vcpu->fpu_active) {
Gregory Haskinsa2fa3e92007-07-27 08:13:10 -04001884 svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR);
1885 svm->vmcb->save.cr0 |= X86_CR0_TS;
Anthony Liguori7807fa62007-04-23 09:17:21 -05001886 vcpu->fpu_active = 0;
1887 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001888}
1889
Avi Kivity6aa8b732006-12-10 02:21:36 -08001890static int is_disabled(void)
1891{
Joerg Roedel6031a612007-06-22 12:29:50 +03001892 u64 vm_cr;
1893
1894 rdmsrl(MSR_VM_CR, vm_cr);
1895 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
1896 return 1;
1897
Avi Kivity6aa8b732006-12-10 02:21:36 -08001898 return 0;
1899}
1900
Ingo Molnar102d8322007-02-19 14:37:47 +02001901static void
1902svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
1903{
1904 /*
1905 * Patch in the VMMCALL instruction:
1906 */
1907 hypercall[0] = 0x0f;
1908 hypercall[1] = 0x01;
1909 hypercall[2] = 0xd9;
Ingo Molnar102d8322007-02-19 14:37:47 +02001910}
1911
Yang, Sheng002c7f72007-07-31 14:23:01 +03001912static void svm_check_processor_compat(void *rtn)
1913{
1914 *(int *)rtn = 0;
1915}
1916
Avi Kivity774ead32007-12-26 13:57:04 +02001917static bool svm_cpu_has_accelerated_tpr(void)
1918{
1919 return false;
1920}
1921
Sheng Yang67253af2008-04-25 10:20:22 +08001922static int get_npt_level(void)
1923{
1924#ifdef CONFIG_X86_64
1925 return PT64_ROOT_LEVEL;
1926#else
1927 return PT32E_ROOT_LEVEL;
1928#endif
1929}
1930
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03001931static struct kvm_x86_ops svm_x86_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001932 .cpu_has_kvm_support = has_svm,
1933 .disabled_by_bios = is_disabled,
1934 .hardware_setup = svm_hardware_setup,
1935 .hardware_unsetup = svm_hardware_unsetup,
Yang, Sheng002c7f72007-07-31 14:23:01 +03001936 .check_processor_compatibility = svm_check_processor_compat,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001937 .hardware_enable = svm_hardware_enable,
1938 .hardware_disable = svm_hardware_disable,
Avi Kivity774ead32007-12-26 13:57:04 +02001939 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001940
1941 .vcpu_create = svm_create_vcpu,
1942 .vcpu_free = svm_free_vcpu,
Avi Kivity04d2cc72007-09-10 18:10:54 +03001943 .vcpu_reset = svm_vcpu_reset,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001944
Avi Kivity04d2cc72007-09-10 18:10:54 +03001945 .prepare_guest_switch = svm_prepare_guest_switch,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001946 .vcpu_load = svm_vcpu_load,
1947 .vcpu_put = svm_vcpu_put,
1948
1949 .set_guest_debug = svm_guest_debug,
1950 .get_msr = svm_get_msr,
1951 .set_msr = svm_set_msr,
1952 .get_segment_base = svm_get_segment_base,
1953 .get_segment = svm_get_segment,
1954 .set_segment = svm_set_segment,
Izik Eidus2e4d2652008-03-24 19:38:34 +02001955 .get_cpl = svm_get_cpl,
Rusty Russell1747fb72007-09-06 01:21:32 +10001956 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
Anthony Liguori25c4c272007-04-27 09:29:21 +03001957 .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001958 .set_cr0 = svm_set_cr0,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001959 .set_cr3 = svm_set_cr3,
1960 .set_cr4 = svm_set_cr4,
1961 .set_efer = svm_set_efer,
1962 .get_idt = svm_get_idt,
1963 .set_idt = svm_set_idt,
1964 .get_gdt = svm_get_gdt,
1965 .set_gdt = svm_set_gdt,
1966 .get_dr = svm_get_dr,
1967 .set_dr = svm_set_dr,
1968 .cache_regs = svm_cache_regs,
1969 .decache_regs = svm_decache_regs,
1970 .get_rflags = svm_get_rflags,
1971 .set_rflags = svm_set_rflags,
1972
Avi Kivity6aa8b732006-12-10 02:21:36 -08001973 .tlb_flush = svm_flush_tlb,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001974
Avi Kivity6aa8b732006-12-10 02:21:36 -08001975 .run = svm_vcpu_run,
Avi Kivity04d2cc72007-09-10 18:10:54 +03001976 .handle_exit = handle_exit,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001977 .skip_emulated_instruction = skip_emulated_instruction,
Ingo Molnar102d8322007-02-19 14:37:47 +02001978 .patch_hypercall = svm_patch_hypercall,
Eddie Dong2a8067f2007-08-06 16:29:07 +03001979 .get_irq = svm_get_irq,
1980 .set_irq = svm_set_irq,
Avi Kivity298101d2007-11-25 13:41:11 +02001981 .queue_exception = svm_queue_exception,
1982 .exception_injected = svm_exception_injected,
Avi Kivity04d2cc72007-09-10 18:10:54 +03001983 .inject_pending_irq = svm_intr_assist,
1984 .inject_pending_vectors = do_interrupt_requests,
Izik Eiduscbc94022007-10-25 00:29:55 +02001985
1986 .set_tss_addr = svm_set_tss_addr,
Sheng Yang67253af2008-04-25 10:20:22 +08001987 .get_tdp_level = get_npt_level,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001988};
1989
1990static int __init svm_init(void)
1991{
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001992 return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
Rusty Russellc16f8622007-07-30 21:12:19 +10001993 THIS_MODULE);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001994}
1995
1996static void __exit svm_exit(void)
1997{
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001998 kvm_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001999}
2000
2001module_init(svm_init)
2002module_exit(svm_exit)