blob: df5a70311be815f95f35cf0760eb4675af3eb414 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
Nicolas Kaiser9611c182010-10-06 14:23:22 +020010 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity6aa8b732006-12-10 02:21:36 -080011 *
12 * Authors:
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
15 *
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
18 *
19 */
20
21/*
22 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
23 * so the code in this file is compiled twice, once per pte size.
24 */
25
26#if PTTYPE == 64
27 #define pt_element_t u64
28 #define guest_walker guest_walker64
29 #define FNAME(name) paging##64_##name
30 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
Joerg Roedele04da982009-07-27 16:30:45 +020031 #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
32 #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
Avi Kivity6aa8b732006-12-10 02:21:36 -080033 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
Avi Kivityc7addb92007-09-16 18:58:32 +020034 #define PT_LEVEL_BITS PT64_LEVEL_BITS
Avi Kivitycea0f0e2007-01-05 16:36:43 -080035 #ifdef CONFIG_X86_64
36 #define PT_MAX_FULL_LEVELS 4
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050037 #define CMPXCHG cmpxchg
Avi Kivitycea0f0e2007-01-05 16:36:43 -080038 #else
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050039 #define CMPXCHG cmpxchg64
Avi Kivitycea0f0e2007-01-05 16:36:43 -080040 #define PT_MAX_FULL_LEVELS 2
41 #endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080042#elif PTTYPE == 32
43 #define pt_element_t u32
44 #define guest_walker guest_walker32
45 #define FNAME(name) paging##32_##name
46 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
Joerg Roedele04da982009-07-27 16:30:45 +020047 #define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
48 #define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
Avi Kivity6aa8b732006-12-10 02:21:36 -080049 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
Avi Kivityc7addb92007-09-16 18:58:32 +020050 #define PT_LEVEL_BITS PT32_LEVEL_BITS
Avi Kivitycea0f0e2007-01-05 16:36:43 -080051 #define PT_MAX_FULL_LEVELS 2
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050052 #define CMPXCHG cmpxchg
Avi Kivity6aa8b732006-12-10 02:21:36 -080053#else
54 #error Invalid PTTYPE value
55#endif
56
Joerg Roedele04da982009-07-27 16:30:45 +020057#define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
58#define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
Avi Kivity5fb07dd2007-11-21 12:35:07 +020059
Avi Kivity6aa8b732006-12-10 02:21:36 -080060/*
61 * The guest_walker structure emulates the behavior of the hardware page
62 * table walker.
63 */
64struct guest_walker {
65 int level;
Avi Kivitycea0f0e2007-01-05 16:36:43 -080066 gfn_t table_gfn[PT_MAX_FULL_LEVELS];
Marcelo Tosatti78190262007-12-11 19:12:27 -050067 pt_element_t ptes[PT_MAX_FULL_LEVELS];
Xiao Guangrong189be382010-08-22 19:13:33 +080068 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
Marcelo Tosatti78190262007-12-11 19:12:27 -050069 gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
Avi Kivityfe135d22007-12-09 16:15:46 +020070 unsigned pt_access;
71 unsigned pte_access;
Avi Kivity815af8d2007-01-05 16:36:44 -080072 gfn_t gfn;
Avi Kivity8c28d032010-11-22 17:53:27 +020073 struct x86_exception fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -080074};
75
Joerg Roedele04da982009-07-27 16:30:45 +020076static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
Avi Kivity5fb07dd2007-11-21 12:35:07 +020077{
Joerg Roedele04da982009-07-27 16:30:45 +020078 return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
Avi Kivity5fb07dd2007-11-21 12:35:07 +020079}
80
Roedel, Joerga78484c2011-04-20 15:33:16 +020081static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
Takuya Yoshikawac8cfbb52011-05-01 14:33:07 +090082 pt_element_t __user *ptep_user, unsigned index,
83 pt_element_t orig_pte, pt_element_t new_pte)
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050084{
Takuya Yoshikawac8cfbb52011-05-01 14:33:07 +090085 int npages;
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050086 pt_element_t ret;
87 pt_element_t *table;
88 struct page *page;
89
Takuya Yoshikawac8cfbb52011-05-01 14:33:07 +090090 npages = get_user_pages_fast((unsigned long)ptep_user, 1, 1, &page);
91 /* Check if the user is doing something meaningless. */
92 if (unlikely(npages != 1))
Roedel, Joerga78484c2011-04-20 15:33:16 +020093 return -EFAULT;
94
Cong Wang8fd75e12011-11-25 23:14:17 +080095 table = kmap_atomic(page);
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050096 ret = CMPXCHG(&table[index], orig_pte, new_pte);
Cong Wang8fd75e12011-11-25 23:14:17 +080097 kunmap_atomic(table);
Marcelo Tosattib3e4e632007-12-07 07:56:58 -050098
99 kvm_release_page_dirty(page);
100
101 return (ret != orig_pte);
102}
103
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800104static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte,
105 bool last)
Avi Kivitybedbe4e2007-12-09 16:52:56 +0200106{
107 unsigned access;
108
109 access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800110 if (last && !is_dirty_gpte(gpte))
111 access &= ~ACC_WRITE_MASK;
112
Avi Kivitybedbe4e2007-12-09 16:52:56 +0200113#if PTTYPE == 64
Joerg Roedel2d48a982010-09-10 17:31:01 +0200114 if (vcpu->arch.mmu.nx)
Avi Kivitybedbe4e2007-12-09 16:52:56 +0200115 access &= ~(gpte >> PT64_NX_SHIFT);
116#endif
117 return access;
118}
119
Takuya Yoshikawa3c8c6522011-07-01 01:37:24 +0900120static bool FNAME(is_last_gpte)(struct guest_walker *walker,
121 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
122 pt_element_t gpte)
123{
124 if (walker->level == PT_PAGE_TABLE_LEVEL)
125 return true;
126
127 if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) &&
128 (PTTYPE == 64 || is_pse(vcpu)))
129 return true;
130
131 if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) &&
132 (mmu->root_level == PT64_ROOT_LEVEL))
133 return true;
134
135 return false;
136}
137
Avi Kivityac79c972007-01-05 16:36:40 -0800138/*
139 * Fetch a guest pte for a guest virtual address
140 */
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200141static int FNAME(walk_addr_generic)(struct guest_walker *walker,
142 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
Xiao Guangrong33770782010-09-28 17:03:14 +0800143 gva_t addr, u32 access)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800144{
Avi Kivity42bf3f02007-10-17 12:18:47 +0200145 pt_element_t pte;
Borislav Petkovb7233632011-05-30 22:11:17 +0200146 pt_element_t __user *uninitialized_var(ptep_user);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800147 gfn_t table_gfn;
Avi Kivityf59c1d22010-07-06 16:20:43 +0300148 unsigned index, pt_access, uninitialized_var(pte_access);
Avi Kivity42bf3f02007-10-17 12:18:47 +0200149 gpa_t pte_gpa;
Yang, Wei Ycd468682011-08-09 18:14:01 +0800150 bool eperm, last_gpte;
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900151 int offset;
152 const int write_fault = access & PFERR_WRITE_MASK;
153 const int user_fault = access & PFERR_USER_MASK;
154 const int fetch_fault = access & PFERR_FETCH_MASK;
155 u16 errcode = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800156
Avi Kivity07420172009-07-06 12:21:32 +0300157 trace_kvm_mmu_pagetable_walk(addr, write_fault, user_fault,
158 fetch_fault);
Takuya Yoshikawa92c1c1e2011-07-01 01:36:07 +0900159retry_walk:
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900160 eperm = false;
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200161 walker->level = mmu->root_level;
162 pte = mmu->get_cr3(vcpu);
163
Avi Kivity1b0973b2007-01-05 16:36:41 -0800164#if PTTYPE == 64
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200165 if (walker->level == PT32E_ROOT_LEVEL) {
Avi Kivitye4e517b2011-07-28 11:36:17 +0300166 pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
Avi Kivity07420172009-07-06 12:21:32 +0300167 trace_kvm_mmu_paging_element(pte, walker->level);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900168 if (!is_present_gpte(pte))
Avi Kivityf59c1d22010-07-06 16:20:43 +0300169 goto error;
Avi Kivity1b0973b2007-01-05 16:36:41 -0800170 --walker->level;
171 }
172#endif
Avi Kivitya9058ec2006-12-29 16:49:37 -0800173 ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200174 (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800175
Avi Kivityfe135d22007-12-09 16:15:46 +0200176 pt_access = ACC_ALL;
Avi Kivityac79c972007-01-05 16:36:40 -0800177
178 for (;;) {
Takuya Yoshikawa6e2ca7d2011-04-22 00:34:44 +0900179 gfn_t real_gfn;
180 unsigned long host_addr;
181
Avi Kivity42bf3f02007-10-17 12:18:47 +0200182 index = PT_INDEX(addr, walker->level);
Avi Kivityac79c972007-01-05 16:36:40 -0800183
Avi Kivity5fb07dd2007-11-21 12:35:07 +0200184 table_gfn = gpte_to_gfn(pte);
Joerg Roedel2329d462010-09-10 17:30:52 +0200185 offset = index * sizeof(pt_element_t);
186 pte_gpa = gfn_to_gpa(table_gfn) + offset;
Avi Kivity42bf3f02007-10-17 12:18:47 +0200187 walker->table_gfn[walker->level - 1] = table_gfn;
Marcelo Tosatti78190262007-12-11 19:12:27 -0500188 walker->pte_gpa[walker->level - 1] = pte_gpa;
Avi Kivityac79c972007-01-05 16:36:40 -0800189
Takuya Yoshikawa6e2ca7d2011-04-22 00:34:44 +0900190 real_gfn = mmu->translate_gpa(vcpu, gfn_to_gpa(table_gfn),
191 PFERR_USER_MASK|PFERR_WRITE_MASK);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900192 if (unlikely(real_gfn == UNMAPPED_GVA))
193 goto error;
Takuya Yoshikawa6e2ca7d2011-04-22 00:34:44 +0900194 real_gfn = gpa_to_gfn(real_gfn);
195
196 host_addr = gfn_to_hva(vcpu->kvm, real_gfn);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900197 if (unlikely(kvm_is_error_hva(host_addr)))
198 goto error;
Takuya Yoshikawa6e2ca7d2011-04-22 00:34:44 +0900199
200 ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900201 if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte))))
202 goto error;
Marcelo Tosattia6085fb2010-01-14 17:41:27 -0200203
Avi Kivity07420172009-07-06 12:21:32 +0300204 trace_kvm_mmu_paging_element(pte, walker->level);
Avi Kivity42bf3f02007-10-17 12:18:47 +0200205
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900206 if (unlikely(!is_present_gpte(pte)))
207 goto error;
Avi Kivity7993ba42007-01-26 00:56:41 -0800208
Avi Kivity781e0742011-04-24 12:25:50 +0300209 if (unlikely(is_rsvd_bits_set(&vcpu->arch.mmu, pte,
210 walker->level))) {
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900211 errcode |= PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
212 goto error;
Avi Kivityf59c1d22010-07-06 16:20:43 +0300213 }
Dong, Eddie82725b22009-03-30 16:21:08 +0800214
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800215 if (!check_write_user_access(vcpu, write_fault, user_fault,
216 pte))
Avi Kivityf59c1d22010-07-06 16:20:43 +0300217 eperm = true;
Avi Kivity7993ba42007-01-26 00:56:41 -0800218
Avi Kivity73b10872007-01-26 00:56:41 -0800219#if PTTYPE == 64
Avi Kivity781e0742011-04-24 12:25:50 +0300220 if (unlikely(fetch_fault && (pte & PT64_NX_MASK)))
Avi Kivityf59c1d22010-07-06 16:20:43 +0300221 eperm = true;
Avi Kivity73b10872007-01-26 00:56:41 -0800222#endif
223
Yang, Wei Ycd468682011-08-09 18:14:01 +0800224 last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
225 if (last_gpte) {
226 pte_access = pt_access &
227 FNAME(gpte_access)(vcpu, pte, true);
228 /* check if the kernel is fetching from user page */
229 if (unlikely(pte_access & PT_USER_MASK) &&
230 kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
231 if (fetch_fault && !user_fault)
232 eperm = true;
233 }
234
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900235 if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) {
Roedel, Joerga78484c2011-04-20 15:33:16 +0200236 int ret;
Avi Kivity07420172009-07-06 12:21:32 +0300237 trace_kvm_mmu_set_accessed_bit(table_gfn, index,
238 sizeof(pte));
Takuya Yoshikawac8cfbb52011-05-01 14:33:07 +0900239 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index,
240 pte, pte|PT_ACCESSED_MASK);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900241 if (unlikely(ret < 0))
242 goto error;
243 else if (ret)
Takuya Yoshikawa92c1c1e2011-07-01 01:36:07 +0900244 goto retry_walk;
Roedel, Joerga78484c2011-04-20 15:33:16 +0200245
Gui Jianfengf3b8c962010-05-05 09:09:21 +0800246 mark_page_dirty(vcpu->kvm, table_gfn);
Avi Kivity42bf3f02007-10-17 12:18:47 +0200247 pte |= PT_ACCESSED_MASK;
Avi Kivitybf3f8e82007-02-19 14:37:46 +0200248 }
Avi Kivityac79c972007-01-05 16:36:40 -0800249
Marcelo Tosatti78190262007-12-11 19:12:27 -0500250 walker->ptes[walker->level - 1] = pte;
251
Yang, Wei Ycd468682011-08-09 18:14:01 +0800252 if (last_gpte) {
Joerg Roedele04da982009-07-27 16:30:45 +0200253 int lvl = walker->level;
Joerg Roedel2329d462010-09-10 17:30:52 +0200254 gpa_t real_gpa;
255 gfn_t gfn;
Xiao Guangrong33770782010-09-28 17:03:14 +0800256 u32 ac;
Avi Kivity815af8d2007-01-05 16:36:44 -0800257
Joerg Roedel2329d462010-09-10 17:30:52 +0200258 gfn = gpte_to_gfn_lvl(pte, lvl);
259 gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT;
Joerg Roedele04da982009-07-27 16:30:45 +0200260
261 if (PTTYPE == 32 &&
262 walker->level == PT_DIRECTORY_LEVEL &&
263 is_cpuid_PSE36())
Joerg Roedel2329d462010-09-10 17:30:52 +0200264 gfn += pse36_gfn_delta(pte);
265
Xiao Guangrong33770782010-09-28 17:03:14 +0800266 ac = write_fault | fetch_fault | user_fault;
Joerg Roedel2329d462010-09-10 17:30:52 +0200267
268 real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn),
Xiao Guangrong33770782010-09-28 17:03:14 +0800269 ac);
Joerg Roedel2329d462010-09-10 17:30:52 +0200270 if (real_gpa == UNMAPPED_GVA)
271 return 0;
272
273 walker->gfn = real_gpa >> PAGE_SHIFT;
Joerg Roedele04da982009-07-27 16:30:45 +0200274
Avi Kivity815af8d2007-01-05 16:36:44 -0800275 break;
276 }
277
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800278 pt_access &= FNAME(gpte_access)(vcpu, pte, false);
Avi Kivityac79c972007-01-05 16:36:40 -0800279 --walker->level;
280 }
Avi Kivity42bf3f02007-10-17 12:18:47 +0200281
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900282 if (unlikely(eperm)) {
283 errcode |= PFERR_PRESENT_MASK;
Avi Kivityf59c1d22010-07-06 16:20:43 +0300284 goto error;
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900285 }
Avi Kivityf59c1d22010-07-06 16:20:43 +0300286
Avi Kivity781e0742011-04-24 12:25:50 +0300287 if (write_fault && unlikely(!is_dirty_gpte(pte))) {
Roedel, Joerga78484c2011-04-20 15:33:16 +0200288 int ret;
Marcelo Tosattib3e4e632007-12-07 07:56:58 -0500289
Avi Kivity07420172009-07-06 12:21:32 +0300290 trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
Takuya Yoshikawac8cfbb52011-05-01 14:33:07 +0900291 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index,
292 pte, pte|PT_DIRTY_MASK);
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900293 if (unlikely(ret < 0))
Roedel, Joerga78484c2011-04-20 15:33:16 +0200294 goto error;
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900295 else if (ret)
Takuya Yoshikawa92c1c1e2011-07-01 01:36:07 +0900296 goto retry_walk;
Roedel, Joerga78484c2011-04-20 15:33:16 +0200297
Gui Jianfengf3b8c962010-05-05 09:09:21 +0800298 mark_page_dirty(vcpu->kvm, table_gfn);
Avi Kivity42bf3f02007-10-17 12:18:47 +0200299 pte |= PT_DIRTY_MASK;
Marcelo Tosatti78190262007-12-11 19:12:27 -0500300 walker->ptes[walker->level - 1] = pte;
Avi Kivity42bf3f02007-10-17 12:18:47 +0200301 }
302
Avi Kivityfe135d22007-12-09 16:15:46 +0200303 walker->pt_access = pt_access;
304 walker->pte_access = pte_access;
305 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
Gui Jianfeng518c5a02010-05-05 09:58:33 +0800306 __func__, (u64)pte, pte_access, pt_access);
Avi Kivity7993ba42007-01-26 00:56:41 -0800307 return 1;
308
Avi Kivityf59c1d22010-07-06 16:20:43 +0300309error:
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900310 errcode |= write_fault | user_fault;
Yang, Wei Ye57d4a32011-06-03 11:14:16 +0800311 if (fetch_fault && (mmu->nx ||
312 kvm_read_cr4_bits(vcpu, X86_CR4_SMEP)))
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900313 errcode |= PFERR_FETCH_MASK;
Joerg Roedel8df25a32010-09-10 17:30:46 +0200314
Takuya Yoshikawa134291b2011-07-01 01:34:56 +0900315 walker->fault.vector = PF_VECTOR;
316 walker->fault.error_code_valid = true;
317 walker->fault.error_code = errcode;
Avi Kivity6389ee92010-11-29 16:12:30 +0200318 walker->fault.address = addr;
319 walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
Joerg Roedel8df25a32010-09-10 17:30:46 +0200320
Avi Kivity8c28d032010-11-22 17:53:27 +0200321 trace_kvm_mmu_walker_error(walker->fault.error_code);
Shaohua Life551882007-07-23 14:51:39 +0800322 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800323}
324
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200325static int FNAME(walk_addr)(struct guest_walker *walker,
Xiao Guangrong33770782010-09-28 17:03:14 +0800326 struct kvm_vcpu *vcpu, gva_t addr, u32 access)
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200327{
328 return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu, addr,
Xiao Guangrong33770782010-09-28 17:03:14 +0800329 access);
Joerg Roedel1e301fe2010-09-10 17:30:47 +0200330}
331
Joerg Roedel6539e732010-09-10 17:30:50 +0200332static int FNAME(walk_addr_nested)(struct guest_walker *walker,
333 struct kvm_vcpu *vcpu, gva_t addr,
Xiao Guangrong33770782010-09-28 17:03:14 +0800334 u32 access)
Joerg Roedel6539e732010-09-10 17:30:50 +0200335{
336 return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.nested_mmu,
Xiao Guangrong33770782010-09-28 17:03:14 +0800337 addr, access);
Joerg Roedel6539e732010-09-10 17:30:50 +0200338}
339
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800340static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
341 struct kvm_mmu_page *sp, u64 *spte,
342 pt_element_t gpte)
343{
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800344 if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
345 goto no_present;
346
Xiao Guangrongc3707952011-07-12 03:28:04 +0800347 if (!is_present_gpte(gpte))
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800348 goto no_present;
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800349
350 if (!(gpte & PT_ACCESSED_MASK))
351 goto no_present;
352
353 return false;
354
355no_present:
Xiao Guangrongc3707952011-07-12 03:28:04 +0800356 drop_spte(vcpu->kvm, spte);
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800357 return true;
358}
359
Xiao Guangrongac3cd032010-06-11 21:28:14 +0800360static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
Xiao Guangrong7c562522011-03-28 10:29:27 +0800361 u64 *spte, const void *pte)
Avi Kivity00284252007-05-01 16:53:31 +0300362{
363 pt_element_t gpte;
Avi Kivity41074d02007-12-09 17:00:02 +0200364 unsigned pte_access;
Anthony Liguori35149e22008-04-02 14:46:56 -0500365 pfn_t pfn;
Avi Kivity00284252007-05-01 16:53:31 +0300366
Avi Kivity00284252007-05-01 16:53:31 +0300367 gpte = *(const pt_element_t *)pte;
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800368 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
Avi Kivityc7addb92007-09-16 18:58:32 +0200369 return;
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800370
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800371 pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800372 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, true);
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800373 pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
Xiao Guangrongd7c55202011-07-12 03:29:38 +0800374 if (mmu_invalid_pfn(pfn)) {
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800375 kvm_release_pfn_clean(pfn);
Avi Kivityd7824ff2007-12-30 12:29:05 +0200376 return;
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800377 }
Xiao Guangrong0f53b5b2011-03-09 15:43:51 +0800378
Izik Eidus14032832009-09-23 21:47:17 +0300379 /*
Lucas De Marchi0d2eb442011-03-17 16:24:16 -0300380 * we call mmu_set_spte() with host_writable = true because that
Izik Eidus14032832009-09-23 21:47:17 +0300381 * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
382 */
Xiao Guangrongac3cd032010-06-11 21:28:14 +0800383 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800384 NULL, PT_PAGE_TABLE_LEVEL,
Izik Eidus14032832009-09-23 21:47:17 +0300385 gpte_to_gfn(gpte), pfn, true, true);
Avi Kivity00284252007-05-01 16:53:31 +0300386}
387
Avi Kivity39c8c672010-07-13 14:27:08 +0300388static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu,
389 struct guest_walker *gw, int level)
390{
Avi Kivity39c8c672010-07-13 14:27:08 +0300391 pt_element_t curr_pte;
Xiao Guangrong189be382010-08-22 19:13:33 +0800392 gpa_t base_gpa, pte_gpa = gw->pte_gpa[level - 1];
393 u64 mask;
394 int r, index;
Avi Kivity39c8c672010-07-13 14:27:08 +0300395
Xiao Guangrong189be382010-08-22 19:13:33 +0800396 if (level == PT_PAGE_TABLE_LEVEL) {
397 mask = PTE_PREFETCH_NUM * sizeof(pt_element_t) - 1;
398 base_gpa = pte_gpa & ~mask;
399 index = (pte_gpa - base_gpa) / sizeof(pt_element_t);
400
401 r = kvm_read_guest_atomic(vcpu->kvm, base_gpa,
402 gw->prefetch_ptes, sizeof(gw->prefetch_ptes));
403 curr_pte = gw->prefetch_ptes[index];
404 } else
405 r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa,
Avi Kivity39c8c672010-07-13 14:27:08 +0300406 &curr_pte, sizeof(curr_pte));
Xiao Guangrong189be382010-08-22 19:13:33 +0800407
Avi Kivity39c8c672010-07-13 14:27:08 +0300408 return r || curr_pte != gw->ptes[level - 1];
409}
410
Xiao Guangrong189be382010-08-22 19:13:33 +0800411static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
412 u64 *sptep)
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800413{
414 struct kvm_mmu_page *sp;
Xiao Guangrong189be382010-08-22 19:13:33 +0800415 pt_element_t *gptep = gw->prefetch_ptes;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800416 u64 *spte;
Xiao Guangrong189be382010-08-22 19:13:33 +0800417 int i;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800418
419 sp = page_header(__pa(sptep));
420
421 if (sp->role.level > PT_PAGE_TABLE_LEVEL)
422 return;
423
424 if (sp->role.direct)
425 return __direct_pte_prefetch(vcpu, sp, sptep);
426
427 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800428 spte = sp->spt + i;
429
430 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
431 pt_element_t gpte;
432 unsigned pte_access;
433 gfn_t gfn;
434 pfn_t pfn;
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800435
436 if (spte == sptep)
437 continue;
438
Xiao Guangrongc3707952011-07-12 03:28:04 +0800439 if (is_shadow_present_pte(*spte))
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800440 continue;
441
442 gpte = gptep[i];
443
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800444 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800445 continue;
446
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800447 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte,
448 true);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800449 gfn = gpte_to_gfn(gpte);
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800450 pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800451 pte_access & ACC_WRITE_MASK);
Xiao Guangrongd7c55202011-07-12 03:29:38 +0800452 if (mmu_invalid_pfn(pfn)) {
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800453 kvm_release_pfn_clean(pfn);
454 break;
455 }
456
457 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800458 NULL, PT_PAGE_TABLE_LEVEL, gfn,
Xiao Guangrong957ed9e2010-08-22 19:12:48 +0800459 pfn, true, true);
460 }
461}
462
Avi Kivity6aa8b732006-12-10 02:21:36 -0800463/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800464 * Fetch a shadow pte for a specific level in the paging hierarchy.
465 */
466static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
Avi Kivitye7a04c92008-12-25 15:10:50 +0200467 struct guest_walker *gw,
Joerg Roedel7e4e4052009-07-27 16:30:46 +0200468 int user_fault, int write_fault, int hlevel,
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800469 int *emulate, pfn_t pfn, bool map_writable,
Xiao Guangrongfb67e142010-12-07 10:35:25 +0800470 bool prefault)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800471{
Avi Kivitye7a04c92008-12-25 15:10:50 +0200472 unsigned access = gw->pt_access;
Avi Kivity5991b332010-07-13 14:27:10 +0300473 struct kvm_mmu_page *sp = NULL;
Avi Kivity5991b332010-07-13 14:27:10 +0300474 int top_level;
Xiao Guangrong84754cd2010-06-30 16:05:00 +0800475 unsigned direct_access;
Avi Kivity24157aa2010-07-13 14:27:11 +0300476 struct kvm_shadow_walk_iterator it;
Avi Kivityac79c972007-01-05 16:36:40 -0800477
Avi Kivity43a37952009-06-10 14:12:05 +0300478 if (!is_present_gpte(gw->ptes[gw->level - 1]))
Avi Kivityac79c972007-01-05 16:36:40 -0800479 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800480
Xiao Guangrongb36c7a72011-07-12 03:25:19 +0800481 direct_access = gw->pte_access;
Xiao Guangrong84754cd2010-06-30 16:05:00 +0800482
Avi Kivity5991b332010-07-13 14:27:10 +0300483 top_level = vcpu->arch.mmu.root_level;
484 if (top_level == PT32E_ROOT_LEVEL)
485 top_level = PT32_ROOT_LEVEL;
486 /*
487 * Verify that the top-level gpte is still there. Since the page
488 * is a root page, it is either write protected (and cannot be
489 * changed from now on) or it is invalid (in which case, we don't
490 * really care if it changes underneath us after this point).
491 */
492 if (FNAME(gpte_changed)(vcpu, gw, top_level))
493 goto out_gpte_changed;
494
Avi Kivity24157aa2010-07-13 14:27:11 +0300495 for (shadow_walk_init(&it, vcpu, addr);
496 shadow_walk_okay(&it) && it.level > gw->level;
497 shadow_walk_next(&it)) {
Avi Kivity0b3c9332010-07-13 14:27:09 +0300498 gfn_t table_gfn;
499
Xiao Guangronga30f47c2011-09-22 16:58:36 +0800500 clear_sp_write_flooding_count(it.sptep);
Avi Kivity24157aa2010-07-13 14:27:11 +0300501 drop_large_spte(vcpu, it.sptep);
Avi Kivitye7a04c92008-12-25 15:10:50 +0200502
Avi Kivity5991b332010-07-13 14:27:10 +0300503 sp = NULL;
Avi Kivity24157aa2010-07-13 14:27:11 +0300504 if (!is_shadow_present_pte(*it.sptep)) {
505 table_gfn = gw->table_gfn[it.level - 2];
506 sp = kvm_mmu_get_page(vcpu, table_gfn, addr, it.level-1,
507 false, access, it.sptep);
Avi Kivity5991b332010-07-13 14:27:10 +0300508 }
Avi Kivity0b3c9332010-07-13 14:27:09 +0300509
510 /*
511 * Verify that the gpte in the page we've just write
512 * protected is still there.
513 */
Avi Kivity24157aa2010-07-13 14:27:11 +0300514 if (FNAME(gpte_changed)(vcpu, gw, it.level - 1))
Avi Kivity0b3c9332010-07-13 14:27:09 +0300515 goto out_gpte_changed;
Avi Kivitye7a04c92008-12-25 15:10:50 +0200516
Avi Kivity5991b332010-07-13 14:27:10 +0300517 if (sp)
Avi Kivity24157aa2010-07-13 14:27:11 +0300518 link_shadow_page(it.sptep, sp);
Avi Kivitye7a04c92008-12-25 15:10:50 +0200519 }
520
Avi Kivity0b3c9332010-07-13 14:27:09 +0300521 for (;
Avi Kivity24157aa2010-07-13 14:27:11 +0300522 shadow_walk_okay(&it) && it.level > hlevel;
523 shadow_walk_next(&it)) {
Avi Kivity0b3c9332010-07-13 14:27:09 +0300524 gfn_t direct_gfn;
525
Xiao Guangronga30f47c2011-09-22 16:58:36 +0800526 clear_sp_write_flooding_count(it.sptep);
Avi Kivity24157aa2010-07-13 14:27:11 +0300527 validate_direct_spte(vcpu, it.sptep, direct_access);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300528
Avi Kivity24157aa2010-07-13 14:27:11 +0300529 drop_large_spte(vcpu, it.sptep);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300530
Avi Kivity24157aa2010-07-13 14:27:11 +0300531 if (is_shadow_present_pte(*it.sptep))
Avi Kivity0b3c9332010-07-13 14:27:09 +0300532 continue;
533
Avi Kivity24157aa2010-07-13 14:27:11 +0300534 direct_gfn = gw->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300535
Avi Kivity24157aa2010-07-13 14:27:11 +0300536 sp = kvm_mmu_get_page(vcpu, direct_gfn, addr, it.level-1,
537 true, direct_access, it.sptep);
538 link_shadow_page(it.sptep, sp);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300539 }
540
Xiao Guangronga30f47c2011-09-22 16:58:36 +0800541 clear_sp_write_flooding_count(it.sptep);
Xiao Guangrongb36c7a72011-07-12 03:25:19 +0800542 mmu_set_spte(vcpu, it.sptep, access, gw->pte_access,
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800543 user_fault, write_fault, emulate, it.level,
Xiao Guangrongfb67e142010-12-07 10:35:25 +0800544 gw->gfn, pfn, prefault, map_writable);
Xiao Guangrong189be382010-08-22 19:13:33 +0800545 FNAME(pte_prefetch)(vcpu, gw, it.sptep);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300546
Avi Kivity24157aa2010-07-13 14:27:11 +0300547 return it.sptep;
Avi Kivity0b3c9332010-07-13 14:27:09 +0300548
549out_gpte_changed:
Avi Kivity5991b332010-07-13 14:27:10 +0300550 if (sp)
Avi Kivity24157aa2010-07-13 14:27:11 +0300551 kvm_mmu_put_page(sp, it.sptep);
Avi Kivity0b3c9332010-07-13 14:27:09 +0300552 kvm_release_pfn_clean(pfn);
553 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800554}
555
556/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800557 * Page fault handler. There are several causes for a page fault:
558 * - there is no shadow pte for the guest pte
559 * - write access through a shadow pte marked read only so that we can set
560 * the dirty bit
561 * - write access to a shadow pte marked read only so we can update the page
562 * dirty bitmap, when userspace requests it
563 * - mmio access; in this case we will never install a present shadow pte
564 * - normal guest page fault due to the guest pte marked not present, not
565 * writable, or not executable
566 *
Avi Kivitye2dec932007-01-05 16:36:54 -0800567 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
568 * a negative value on error.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800569 */
Gleb Natapov56028d02010-10-17 18:13:42 +0200570static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
Xiao Guangrong78b2c542010-12-07 10:48:06 +0800571 bool prefault)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800572{
573 int write_fault = error_code & PFERR_WRITE_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800574 int user_fault = error_code & PFERR_USER_MASK;
575 struct guest_walker walker;
Avi Kivityd555c332009-06-10 14:24:23 +0300576 u64 *sptep;
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800577 int emulate = 0;
Avi Kivitye2dec932007-01-05 16:36:54 -0800578 int r;
Anthony Liguori35149e22008-04-02 14:46:56 -0500579 pfn_t pfn;
Joerg Roedel7e4e4052009-07-27 16:30:46 +0200580 int level = PT_PAGE_TABLE_LEVEL;
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -0800581 int force_pt_level;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200582 unsigned long mmu_seq;
Marcelo Tosatti612819c2010-10-22 14:18:18 -0200583 bool map_writable;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800584
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800585 pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
Avi Kivity714b93d2007-01-05 16:36:53 -0800586
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800587 if (unlikely(error_code & PFERR_RSVD_MASK))
588 return handle_mmio_page_fault(vcpu, addr, error_code,
589 mmu_is_nested(vcpu));
590
Avi Kivitye2dec932007-01-05 16:36:54 -0800591 r = mmu_topup_memory_caches(vcpu);
592 if (r)
593 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -0800594
Avi Kivity6aa8b732006-12-10 02:21:36 -0800595 /*
Eddie Donga8b876b2009-03-26 15:28:40 +0800596 * Look up the guest pte for the faulting address.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800597 */
Xiao Guangrong33770782010-09-28 17:03:14 +0800598 r = FNAME(walk_addr)(&walker, vcpu, addr, error_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800599
600 /*
601 * The page is not mapped by the guest. Let the guest handle it.
602 */
Avi Kivity7993ba42007-01-26 00:56:41 -0800603 if (!r) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800604 pgprintk("%s: guest page fault\n", __func__);
Xiao Guangronga30f47c2011-09-22 16:58:36 +0800605 if (!prefault)
Xiao Guangrongfb67e142010-12-07 10:35:25 +0800606 inject_page_fault(vcpu, &walker.fault);
Xiao Guangronga30f47c2011-09-22 16:58:36 +0800607
Avi Kivity6aa8b732006-12-10 02:21:36 -0800608 return 0;
609 }
610
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -0800611 if (walker.level >= PT_DIRECTORY_LEVEL)
612 force_pt_level = mapping_level_dirty_bitmap(vcpu, walker.gfn);
613 else
614 force_pt_level = 1;
615 if (!force_pt_level) {
Joerg Roedel7e4e4052009-07-27 16:30:46 +0200616 level = min(walker.level, mapping_level(vcpu, walker.gfn));
617 walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300618 }
Joerg Roedel7e4e4052009-07-27 16:30:46 +0200619
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200620 mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300621 smp_rmb();
Gleb Natapovaf585b92010-10-14 11:22:46 +0200622
Xiao Guangrong78b2c542010-12-07 10:48:06 +0800623 if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault,
Marcelo Tosatti612819c2010-10-22 14:18:18 -0200624 &map_writable))
Gleb Natapovaf585b92010-10-14 11:22:46 +0200625 return 0;
Avi Kivityd7824ff2007-12-30 12:29:05 +0200626
Xiao Guangrongd7c55202011-07-12 03:29:38 +0800627 if (handle_abnormal_pfn(vcpu, mmu_is_nested(vcpu) ? 0 : addr,
628 walker.gfn, pfn, walker.pte_access, &r))
629 return r;
630
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -0500631 spin_lock(&vcpu->kvm->mmu_lock);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200632 if (mmu_notifier_retry(vcpu, mmu_seq))
633 goto out_unlock;
Xiao Guangrongbc32ce22010-08-28 19:22:46 +0800634
Xiao Guangrong0375f7f2011-11-28 20:41:00 +0800635 kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT);
Avi Kivityeb787d12007-12-31 15:27:49 +0200636 kvm_mmu_free_some_pages(vcpu);
Andrea Arcangeli936a5fe2011-01-13 15:46:48 -0800637 if (!force_pt_level)
638 transparent_hugepage_adjust(vcpu, &walker.gfn, &pfn, &level);
Avi Kivityd555c332009-06-10 14:24:23 +0300639 sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800640 level, &emulate, pfn, map_writable, prefault);
Andi Kleena24e8092010-06-10 13:10:55 +0200641 (void)sptep;
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800642 pgprintk("%s: shadow pte %p %llx emulate %d\n", __func__,
643 sptep, *sptep, emulate);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800644
Avi Kivity1165f5f2007-04-19 17:27:43 +0300645 ++vcpu->stat.pf_fixed;
Xiao Guangrong0375f7f2011-11-28 20:41:00 +0800646 kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -0500647 spin_unlock(&vcpu->kvm->mmu_lock);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800648
Xiao Guangrongb90a0e62011-07-12 03:25:56 +0800649 return emulate;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200650
651out_unlock:
652 spin_unlock(&vcpu->kvm->mmu_lock);
653 kvm_release_pfn_clean(pfn);
654 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800655}
656
Xiao Guangrong505aef82011-09-22 16:56:06 +0800657static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp)
658{
659 int offset = 0;
660
661 WARN_ON(sp->role.level != 1);
662
663 if (PTTYPE == 32)
664 offset = sp->role.quadrant << PT64_LEVEL_BITS;
665
666 return gfn_to_gpa(sp->gfn) + offset * sizeof(pt_element_t);
667}
668
Marcelo Tosattia7052892008-09-23 13:18:35 -0300669static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
670{
Avi Kivitya4619302008-12-25 15:19:00 +0200671 struct kvm_shadow_walk_iterator iterator;
Xiao Guangrongf78978a2010-05-15 18:53:35 +0800672 struct kvm_mmu_page *sp;
Avi Kivitya4619302008-12-25 15:19:00 +0200673 int level;
674 u64 *sptep;
Marcelo Tosattia7052892008-09-23 13:18:35 -0300675
Xiao Guangrongbebb1062011-07-12 03:23:20 +0800676 vcpu_clear_mmio_info(vcpu, gva);
677
Xiao Guangrongf57f2ef2011-09-22 16:56:39 +0800678 /*
679 * No need to check return value here, rmap_can_add() can
680 * help us to skip pte prefetch later.
681 */
682 mmu_topup_memory_caches(vcpu);
Avi Kivitya4619302008-12-25 15:19:00 +0200683
Xiao Guangrongf57f2ef2011-09-22 16:56:39 +0800684 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivitya4619302008-12-25 15:19:00 +0200685 for_each_shadow_entry(vcpu, gva, iterator) {
686 level = iterator.level;
687 sptep = iterator.sptep;
688
Xiao Guangrongf78978a2010-05-15 18:53:35 +0800689 sp = page_header(__pa(sptep));
Xiao Guangrong884a0ff2010-04-28 11:55:15 +0800690 if (is_last_spte(*sptep, level)) {
Xiao Guangrongf57f2ef2011-09-22 16:56:39 +0800691 pt_element_t gpte;
692 gpa_t pte_gpa;
693
Xiao Guangrongf78978a2010-05-15 18:53:35 +0800694 if (!sp->unsync)
695 break;
696
Xiao Guangrong505aef82011-09-22 16:56:06 +0800697 pte_gpa = FNAME(get_level1_sp_gpa)(sp);
Avi Kivity08e850c2010-03-15 13:59:57 +0200698 pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t);
Avi Kivitya4619302008-12-25 15:19:00 +0200699
Xiao Guangrong505aef82011-09-22 16:56:06 +0800700 if (mmu_page_zap_pte(vcpu->kvm, sp, sptep))
701 kvm_flush_remote_tlbs(vcpu->kvm);
Xiao Guangrongf57f2ef2011-09-22 16:56:39 +0800702
703 if (!rmap_can_add(vcpu))
704 break;
705
706 if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte,
707 sizeof(pt_element_t)))
708 break;
709
710 FNAME(update_pte)(vcpu, sp, sptep, &gpte);
Avi Kivitya4619302008-12-25 15:19:00 +0200711 }
712
Xiao Guangrongf78978a2010-05-15 18:53:35 +0800713 if (!is_shadow_present_pte(*sptep) || !sp->unsync_children)
Avi Kivitya4619302008-12-25 15:19:00 +0200714 break;
715 }
Marcelo Tosattiad218f82008-12-01 22:32:05 -0200716 spin_unlock(&vcpu->kvm->mmu_lock);
Marcelo Tosattia7052892008-09-23 13:18:35 -0300717}
718
Gleb Natapov1871c602010-02-10 14:21:32 +0200719static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
Avi Kivityab9ae312010-11-22 17:53:26 +0200720 struct x86_exception *exception)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800721{
722 struct guest_walker walker;
Avi Kivitye119d112007-02-12 00:54:36 -0800723 gpa_t gpa = UNMAPPED_GVA;
724 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800725
Xiao Guangrong33770782010-09-28 17:03:14 +0800726 r = FNAME(walk_addr)(&walker, vcpu, vaddr, access);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800727
Avi Kivitye119d112007-02-12 00:54:36 -0800728 if (r) {
Avi Kivity1755fbc2007-11-21 14:44:45 +0200729 gpa = gfn_to_gpa(walker.gfn);
Avi Kivitye119d112007-02-12 00:54:36 -0800730 gpa |= vaddr & ~PAGE_MASK;
Avi Kivity8c28d032010-11-22 17:53:27 +0200731 } else if (exception)
732 *exception = walker.fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800733
734 return gpa;
735}
736
Joerg Roedel6539e732010-09-10 17:30:50 +0200737static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
Avi Kivityab9ae312010-11-22 17:53:26 +0200738 u32 access,
739 struct x86_exception *exception)
Joerg Roedel6539e732010-09-10 17:30:50 +0200740{
741 struct guest_walker walker;
742 gpa_t gpa = UNMAPPED_GVA;
743 int r;
744
Xiao Guangrong33770782010-09-28 17:03:14 +0800745 r = FNAME(walk_addr_nested)(&walker, vcpu, vaddr, access);
Joerg Roedel6539e732010-09-10 17:30:50 +0200746
747 if (r) {
748 gpa = gfn_to_gpa(walker.gfn);
749 gpa |= vaddr & ~PAGE_MASK;
Avi Kivity8c28d032010-11-22 17:53:27 +0200750 } else if (exception)
751 *exception = walker.fault;
Joerg Roedel6539e732010-09-10 17:30:50 +0200752
753 return gpa;
754}
755
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300756/*
757 * Using the cached information from sp->gfns is safe because:
758 * - The spte has a reference to the struct page, so the pfn for a given gfn
759 * can't change unless all sptes pointing to it are nuked first.
Xiao Guangronga4ee1ca2010-11-23 11:13:00 +0800760 *
761 * Note:
762 * We should flush all tlbs if spte is dropped even though guest is
763 * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
764 * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
765 * used by guest then tlbs are not flushed, so guest is allowed to access the
766 * freed pages.
767 * And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300768 */
Xiao Guangronga4a8e6f2010-11-19 17:04:03 +0800769static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300770{
Xiao Guangrong505aef82011-09-22 16:56:06 +0800771 int i, nr_present = 0;
Lai Jiangshan9bdbba12010-11-19 17:03:22 +0800772 bool host_writable;
Gui Jianfeng51fb60d2010-04-16 17:16:40 +0800773 gpa_t first_pte_gpa;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300774
Lai Jiangshan2032a932010-05-26 16:49:59 +0800775 /* direct kvm_mmu_page can not be unsync. */
776 BUG_ON(sp->role.direct);
777
Xiao Guangrong505aef82011-09-22 16:56:06 +0800778 first_pte_gpa = FNAME(get_level1_sp_gpa)(sp);
Gui Jianfeng51fb60d2010-04-16 17:16:40 +0800779
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300780 for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
781 unsigned pte_access;
782 pt_element_t gpte;
783 gpa_t pte_gpa;
Xiao Guangrongf55c3f42010-05-13 10:08:08 +0800784 gfn_t gfn;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300785
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800786 if (!sp->spt[i])
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300787 continue;
788
Gui Jianfeng51fb60d2010-04-16 17:16:40 +0800789 pte_gpa = first_pte_gpa + i * sizeof(pt_element_t);
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300790
791 if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte,
792 sizeof(pt_element_t)))
793 return -EINVAL;
794
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800795 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) {
Xiao Guangronga4ee1ca2010-11-23 11:13:00 +0800796 vcpu->kvm->tlbs_dirty++;
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800797 continue;
798 }
799
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800800 gfn = gpte_to_gfn(gpte);
801 pte_access = sp->role.access;
802 pte_access &= FNAME(gpte_access)(vcpu, gpte, true);
803
804 if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present))
805 continue;
806
Xiao Guangrong407c61c2010-11-23 11:08:42 +0800807 if (gfn != sp->gfns[i]) {
Xiao Guangrongc3707952011-07-12 03:28:04 +0800808 drop_spte(vcpu->kvm, &sp->spt[i]);
Xiao Guangronga4ee1ca2010-11-23 11:13:00 +0800809 vcpu->kvm->tlbs_dirty++;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300810 continue;
811 }
812
813 nr_present++;
Xiao Guangrongce88dec2011-07-12 03:33:44 +0800814
Xiao Guangrongf8e453b2010-12-23 16:09:29 +0800815 host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE;
816
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300817 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
Xiao Guangrong640d9b02011-07-12 03:24:39 +0800818 PT_PAGE_TABLE_LEVEL, gfn,
Izik Eidus14032832009-09-23 21:47:17 +0300819 spte_to_pfn(sp->spt[i]), true, false,
Lai Jiangshan9bdbba12010-11-19 17:03:22 +0800820 host_writable);
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300821 }
822
823 return !nr_present;
824}
825
Avi Kivity6aa8b732006-12-10 02:21:36 -0800826#undef pt_element_t
827#undef guest_walker
828#undef FNAME
829#undef PT_BASE_ADDR_MASK
830#undef PT_INDEX
Joerg Roedele04da982009-07-27 16:30:45 +0200831#undef PT_LVL_ADDR_MASK
832#undef PT_LVL_OFFSET_MASK
Avi Kivityc7addb92007-09-16 18:58:32 +0200833#undef PT_LEVEL_BITS
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800834#undef PT_MAX_FULL_LEVELS
Avi Kivity5fb07dd2007-11-21 12:35:07 +0200835#undef gpte_to_gfn
Joerg Roedele04da982009-07-27 16:30:45 +0200836#undef gpte_to_gfn_lvl
Marcelo Tosattib3e4e632007-12-07 07:56:58 -0500837#undef CMPXCHG