blob: 7246b60afb96a19d32e2d158749bf10fad553470 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
Avi Kivity6aa8b732006-12-10 02:21:36 -080019
20#include "vmx.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080021#include "mmu.h"
Avi Kivity6aa8b732006-12-10 02:21:36 -080022
Avi Kivityedf88412007-12-16 11:02:48 +020023#include <linux/kvm_host.h>
Avi Kivitye4956062007-06-28 14:15:57 -040024#include <linux/types.h>
25#include <linux/string.h>
26#include <linux/mm.h>
27#include <linux/highmem.h>
28#include <linux/module.h>
Izik Eidus448353c2007-11-26 14:08:14 +020029#include <linux/swap.h>
Marcelo Tosatti05da4552008-02-23 11:44:30 -030030#include <linux/hugetlb.h>
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050031#include <linux/compiler.h>
Avi Kivitye4956062007-06-28 14:15:57 -040032
33#include <asm/page.h>
34#include <asm/cmpxchg.h>
Avi Kivity4e542372007-11-21 14:08:40 +020035#include <asm/io.h>
Avi Kivitye4956062007-06-28 14:15:57 -040036
Joerg Roedel18552672008-02-07 13:47:41 +010037/*
38 * When setting this variable to true it enables Two-Dimensional-Paging
39 * where the hardware walks 2 page tables:
40 * 1. the guest-virtual to guest-physical
41 * 2. while doing 1. it walks guest-physical to host-physical
42 * If the hardware supports that we don't need to do shadow paging.
43 */
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050044bool tdp_enabled = false;
Joerg Roedel18552672008-02-07 13:47:41 +010045
Avi Kivity37a7d8b2007-01-05 16:36:56 -080046#undef MMU_DEBUG
47
48#undef AUDIT
49
50#ifdef AUDIT
51static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
52#else
53static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
54#endif
55
56#ifdef MMU_DEBUG
57
58#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
59#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
60
61#else
62
63#define pgprintk(x...) do { } while (0)
64#define rmap_printk(x...) do { } while (0)
65
66#endif
67
68#if defined(MMU_DEBUG) || defined(AUDIT)
69static int dbg = 1;
70#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080071
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080072#ifndef MMU_DEBUG
73#define ASSERT(x) do { } while (0)
74#else
Avi Kivity6aa8b732006-12-10 02:21:36 -080075#define ASSERT(x) \
76 if (!(x)) { \
77 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
78 __FILE__, __LINE__, #x); \
79 }
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080080#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080081
Avi Kivity6aa8b732006-12-10 02:21:36 -080082#define PT_FIRST_AVAIL_BITS_SHIFT 9
83#define PT64_SECOND_AVAIL_BITS_SHIFT 52
84
Avi Kivity6aa8b732006-12-10 02:21:36 -080085#define VALID_PAGE(x) ((x) != INVALID_PAGE)
86
87#define PT64_LEVEL_BITS 9
88
89#define PT64_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -040090 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080091
92#define PT64_LEVEL_MASK(level) \
93 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
94
95#define PT64_INDEX(address, level)\
96 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
97
98
99#define PT32_LEVEL_BITS 10
100
101#define PT32_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -0400102 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800103
104#define PT32_LEVEL_MASK(level) \
105 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
106
107#define PT32_INDEX(address, level)\
108 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
109
110
Avi Kivity27aba762007-03-09 13:04:31 +0200111#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800112#define PT64_DIR_BASE_ADDR_MASK \
113 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
114
115#define PT32_BASE_ADDR_MASK PAGE_MASK
116#define PT32_DIR_BASE_ADDR_MASK \
117 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
118
Avi Kivity79539ce2007-11-21 02:06:21 +0200119#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
120 | PT64_NX_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800121
122#define PFERR_PRESENT_MASK (1U << 0)
123#define PFERR_WRITE_MASK (1U << 1)
124#define PFERR_USER_MASK (1U << 2)
Avi Kivity73b10872007-01-26 00:56:41 -0800125#define PFERR_FETCH_MASK (1U << 4)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800126
Avi Kivity6aa8b732006-12-10 02:21:36 -0800127#define PT_DIRECTORY_LEVEL 2
128#define PT_PAGE_TABLE_LEVEL 1
129
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800130#define RMAP_EXT 4
131
Avi Kivityfe135d22007-12-09 16:15:46 +0200132#define ACC_EXEC_MASK 1
133#define ACC_WRITE_MASK PT_WRITABLE_MASK
134#define ACC_USER_MASK PT_USER_MASK
135#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
136
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -0500137struct kvm_pv_mmu_op_buffer {
138 void *ptr;
139 unsigned len;
140 unsigned processed;
141 char buf[512] __aligned(sizeof(long));
142};
143
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800144struct kvm_rmap_desc {
145 u64 *shadow_ptes[RMAP_EXT];
146 struct kvm_rmap_desc *more;
147};
148
Avi Kivityb5a33a72007-04-15 16:31:09 +0300149static struct kmem_cache *pte_chain_cache;
150static struct kmem_cache *rmap_desc_cache;
Avi Kivityd3d25b02007-05-30 12:34:53 +0300151static struct kmem_cache *mmu_page_header_cache;
Avi Kivityb5a33a72007-04-15 16:31:09 +0300152
Avi Kivityc7addb92007-09-16 18:58:32 +0200153static u64 __read_mostly shadow_trap_nonpresent_pte;
154static u64 __read_mostly shadow_notrap_nonpresent_pte;
Sheng Yang7b523452008-04-25 21:13:50 +0800155static u64 __read_mostly shadow_base_present_pte;
156static u64 __read_mostly shadow_nx_mask;
157static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
158static u64 __read_mostly shadow_user_mask;
159static u64 __read_mostly shadow_accessed_mask;
160static u64 __read_mostly shadow_dirty_mask;
Avi Kivityc7addb92007-09-16 18:58:32 +0200161
162void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
163{
164 shadow_trap_nonpresent_pte = trap_pte;
165 shadow_notrap_nonpresent_pte = notrap_pte;
166}
167EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
168
Sheng Yang7b523452008-04-25 21:13:50 +0800169void kvm_mmu_set_base_ptes(u64 base_pte)
170{
171 shadow_base_present_pte = base_pte;
172}
173EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
174
175void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
176 u64 dirty_mask, u64 nx_mask, u64 x_mask)
177{
178 shadow_user_mask = user_mask;
179 shadow_accessed_mask = accessed_mask;
180 shadow_dirty_mask = dirty_mask;
181 shadow_nx_mask = nx_mask;
182 shadow_x_mask = x_mask;
183}
184EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
185
Avi Kivity6aa8b732006-12-10 02:21:36 -0800186static int is_write_protection(struct kvm_vcpu *vcpu)
187{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800188 return vcpu->arch.cr0 & X86_CR0_WP;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800189}
190
191static int is_cpuid_PSE36(void)
192{
193 return 1;
194}
195
Avi Kivity73b10872007-01-26 00:56:41 -0800196static int is_nx(struct kvm_vcpu *vcpu)
197{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800198 return vcpu->arch.shadow_efer & EFER_NX;
Avi Kivity73b10872007-01-26 00:56:41 -0800199}
200
Avi Kivity6aa8b732006-12-10 02:21:36 -0800201static int is_present_pte(unsigned long pte)
202{
203 return pte & PT_PRESENT_MASK;
204}
205
Avi Kivityc7addb92007-09-16 18:58:32 +0200206static int is_shadow_present_pte(u64 pte)
207{
Avi Kivityc7addb92007-09-16 18:58:32 +0200208 return pte != shadow_trap_nonpresent_pte
209 && pte != shadow_notrap_nonpresent_pte;
210}
211
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300212static int is_large_pte(u64 pte)
213{
214 return pte & PT_PAGE_SIZE_MASK;
215}
216
Avi Kivity6aa8b732006-12-10 02:21:36 -0800217static int is_writeble_pte(unsigned long pte)
218{
219 return pte & PT_WRITABLE_MASK;
220}
221
Avi Kivitye3c5e7ec2007-10-11 12:32:30 +0200222static int is_dirty_pte(unsigned long pte)
223{
Sheng Yang7b523452008-04-25 21:13:50 +0800224 return pte & shadow_dirty_mask;
Avi Kivitye3c5e7ec2007-10-11 12:32:30 +0200225}
226
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800227static int is_rmap_pte(u64 pte)
228{
Avi Kivity4b1a80f2008-03-23 12:18:19 +0200229 return is_shadow_present_pte(pte);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800230}
231
Anthony Liguori35149e22008-04-02 14:46:56 -0500232static pfn_t spte_to_pfn(u64 pte)
Avi Kivity0b49ea82008-03-23 15:06:23 +0200233{
Anthony Liguori35149e22008-04-02 14:46:56 -0500234 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
Avi Kivity0b49ea82008-03-23 15:06:23 +0200235}
236
Avi Kivityda928522007-11-21 13:54:47 +0200237static gfn_t pse36_gfn_delta(u32 gpte)
238{
239 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
240
241 return (gpte & PT32_DIR_PSE36_MASK) << shift;
242}
243
Avi Kivitye663ee62007-05-31 15:46:04 +0300244static void set_shadow_pte(u64 *sptep, u64 spte)
245{
246#ifdef CONFIG_X86_64
247 set_64bit((unsigned long *)sptep, spte);
248#else
249 set_64bit((unsigned long long *)sptep, spte);
250#endif
251}
252
Avi Kivitye2dec932007-01-05 16:36:54 -0800253static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300254 struct kmem_cache *base_cache, int min)
Avi Kivity714b93d2007-01-05 16:36:53 -0800255{
256 void *obj;
257
258 if (cache->nobjs >= min)
Avi Kivitye2dec932007-01-05 16:36:54 -0800259 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800260 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300261 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
Avi Kivity714b93d2007-01-05 16:36:53 -0800262 if (!obj)
Avi Kivitye2dec932007-01-05 16:36:54 -0800263 return -ENOMEM;
Avi Kivity714b93d2007-01-05 16:36:53 -0800264 cache->objects[cache->nobjs++] = obj;
265 }
Avi Kivitye2dec932007-01-05 16:36:54 -0800266 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800267}
268
269static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
270{
271 while (mc->nobjs)
272 kfree(mc->objects[--mc->nobjs]);
273}
274
Avi Kivityc1158e62007-07-20 08:18:27 +0300275static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300276 int min)
Avi Kivityc1158e62007-07-20 08:18:27 +0300277{
278 struct page *page;
279
280 if (cache->nobjs >= min)
281 return 0;
282 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300283 page = alloc_page(GFP_KERNEL);
Avi Kivityc1158e62007-07-20 08:18:27 +0300284 if (!page)
285 return -ENOMEM;
286 set_page_private(page, 0);
287 cache->objects[cache->nobjs++] = page_address(page);
288 }
289 return 0;
290}
291
292static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
293{
294 while (mc->nobjs)
Avi Kivityc4d198d2007-07-21 09:06:46 +0300295 free_page((unsigned long)mc->objects[--mc->nobjs]);
Avi Kivityc1158e62007-07-20 08:18:27 +0300296}
297
Avi Kivity8c438502007-04-16 11:53:17 +0300298static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
299{
300 int r;
301
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800302 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300303 pte_chain_cache, 4);
304 if (r)
305 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800306 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300307 rmap_desc_cache, 1);
308 if (r)
309 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800310 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300311 if (r)
312 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800313 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300314 mmu_page_header_cache, 4);
315out:
Avi Kivity8c438502007-04-16 11:53:17 +0300316 return r;
317}
318
Avi Kivity714b93d2007-01-05 16:36:53 -0800319static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
320{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800321 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
322 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
323 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
324 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800325}
326
327static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
328 size_t size)
329{
330 void *p;
331
332 BUG_ON(!mc->nobjs);
333 p = mc->objects[--mc->nobjs];
334 memset(p, 0, size);
335 return p;
336}
337
Avi Kivity714b93d2007-01-05 16:36:53 -0800338static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
339{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800340 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
Avi Kivity714b93d2007-01-05 16:36:53 -0800341 sizeof(struct kvm_pte_chain));
342}
343
Avi Kivity90cb0522007-07-17 13:04:56 +0300344static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
Avi Kivity714b93d2007-01-05 16:36:53 -0800345{
Avi Kivity90cb0522007-07-17 13:04:56 +0300346 kfree(pc);
Avi Kivity714b93d2007-01-05 16:36:53 -0800347}
348
349static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
350{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800351 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
Avi Kivity714b93d2007-01-05 16:36:53 -0800352 sizeof(struct kvm_rmap_desc));
353}
354
Avi Kivity90cb0522007-07-17 13:04:56 +0300355static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
Avi Kivity714b93d2007-01-05 16:36:53 -0800356{
Avi Kivity90cb0522007-07-17 13:04:56 +0300357 kfree(rd);
Avi Kivity714b93d2007-01-05 16:36:53 -0800358}
359
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800360/*
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300361 * Return the pointer to the largepage write count for a given
362 * gfn, handling slots that are not large page aligned.
363 */
364static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
365{
366 unsigned long idx;
367
368 idx = (gfn / KVM_PAGES_PER_HPAGE) -
369 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
370 return &slot->lpage_info[idx].write_count;
371}
372
373static void account_shadowed(struct kvm *kvm, gfn_t gfn)
374{
375 int *write_count;
376
377 write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
378 *write_count += 1;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300379}
380
381static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
382{
383 int *write_count;
384
385 write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
386 *write_count -= 1;
387 WARN_ON(*write_count < 0);
388}
389
390static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
391{
392 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
393 int *largepage_idx;
394
395 if (slot) {
396 largepage_idx = slot_largepage_idx(gfn, slot);
397 return *largepage_idx;
398 }
399
400 return 1;
401}
402
403static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
404{
405 struct vm_area_struct *vma;
406 unsigned long addr;
407
408 addr = gfn_to_hva(kvm, gfn);
409 if (kvm_is_error_hva(addr))
410 return 0;
411
412 vma = find_vma(current->mm, addr);
413 if (vma && is_vm_hugetlb_page(vma))
414 return 1;
415
416 return 0;
417}
418
419static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
420{
421 struct kvm_memory_slot *slot;
422
423 if (has_wrprotected_page(vcpu->kvm, large_gfn))
424 return 0;
425
426 if (!host_largepage_backed(vcpu->kvm, large_gfn))
427 return 0;
428
429 slot = gfn_to_memslot(vcpu->kvm, large_gfn);
430 if (slot && slot->dirty_bitmap)
431 return 0;
432
433 return 1;
434}
435
436/*
Izik Eidus290fc382007-09-27 14:11:22 +0200437 * Take gfn and return the reverse mapping to it.
438 * Note: gfn must be unaliased before this function get called
439 */
440
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300441static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
Izik Eidus290fc382007-09-27 14:11:22 +0200442{
443 struct kvm_memory_slot *slot;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300444 unsigned long idx;
Izik Eidus290fc382007-09-27 14:11:22 +0200445
446 slot = gfn_to_memslot(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300447 if (!lpage)
448 return &slot->rmap[gfn - slot->base_gfn];
449
450 idx = (gfn / KVM_PAGES_PER_HPAGE) -
451 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
452
453 return &slot->lpage_info[idx].rmap_pde;
Izik Eidus290fc382007-09-27 14:11:22 +0200454}
455
456/*
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800457 * Reverse mapping data structures:
458 *
Izik Eidus290fc382007-09-27 14:11:22 +0200459 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
460 * that points to page_address(page).
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800461 *
Izik Eidus290fc382007-09-27 14:11:22 +0200462 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
463 * containing more mappings.
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800464 */
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300465static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800466{
Avi Kivity4db35312007-11-21 15:28:32 +0200467 struct kvm_mmu_page *sp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800468 struct kvm_rmap_desc *desc;
Izik Eidus290fc382007-09-27 14:11:22 +0200469 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800470 int i;
471
472 if (!is_rmap_pte(*spte))
473 return;
Izik Eidus290fc382007-09-27 14:11:22 +0200474 gfn = unalias_gfn(vcpu->kvm, gfn);
Avi Kivity4db35312007-11-21 15:28:32 +0200475 sp = page_header(__pa(spte));
476 sp->gfns[spte - sp->spt] = gfn;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300477 rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
Izik Eidus290fc382007-09-27 14:11:22 +0200478 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800479 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200480 *rmapp = (unsigned long)spte;
481 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800482 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
Avi Kivity714b93d2007-01-05 16:36:53 -0800483 desc = mmu_alloc_rmap_desc(vcpu);
Izik Eidus290fc382007-09-27 14:11:22 +0200484 desc->shadow_ptes[0] = (u64 *)*rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800485 desc->shadow_ptes[1] = spte;
Izik Eidus290fc382007-09-27 14:11:22 +0200486 *rmapp = (unsigned long)desc | 1;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800487 } else {
488 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200489 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800490 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
491 desc = desc->more;
492 if (desc->shadow_ptes[RMAP_EXT-1]) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800493 desc->more = mmu_alloc_rmap_desc(vcpu);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800494 desc = desc->more;
495 }
496 for (i = 0; desc->shadow_ptes[i]; ++i)
497 ;
498 desc->shadow_ptes[i] = spte;
499 }
500}
501
Izik Eidus290fc382007-09-27 14:11:22 +0200502static void rmap_desc_remove_entry(unsigned long *rmapp,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800503 struct kvm_rmap_desc *desc,
504 int i,
505 struct kvm_rmap_desc *prev_desc)
506{
507 int j;
508
509 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
510 ;
511 desc->shadow_ptes[i] = desc->shadow_ptes[j];
Al Viro11718b4d2007-02-09 16:39:20 +0000512 desc->shadow_ptes[j] = NULL;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800513 if (j != 0)
514 return;
515 if (!prev_desc && !desc->more)
Izik Eidus290fc382007-09-27 14:11:22 +0200516 *rmapp = (unsigned long)desc->shadow_ptes[0];
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800517 else
518 if (prev_desc)
519 prev_desc->more = desc->more;
520 else
Izik Eidus290fc382007-09-27 14:11:22 +0200521 *rmapp = (unsigned long)desc->more | 1;
Avi Kivity90cb0522007-07-17 13:04:56 +0300522 mmu_free_rmap_desc(desc);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800523}
524
Izik Eidus290fc382007-09-27 14:11:22 +0200525static void rmap_remove(struct kvm *kvm, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800526{
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800527 struct kvm_rmap_desc *desc;
528 struct kvm_rmap_desc *prev_desc;
Avi Kivity4db35312007-11-21 15:28:32 +0200529 struct kvm_mmu_page *sp;
Anthony Liguori35149e22008-04-02 14:46:56 -0500530 pfn_t pfn;
Izik Eidus290fc382007-09-27 14:11:22 +0200531 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800532 int i;
533
534 if (!is_rmap_pte(*spte))
535 return;
Avi Kivity4db35312007-11-21 15:28:32 +0200536 sp = page_header(__pa(spte));
Anthony Liguori35149e22008-04-02 14:46:56 -0500537 pfn = spte_to_pfn(*spte);
Sheng Yang7b523452008-04-25 21:13:50 +0800538 if (*spte & shadow_accessed_mask)
Anthony Liguori35149e22008-04-02 14:46:56 -0500539 kvm_set_pfn_accessed(pfn);
Izik Eidusb4231d62007-11-20 11:49:33 +0200540 if (is_writeble_pte(*spte))
Anthony Liguori35149e22008-04-02 14:46:56 -0500541 kvm_release_pfn_dirty(pfn);
Izik Eidusb4231d62007-11-20 11:49:33 +0200542 else
Anthony Liguori35149e22008-04-02 14:46:56 -0500543 kvm_release_pfn_clean(pfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300544 rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
Izik Eidus290fc382007-09-27 14:11:22 +0200545 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800546 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
547 BUG();
Izik Eidus290fc382007-09-27 14:11:22 +0200548 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800549 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200550 if ((u64 *)*rmapp != spte) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800551 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
552 spte, *spte);
553 BUG();
554 }
Izik Eidus290fc382007-09-27 14:11:22 +0200555 *rmapp = 0;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800556 } else {
557 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200558 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800559 prev_desc = NULL;
560 while (desc) {
561 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
562 if (desc->shadow_ptes[i] == spte) {
Izik Eidus290fc382007-09-27 14:11:22 +0200563 rmap_desc_remove_entry(rmapp,
Avi Kivity714b93d2007-01-05 16:36:53 -0800564 desc, i,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800565 prev_desc);
566 return;
567 }
568 prev_desc = desc;
569 desc = desc->more;
570 }
571 BUG();
572 }
573}
574
Izik Eidus98348e92007-10-16 14:42:30 +0200575static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
Avi Kivity374cbac2007-01-05 16:36:43 -0800576{
Avi Kivity374cbac2007-01-05 16:36:43 -0800577 struct kvm_rmap_desc *desc;
Izik Eidus98348e92007-10-16 14:42:30 +0200578 struct kvm_rmap_desc *prev_desc;
579 u64 *prev_spte;
580 int i;
581
582 if (!*rmapp)
583 return NULL;
584 else if (!(*rmapp & 1)) {
585 if (!spte)
586 return (u64 *)*rmapp;
587 return NULL;
588 }
589 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
590 prev_desc = NULL;
591 prev_spte = NULL;
592 while (desc) {
593 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
594 if (prev_spte == spte)
595 return desc->shadow_ptes[i];
596 prev_spte = desc->shadow_ptes[i];
597 }
598 desc = desc->more;
599 }
600 return NULL;
601}
602
603static void rmap_write_protect(struct kvm *kvm, u64 gfn)
604{
Izik Eidus290fc382007-09-27 14:11:22 +0200605 unsigned long *rmapp;
Avi Kivity374cbac2007-01-05 16:36:43 -0800606 u64 *spte;
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800607 int write_protected = 0;
Avi Kivity374cbac2007-01-05 16:36:43 -0800608
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500609 gfn = unalias_gfn(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300610 rmapp = gfn_to_rmap(kvm, gfn, 0);
Avi Kivity374cbac2007-01-05 16:36:43 -0800611
Izik Eidus98348e92007-10-16 14:42:30 +0200612 spte = rmap_next(kvm, rmapp, NULL);
613 while (spte) {
Avi Kivity374cbac2007-01-05 16:36:43 -0800614 BUG_ON(!spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800615 BUG_ON(!(*spte & PT_PRESENT_MASK));
Avi Kivity374cbac2007-01-05 16:36:43 -0800616 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800617 if (is_writeble_pte(*spte)) {
Izik Eidus9647c142007-10-16 14:43:46 +0200618 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800619 write_protected = 1;
620 }
Izik Eidus9647c142007-10-16 14:43:46 +0200621 spte = rmap_next(kvm, rmapp, spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800622 }
Izik Eidus855149a2008-03-20 18:17:24 +0200623 if (write_protected) {
Anthony Liguori35149e22008-04-02 14:46:56 -0500624 pfn_t pfn;
Izik Eidus855149a2008-03-20 18:17:24 +0200625
626 spte = rmap_next(kvm, rmapp, NULL);
Anthony Liguori35149e22008-04-02 14:46:56 -0500627 pfn = spte_to_pfn(*spte);
628 kvm_set_pfn_dirty(pfn);
Izik Eidus855149a2008-03-20 18:17:24 +0200629 }
630
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300631 /* check for huge page mappings */
632 rmapp = gfn_to_rmap(kvm, gfn, 1);
633 spte = rmap_next(kvm, rmapp, NULL);
634 while (spte) {
635 BUG_ON(!spte);
636 BUG_ON(!(*spte & PT_PRESENT_MASK));
637 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
638 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
639 if (is_writeble_pte(*spte)) {
640 rmap_remove(kvm, spte);
641 --kvm->stat.lpages;
642 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
643 write_protected = 1;
644 }
645 spte = rmap_next(kvm, rmapp, spte);
646 }
647
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800648 if (write_protected)
649 kvm_flush_remote_tlbs(kvm);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300650
651 account_shadowed(kvm, gfn);
Avi Kivity374cbac2007-01-05 16:36:43 -0800652}
653
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800654#ifdef MMU_DEBUG
Avi Kivity47ad8e62007-05-06 15:50:58 +0300655static int is_empty_shadow_page(u64 *spt)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800656{
Avi Kivity139bdb22007-01-05 16:36:50 -0800657 u64 *pos;
658 u64 *end;
659
Avi Kivity47ad8e62007-05-06 15:50:58 +0300660 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
Avi Kivityd196e342008-01-24 11:44:11 +0200661 if (*pos != shadow_trap_nonpresent_pte) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800662 printk(KERN_ERR "%s: %p %llx\n", __func__,
Avi Kivity139bdb22007-01-05 16:36:50 -0800663 pos, *pos);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800664 return 0;
Avi Kivity139bdb22007-01-05 16:36:50 -0800665 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800666 return 1;
667}
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800668#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800669
Avi Kivity4db35312007-11-21 15:28:32 +0200670static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivity260746c2007-01-05 16:36:49 -0800671{
Avi Kivity4db35312007-11-21 15:28:32 +0200672 ASSERT(is_empty_shadow_page(sp->spt));
673 list_del(&sp->link);
674 __free_page(virt_to_page(sp->spt));
675 __free_page(virt_to_page(sp->gfns));
676 kfree(sp);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800677 ++kvm->arch.n_free_mmu_pages;
Avi Kivity260746c2007-01-05 16:36:49 -0800678}
679
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800680static unsigned kvm_page_table_hashfn(gfn_t gfn)
681{
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200682 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800683}
684
Avi Kivity25c0de22007-01-05 16:36:42 -0800685static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
686 u64 *parent_pte)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800687{
Avi Kivity4db35312007-11-21 15:28:32 +0200688 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800689
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800690 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
691 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
692 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
Avi Kivity4db35312007-11-21 15:28:32 +0200693 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800694 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
Avi Kivity4db35312007-11-21 15:28:32 +0200695 ASSERT(is_empty_shadow_page(sp->spt));
696 sp->slot_bitmap = 0;
697 sp->multimapped = 0;
698 sp->parent_pte = parent_pte;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800699 --vcpu->kvm->arch.n_free_mmu_pages;
Avi Kivity4db35312007-11-21 15:28:32 +0200700 return sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800701}
702
Avi Kivity714b93d2007-01-05 16:36:53 -0800703static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +0200704 struct kvm_mmu_page *sp, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800705{
706 struct kvm_pte_chain *pte_chain;
707 struct hlist_node *node;
708 int i;
709
710 if (!parent_pte)
711 return;
Avi Kivity4db35312007-11-21 15:28:32 +0200712 if (!sp->multimapped) {
713 u64 *old = sp->parent_pte;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800714
715 if (!old) {
Avi Kivity4db35312007-11-21 15:28:32 +0200716 sp->parent_pte = parent_pte;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800717 return;
718 }
Avi Kivity4db35312007-11-21 15:28:32 +0200719 sp->multimapped = 1;
Avi Kivity714b93d2007-01-05 16:36:53 -0800720 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivity4db35312007-11-21 15:28:32 +0200721 INIT_HLIST_HEAD(&sp->parent_ptes);
722 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800723 pte_chain->parent_ptes[0] = old;
724 }
Avi Kivity4db35312007-11-21 15:28:32 +0200725 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800726 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
727 continue;
728 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
729 if (!pte_chain->parent_ptes[i]) {
730 pte_chain->parent_ptes[i] = parent_pte;
731 return;
732 }
733 }
Avi Kivity714b93d2007-01-05 16:36:53 -0800734 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800735 BUG_ON(!pte_chain);
Avi Kivity4db35312007-11-21 15:28:32 +0200736 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800737 pte_chain->parent_ptes[0] = parent_pte;
738}
739
Avi Kivity4db35312007-11-21 15:28:32 +0200740static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800741 u64 *parent_pte)
742{
743 struct kvm_pte_chain *pte_chain;
744 struct hlist_node *node;
745 int i;
746
Avi Kivity4db35312007-11-21 15:28:32 +0200747 if (!sp->multimapped) {
748 BUG_ON(sp->parent_pte != parent_pte);
749 sp->parent_pte = NULL;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800750 return;
751 }
Avi Kivity4db35312007-11-21 15:28:32 +0200752 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800753 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
754 if (!pte_chain->parent_ptes[i])
755 break;
756 if (pte_chain->parent_ptes[i] != parent_pte)
757 continue;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800758 while (i + 1 < NR_PTE_CHAIN_ENTRIES
759 && pte_chain->parent_ptes[i + 1]) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800760 pte_chain->parent_ptes[i]
761 = pte_chain->parent_ptes[i + 1];
762 ++i;
763 }
764 pte_chain->parent_ptes[i] = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800765 if (i == 0) {
766 hlist_del(&pte_chain->link);
Avi Kivity90cb0522007-07-17 13:04:56 +0300767 mmu_free_pte_chain(pte_chain);
Avi Kivity4db35312007-11-21 15:28:32 +0200768 if (hlist_empty(&sp->parent_ptes)) {
769 sp->multimapped = 0;
770 sp->parent_pte = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800771 }
772 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800773 return;
774 }
775 BUG();
776}
777
Avi Kivity4db35312007-11-21 15:28:32 +0200778static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800779{
780 unsigned index;
781 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +0200782 struct kvm_mmu_page *sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800783 struct hlist_node *node;
784
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800785 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200786 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800787 bucket = &kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +0200788 hlist_for_each_entry(sp, node, bucket, hash_link)
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500789 if (sp->gfn == gfn && !sp->role.metaphysical
790 && !sp->role.invalid) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800791 pgprintk("%s: found role %x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800792 __func__, sp->role.word);
Avi Kivity4db35312007-11-21 15:28:32 +0200793 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800794 }
795 return NULL;
796}
797
798static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
799 gfn_t gfn,
800 gva_t gaddr,
801 unsigned level,
802 int metaphysical,
Avi Kivity41074d02007-12-09 17:00:02 +0200803 unsigned access,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +0200804 u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800805{
806 union kvm_mmu_page_role role;
807 unsigned index;
808 unsigned quadrant;
809 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +0200810 struct kvm_mmu_page *sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800811 struct hlist_node *node;
812
813 role.word = 0;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800814 role.glevels = vcpu->arch.mmu.root_level;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800815 role.level = level;
816 role.metaphysical = metaphysical;
Avi Kivity41074d02007-12-09 17:00:02 +0200817 role.access = access;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800818 if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800819 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
820 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
821 role.quadrant = quadrant;
822 }
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800823 pgprintk("%s: looking gfn %lx role %x\n", __func__,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800824 gfn, role.word);
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200825 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800826 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +0200827 hlist_for_each_entry(sp, node, bucket, hash_link)
828 if (sp->gfn == gfn && sp->role.word == role.word) {
829 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800830 pgprintk("%s: found\n", __func__);
Avi Kivity4db35312007-11-21 15:28:32 +0200831 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800832 }
Avi Kivitydfc5aa02007-12-18 19:47:18 +0200833 ++vcpu->kvm->stat.mmu_cache_miss;
Avi Kivity4db35312007-11-21 15:28:32 +0200834 sp = kvm_mmu_alloc_page(vcpu, parent_pte);
835 if (!sp)
836 return sp;
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800837 pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
Avi Kivity4db35312007-11-21 15:28:32 +0200838 sp->gfn = gfn;
839 sp->role = role;
840 hlist_add_head(&sp->hash_link, bucket);
Avi Kivity374cbac2007-01-05 16:36:43 -0800841 if (!metaphysical)
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500842 rmap_write_protect(vcpu->kvm, gfn);
Marcelo Tosattibed1d1d2008-04-04 14:56:44 -0300843 vcpu->arch.mmu.prefetch_page(vcpu, sp);
Avi Kivity4db35312007-11-21 15:28:32 +0200844 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800845}
846
Avi Kivity90cb0522007-07-17 13:04:56 +0300847static void kvm_mmu_page_unlink_children(struct kvm *kvm,
Avi Kivity4db35312007-11-21 15:28:32 +0200848 struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -0800849{
Avi Kivity697fe2e2007-01-05 16:36:46 -0800850 unsigned i;
851 u64 *pt;
852 u64 ent;
853
Avi Kivity4db35312007-11-21 15:28:32 +0200854 pt = sp->spt;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800855
Avi Kivity4db35312007-11-21 15:28:32 +0200856 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
Avi Kivity697fe2e2007-01-05 16:36:46 -0800857 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
Avi Kivityc7addb92007-09-16 18:58:32 +0200858 if (is_shadow_present_pte(pt[i]))
Izik Eidus290fc382007-09-27 14:11:22 +0200859 rmap_remove(kvm, &pt[i]);
Avi Kivityc7addb92007-09-16 18:58:32 +0200860 pt[i] = shadow_trap_nonpresent_pte;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800861 }
Avi Kivity90cb0522007-07-17 13:04:56 +0300862 kvm_flush_remote_tlbs(kvm);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800863 return;
864 }
865
866 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
867 ent = pt[i];
868
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300869 if (is_shadow_present_pte(ent)) {
870 if (!is_large_pte(ent)) {
871 ent &= PT64_BASE_ADDR_MASK;
872 mmu_page_remove_parent_pte(page_header(ent),
873 &pt[i]);
874 } else {
875 --kvm->stat.lpages;
876 rmap_remove(kvm, &pt[i]);
877 }
878 }
Avi Kivityc7addb92007-09-16 18:58:32 +0200879 pt[i] = shadow_trap_nonpresent_pte;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800880 }
Avi Kivity90cb0522007-07-17 13:04:56 +0300881 kvm_flush_remote_tlbs(kvm);
Avi Kivitya4360362007-01-05 16:36:45 -0800882}
883
Avi Kivity4db35312007-11-21 15:28:32 +0200884static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800885{
Avi Kivity4db35312007-11-21 15:28:32 +0200886 mmu_page_remove_parent_pte(sp, parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800887}
888
Avi Kivity12b7d282007-09-23 14:10:49 +0200889static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
890{
891 int i;
892
893 for (i = 0; i < KVM_MAX_VCPUS; ++i)
894 if (kvm->vcpus[i])
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800895 kvm->vcpus[i]->arch.last_pte_updated = NULL;
Avi Kivity12b7d282007-09-23 14:10:49 +0200896}
897
Avi Kivity4db35312007-11-21 15:28:32 +0200898static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -0800899{
900 u64 *parent_pte;
901
Avi Kivity4cee5762007-11-18 16:37:07 +0200902 ++kvm->stat.mmu_shadow_zapped;
Avi Kivity4db35312007-11-21 15:28:32 +0200903 while (sp->multimapped || sp->parent_pte) {
904 if (!sp->multimapped)
905 parent_pte = sp->parent_pte;
Avi Kivitya4360362007-01-05 16:36:45 -0800906 else {
907 struct kvm_pte_chain *chain;
908
Avi Kivity4db35312007-11-21 15:28:32 +0200909 chain = container_of(sp->parent_ptes.first,
Avi Kivitya4360362007-01-05 16:36:45 -0800910 struct kvm_pte_chain, link);
911 parent_pte = chain->parent_ptes[0];
912 }
Avi Kivity697fe2e2007-01-05 16:36:46 -0800913 BUG_ON(!parent_pte);
Avi Kivity4db35312007-11-21 15:28:32 +0200914 kvm_mmu_put_page(sp, parent_pte);
Avi Kivityc7addb92007-09-16 18:58:32 +0200915 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800916 }
Avi Kivity4db35312007-11-21 15:28:32 +0200917 kvm_mmu_page_unlink_children(kvm, sp);
918 if (!sp->root_count) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300919 if (!sp->role.metaphysical)
920 unaccount_shadowed(kvm, sp->gfn);
Avi Kivity4db35312007-11-21 15:28:32 +0200921 hlist_del(&sp->hash_link);
922 kvm_mmu_free_page(kvm, sp);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500923 } else {
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800924 list_move(&sp->link, &kvm->arch.active_mmu_pages);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500925 sp->role.invalid = 1;
926 kvm_reload_remote_mmus(kvm);
927 }
Avi Kivity12b7d282007-09-23 14:10:49 +0200928 kvm_mmu_reset_last_pte_updated(kvm);
Avi Kivitya4360362007-01-05 16:36:45 -0800929}
930
Izik Eidus82ce2c92007-10-02 18:52:55 +0200931/*
932 * Changing the number of mmu pages allocated to the vm
933 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
934 */
935void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
936{
937 /*
938 * If we set the number of mmu pages to be smaller be than the
939 * number of actived pages , we must to free some mmu pages before we
940 * change the value
941 */
942
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800943 if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
Izik Eidus82ce2c92007-10-02 18:52:55 +0200944 kvm_nr_mmu_pages) {
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800945 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
946 - kvm->arch.n_free_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +0200947
948 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
949 struct kvm_mmu_page *page;
950
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800951 page = container_of(kvm->arch.active_mmu_pages.prev,
Izik Eidus82ce2c92007-10-02 18:52:55 +0200952 struct kvm_mmu_page, link);
953 kvm_mmu_zap_page(kvm, page);
954 n_used_mmu_pages--;
955 }
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800956 kvm->arch.n_free_mmu_pages = 0;
Izik Eidus82ce2c92007-10-02 18:52:55 +0200957 }
958 else
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800959 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
960 - kvm->arch.n_alloc_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +0200961
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800962 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +0200963}
964
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500965static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitya4360362007-01-05 16:36:45 -0800966{
967 unsigned index;
968 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +0200969 struct kvm_mmu_page *sp;
Avi Kivitya4360362007-01-05 16:36:45 -0800970 struct hlist_node *node, *n;
971 int r;
972
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800973 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
Avi Kivitya4360362007-01-05 16:36:45 -0800974 r = 0;
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200975 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800976 bucket = &kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +0200977 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
978 if (sp->gfn == gfn && !sp->role.metaphysical) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800979 pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
Avi Kivity4db35312007-11-21 15:28:32 +0200980 sp->role.word);
981 kvm_mmu_zap_page(kvm, sp);
Avi Kivitya4360362007-01-05 16:36:45 -0800982 r = 1;
983 }
984 return r;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800985}
986
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500987static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
Avi Kivity97a0a012007-05-31 15:08:29 +0300988{
Avi Kivity4db35312007-11-21 15:28:32 +0200989 struct kvm_mmu_page *sp;
Avi Kivity97a0a012007-05-31 15:08:29 +0300990
Avi Kivity4db35312007-11-21 15:28:32 +0200991 while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800992 pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word);
Avi Kivity4db35312007-11-21 15:28:32 +0200993 kvm_mmu_zap_page(kvm, sp);
Avi Kivity97a0a012007-05-31 15:08:29 +0300994 }
995}
996
Avi Kivity38c335f2007-11-21 14:20:22 +0200997static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800998{
Avi Kivity38c335f2007-11-21 14:20:22 +0200999 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
Avi Kivity4db35312007-11-21 15:28:32 +02001000 struct kvm_mmu_page *sp = page_header(__pa(pte));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001001
Avi Kivity4db35312007-11-21 15:28:32 +02001002 __set_bit(slot, &sp->slot_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001003}
1004
Avi Kivity039576c2007-03-20 12:46:50 +02001005struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
1006{
Izik Eidus72dc67a2008-02-10 18:04:15 +02001007 struct page *page;
1008
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001009 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
Avi Kivity039576c2007-03-20 12:46:50 +02001010
1011 if (gpa == UNMAPPED_GVA)
1012 return NULL;
Izik Eidus72dc67a2008-02-10 18:04:15 +02001013
1014 down_read(&current->mm->mmap_sem);
1015 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1016 up_read(&current->mm->mmap_sem);
1017
1018 return page;
Avi Kivity039576c2007-03-20 12:46:50 +02001019}
1020
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001021static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1022 unsigned pt_access, unsigned pte_access,
1023 int user_fault, int write_fault, int dirty,
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001024 int *ptwrite, int largepage, gfn_t gfn,
Anthony Liguori35149e22008-04-02 14:46:56 -05001025 pfn_t pfn, bool speculative)
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001026{
1027 u64 spte;
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001028 int was_rmapped = 0;
Izik Eidus75e68e62008-01-12 23:49:09 +02001029 int was_writeble = is_writeble_pte(*shadow_pte);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001030
Avi Kivitybc750ba2007-12-09 18:39:41 +02001031 pgprintk("%s: spte %llx access %x write_fault %d"
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001032 " user_fault %d gfn %lx\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001033 __func__, *shadow_pte, pt_access,
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001034 write_fault, user_fault, gfn);
1035
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001036 if (is_rmap_pte(*shadow_pte)) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001037 /*
1038 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
1039 * the parent of the now unreachable PTE.
1040 */
1041 if (largepage && !is_large_pte(*shadow_pte)) {
1042 struct kvm_mmu_page *child;
1043 u64 pte = *shadow_pte;
1044
1045 child = page_header(pte & PT64_BASE_ADDR_MASK);
1046 mmu_page_remove_parent_pte(child, shadow_pte);
Anthony Liguori35149e22008-04-02 14:46:56 -05001047 } else if (pfn != spte_to_pfn(*shadow_pte)) {
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001048 pgprintk("hfn old %lx new %lx\n",
Anthony Liguori35149e22008-04-02 14:46:56 -05001049 spte_to_pfn(*shadow_pte), pfn);
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001050 rmap_remove(vcpu->kvm, shadow_pte);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001051 } else {
1052 if (largepage)
1053 was_rmapped = is_large_pte(*shadow_pte);
1054 else
1055 was_rmapped = 1;
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001056 }
Marcelo Tosatti15aaa812008-03-17 10:08:18 -03001057 }
1058
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001059 /*
1060 * We don't set the accessed bit, since we sometimes want to see
1061 * whether the guest actually used the pte (in order to detect
1062 * demand paging).
1063 */
Sheng Yang7b523452008-04-25 21:13:50 +08001064 spte = shadow_base_present_pte | shadow_dirty_mask;
Avi Kivity947da532008-03-18 11:05:52 +02001065 if (!speculative)
1066 pte_access |= PT_ACCESSED_MASK;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001067 if (!dirty)
1068 pte_access &= ~ACC_WRITE_MASK;
Sheng Yang7b523452008-04-25 21:13:50 +08001069 if (pte_access & ACC_EXEC_MASK)
1070 spte |= shadow_x_mask;
1071 else
1072 spte |= shadow_nx_mask;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001073 if (pte_access & ACC_USER_MASK)
Sheng Yang7b523452008-04-25 21:13:50 +08001074 spte |= shadow_user_mask;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001075 if (largepage)
1076 spte |= PT_PAGE_SIZE_MASK;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001077
Anthony Liguori35149e22008-04-02 14:46:56 -05001078 spte |= (u64)pfn << PAGE_SHIFT;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001079
1080 if ((pte_access & ACC_WRITE_MASK)
1081 || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
1082 struct kvm_mmu_page *shadow;
1083
1084 spte |= PT_WRITABLE_MASK;
1085 if (user_fault) {
1086 mmu_unshadow(vcpu->kvm, gfn);
1087 goto unshadowed;
1088 }
1089
1090 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001091 if (shadow ||
1092 (largepage && has_wrprotected_page(vcpu->kvm, gfn))) {
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001093 pgprintk("%s: found shadow page for %lx, marking ro\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001094 __func__, gfn);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001095 pte_access &= ~ACC_WRITE_MASK;
1096 if (is_writeble_pte(spte)) {
1097 spte &= ~PT_WRITABLE_MASK;
1098 kvm_x86_ops->tlb_flush(vcpu);
1099 }
1100 if (write_fault)
1101 *ptwrite = 1;
1102 }
1103 }
1104
1105unshadowed:
1106
1107 if (pte_access & ACC_WRITE_MASK)
1108 mark_page_dirty(vcpu->kvm, gfn);
1109
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001110 pgprintk("%s: setting spte %llx\n", __func__, spte);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001111 pgprintk("instantiating %s PTE (%s) at %d (%llx) addr %llx\n",
1112 (spte&PT_PAGE_SIZE_MASK)? "2MB" : "4kB",
1113 (spte&PT_WRITABLE_MASK)?"RW":"R", gfn, spte, shadow_pte);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001114 set_shadow_pte(shadow_pte, spte);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001115 if (!was_rmapped && (spte & PT_PAGE_SIZE_MASK)
1116 && (spte & PT_PRESENT_MASK))
1117 ++vcpu->kvm->stat.lpages;
1118
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001119 page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
1120 if (!was_rmapped) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001121 rmap_add(vcpu, shadow_pte, gfn, largepage);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001122 if (!is_rmap_pte(*shadow_pte))
Anthony Liguori35149e22008-04-02 14:46:56 -05001123 kvm_release_pfn_clean(pfn);
Izik Eidus75e68e62008-01-12 23:49:09 +02001124 } else {
1125 if (was_writeble)
Anthony Liguori35149e22008-04-02 14:46:56 -05001126 kvm_release_pfn_dirty(pfn);
Izik Eidus75e68e62008-01-12 23:49:09 +02001127 else
Anthony Liguori35149e22008-04-02 14:46:56 -05001128 kvm_release_pfn_clean(pfn);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001129 }
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001130 if (!ptwrite || !*ptwrite)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001131 vcpu->arch.last_pte_updated = shadow_pte;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001132}
1133
Avi Kivity6aa8b732006-12-10 02:21:36 -08001134static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
1135{
1136}
1137
Joerg Roedel4d9976b2008-02-07 13:47:42 +01001138static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
Anthony Liguori35149e22008-04-02 14:46:56 -05001139 int largepage, gfn_t gfn, pfn_t pfn,
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001140 int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001141{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001142 hpa_t table_addr = vcpu->arch.mmu.root_hpa;
Avi Kivitye8332402007-12-09 18:43:00 +02001143 int pt_write = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001144
1145 for (; ; level--) {
1146 u32 index = PT64_INDEX(v, level);
1147 u64 *table;
1148
1149 ASSERT(VALID_PAGE(table_addr));
1150 table = __va(table_addr);
1151
1152 if (level == 1) {
Avi Kivitye8332402007-12-09 18:43:00 +02001153 mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
Anthony Liguori35149e22008-04-02 14:46:56 -05001154 0, write, 1, &pt_write, 0, gfn, pfn, false);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001155 return pt_write;
1156 }
1157
1158 if (largepage && level == 2) {
1159 mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
Anthony Liguori35149e22008-04-02 14:46:56 -05001160 0, write, 1, &pt_write, 1, gfn, pfn, false);
Avi Kivityd196e342008-01-24 11:44:11 +02001161 return pt_write;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001162 }
1163
Avi Kivityc7addb92007-09-16 18:58:32 +02001164 if (table[index] == shadow_trap_nonpresent_pte) {
Avi Kivity25c0de22007-01-05 16:36:42 -08001165 struct kvm_mmu_page *new_table;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001166 gfn_t pseudo_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001167
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001168 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
1169 >> PAGE_SHIFT;
1170 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
1171 v, level - 1,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02001172 1, ACC_ALL, &table[index]);
Avi Kivity25c0de22007-01-05 16:36:42 -08001173 if (!new_table) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001174 pgprintk("nonpaging_map: ENOMEM\n");
Anthony Liguori35149e22008-04-02 14:46:56 -05001175 kvm_release_pfn_clean(pfn);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001176 return -ENOMEM;
1177 }
1178
Sheng Yang14394422008-04-28 12:24:45 +08001179 table[index] = __pa(new_table->spt)
1180 | PT_PRESENT_MASK | PT_WRITABLE_MASK
1181 | shadow_user_mask | shadow_x_mask;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001182 }
1183 table_addr = table[index] & PT64_BASE_ADDR_MASK;
1184 }
1185}
1186
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001187static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
1188{
1189 int r;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001190 int largepage = 0;
Anthony Liguori35149e22008-04-02 14:46:56 -05001191 pfn_t pfn;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001192
1193 down_read(&current->mm->mmap_sem);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001194 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1195 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1196 largepage = 1;
1197 }
1198
Anthony Liguori35149e22008-04-02 14:46:56 -05001199 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Izik Eidus72dc67a2008-02-10 18:04:15 +02001200 up_read(&current->mm->mmap_sem);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001201
Avi Kivityd196e342008-01-24 11:44:11 +02001202 /* mmio */
Anthony Liguori35149e22008-04-02 14:46:56 -05001203 if (is_error_pfn(pfn)) {
1204 kvm_release_pfn_clean(pfn);
Avi Kivityd196e342008-01-24 11:44:11 +02001205 return 1;
1206 }
1207
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001208 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivityeb787d12007-12-31 15:27:49 +02001209 kvm_mmu_free_some_pages(vcpu);
Anthony Liguori35149e22008-04-02 14:46:56 -05001210 r = __direct_map(vcpu, v, write, largepage, gfn, pfn,
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001211 PT32E_ROOT_LEVEL);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001212 spin_unlock(&vcpu->kvm->mmu_lock);
1213
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001214
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001215 return r;
1216}
1217
1218
Avi Kivityc7addb92007-09-16 18:58:32 +02001219static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
1220 struct kvm_mmu_page *sp)
1221{
1222 int i;
1223
1224 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1225 sp->spt[i] = shadow_trap_nonpresent_pte;
1226}
1227
Avi Kivity17ac10a2007-01-05 16:36:40 -08001228static void mmu_free_roots(struct kvm_vcpu *vcpu)
1229{
1230 int i;
Avi Kivity4db35312007-11-21 15:28:32 +02001231 struct kvm_mmu_page *sp;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001232
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001233 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
Avi Kivity7b53aa52007-06-05 12:17:03 +03001234 return;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001235 spin_lock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001236 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1237 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001238
Avi Kivity4db35312007-11-21 15:28:32 +02001239 sp = page_header(root);
1240 --sp->root_count;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001241 if (!sp->root_count && sp->role.invalid)
1242 kvm_mmu_zap_page(vcpu->kvm, sp);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001243 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001244 spin_unlock(&vcpu->kvm->mmu_lock);
Avi Kivity17ac10a2007-01-05 16:36:40 -08001245 return;
1246 }
Avi Kivity17ac10a2007-01-05 16:36:40 -08001247 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001248 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08001249
Avi Kivity417726a2007-04-12 17:35:58 +03001250 if (root) {
Avi Kivity417726a2007-04-12 17:35:58 +03001251 root &= PT64_BASE_ADDR_MASK;
Avi Kivity4db35312007-11-21 15:28:32 +02001252 sp = page_header(root);
1253 --sp->root_count;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001254 if (!sp->root_count && sp->role.invalid)
1255 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity417726a2007-04-12 17:35:58 +03001256 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001257 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001258 }
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001259 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001260 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001261}
1262
1263static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
1264{
1265 int i;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001266 gfn_t root_gfn;
Avi Kivity4db35312007-11-21 15:28:32 +02001267 struct kvm_mmu_page *sp;
Joerg Roedelfb72d162008-02-07 13:47:44 +01001268 int metaphysical = 0;
Avi Kivity3bb65a22007-01-05 16:36:51 -08001269
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001270 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001271
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001272 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1273 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001274
1275 ASSERT(!VALID_PAGE(root));
Joerg Roedelfb72d162008-02-07 13:47:44 +01001276 if (tdp_enabled)
1277 metaphysical = 1;
Avi Kivity4db35312007-11-21 15:28:32 +02001278 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
Joerg Roedelfb72d162008-02-07 13:47:44 +01001279 PT64_ROOT_LEVEL, metaphysical,
1280 ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02001281 root = __pa(sp->spt);
1282 ++sp->root_count;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001283 vcpu->arch.mmu.root_hpa = root;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001284 return;
1285 }
Joerg Roedelfb72d162008-02-07 13:47:44 +01001286 metaphysical = !is_paging(vcpu);
1287 if (tdp_enabled)
1288 metaphysical = 1;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001289 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001290 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08001291
1292 ASSERT(!VALID_PAGE(root));
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001293 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1294 if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1295 vcpu->arch.mmu.pae_root[i] = 0;
Avi Kivity417726a2007-04-12 17:35:58 +03001296 continue;
1297 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001298 root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1299 } else if (vcpu->arch.mmu.root_level == 0)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001300 root_gfn = 0;
Avi Kivity4db35312007-11-21 15:28:32 +02001301 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
Joerg Roedelfb72d162008-02-07 13:47:44 +01001302 PT32_ROOT_LEVEL, metaphysical,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02001303 ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02001304 root = __pa(sp->spt);
1305 ++sp->root_count;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001306 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001307 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001308 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
Avi Kivity17ac10a2007-01-05 16:36:40 -08001309}
1310
Avi Kivity6aa8b732006-12-10 02:21:36 -08001311static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1312{
1313 return vaddr;
1314}
1315
1316static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
Avi Kivity3f3e7122007-11-21 14:54:16 +02001317 u32 error_code)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001318{
Avi Kivitye8332402007-12-09 18:43:00 +02001319 gfn_t gfn;
Avi Kivitye2dec932007-01-05 16:36:54 -08001320 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001321
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001322 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
Avi Kivitye2dec932007-01-05 16:36:54 -08001323 r = mmu_topup_memory_caches(vcpu);
1324 if (r)
1325 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -08001326
Avi Kivity6aa8b732006-12-10 02:21:36 -08001327 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001328 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001329
Avi Kivitye8332402007-12-09 18:43:00 +02001330 gfn = gva >> PAGE_SHIFT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001331
Avi Kivitye8332402007-12-09 18:43:00 +02001332 return nonpaging_map(vcpu, gva & PAGE_MASK,
1333 error_code & PFERR_WRITE_MASK, gfn);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001334}
1335
Joerg Roedelfb72d162008-02-07 13:47:44 +01001336static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
1337 u32 error_code)
1338{
Anthony Liguori35149e22008-04-02 14:46:56 -05001339 pfn_t pfn;
Joerg Roedelfb72d162008-02-07 13:47:44 +01001340 int r;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001341 int largepage = 0;
1342 gfn_t gfn = gpa >> PAGE_SHIFT;
Joerg Roedelfb72d162008-02-07 13:47:44 +01001343
1344 ASSERT(vcpu);
1345 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1346
1347 r = mmu_topup_memory_caches(vcpu);
1348 if (r)
1349 return r;
1350
1351 down_read(&current->mm->mmap_sem);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001352 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1353 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1354 largepage = 1;
1355 }
Anthony Liguori35149e22008-04-02 14:46:56 -05001356 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Marcelo Tosatti3200f402008-03-29 20:17:59 -03001357 up_read(&current->mm->mmap_sem);
Anthony Liguori35149e22008-04-02 14:46:56 -05001358 if (is_error_pfn(pfn)) {
1359 kvm_release_pfn_clean(pfn);
Joerg Roedelfb72d162008-02-07 13:47:44 +01001360 return 1;
1361 }
1362 spin_lock(&vcpu->kvm->mmu_lock);
1363 kvm_mmu_free_some_pages(vcpu);
1364 r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
Sheng Yang67253af2008-04-25 10:20:22 +08001365 largepage, gfn, pfn, kvm_x86_ops->get_tdp_level());
Joerg Roedelfb72d162008-02-07 13:47:44 +01001366 spin_unlock(&vcpu->kvm->mmu_lock);
Joerg Roedelfb72d162008-02-07 13:47:44 +01001367
1368 return r;
1369}
1370
Avi Kivity6aa8b732006-12-10 02:21:36 -08001371static void nonpaging_free(struct kvm_vcpu *vcpu)
1372{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001373 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001374}
1375
1376static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1377{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001378 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001379
1380 context->new_cr3 = nonpaging_new_cr3;
1381 context->page_fault = nonpaging_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001382 context->gva_to_gpa = nonpaging_gva_to_gpa;
1383 context->free = nonpaging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02001384 context->prefetch_page = nonpaging_prefetch_page;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001385 context->root_level = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001386 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001387 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001388 return 0;
1389}
1390
Avi Kivityd835dfe2007-11-21 02:57:59 +02001391void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001392{
Avi Kivity1165f5f2007-04-19 17:27:43 +03001393 ++vcpu->stat.tlb_flush;
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03001394 kvm_x86_ops->tlb_flush(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001395}
1396
1397static void paging_new_cr3(struct kvm_vcpu *vcpu)
1398{
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001399 pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001400 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001401}
1402
Avi Kivity6aa8b732006-12-10 02:21:36 -08001403static void inject_page_fault(struct kvm_vcpu *vcpu,
1404 u64 addr,
1405 u32 err_code)
1406{
Avi Kivityc3c91fe2007-11-25 14:04:58 +02001407 kvm_inject_page_fault(vcpu, addr, err_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001408}
1409
Avi Kivity6aa8b732006-12-10 02:21:36 -08001410static void paging_free(struct kvm_vcpu *vcpu)
1411{
1412 nonpaging_free(vcpu);
1413}
1414
1415#define PTTYPE 64
1416#include "paging_tmpl.h"
1417#undef PTTYPE
1418
1419#define PTTYPE 32
1420#include "paging_tmpl.h"
1421#undef PTTYPE
1422
Avi Kivity17ac10a2007-01-05 16:36:40 -08001423static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001424{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001425 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001426
1427 ASSERT(is_pae(vcpu));
1428 context->new_cr3 = paging_new_cr3;
1429 context->page_fault = paging64_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001430 context->gva_to_gpa = paging64_gva_to_gpa;
Avi Kivityc7addb92007-09-16 18:58:32 +02001431 context->prefetch_page = paging64_prefetch_page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001432 context->free = paging_free;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001433 context->root_level = level;
1434 context->shadow_root_level = level;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001435 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001436 return 0;
1437}
1438
Avi Kivity17ac10a2007-01-05 16:36:40 -08001439static int paging64_init_context(struct kvm_vcpu *vcpu)
1440{
1441 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1442}
1443
Avi Kivity6aa8b732006-12-10 02:21:36 -08001444static int paging32_init_context(struct kvm_vcpu *vcpu)
1445{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001446 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001447
1448 context->new_cr3 = paging_new_cr3;
1449 context->page_fault = paging32_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001450 context->gva_to_gpa = paging32_gva_to_gpa;
1451 context->free = paging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02001452 context->prefetch_page = paging32_prefetch_page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001453 context->root_level = PT32_ROOT_LEVEL;
1454 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001455 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001456 return 0;
1457}
1458
1459static int paging32E_init_context(struct kvm_vcpu *vcpu)
1460{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001461 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001462}
1463
Joerg Roedelfb72d162008-02-07 13:47:44 +01001464static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
1465{
1466 struct kvm_mmu *context = &vcpu->arch.mmu;
1467
1468 context->new_cr3 = nonpaging_new_cr3;
1469 context->page_fault = tdp_page_fault;
1470 context->free = nonpaging_free;
1471 context->prefetch_page = nonpaging_prefetch_page;
Sheng Yang67253af2008-04-25 10:20:22 +08001472 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
Joerg Roedelfb72d162008-02-07 13:47:44 +01001473 context->root_hpa = INVALID_PAGE;
1474
1475 if (!is_paging(vcpu)) {
1476 context->gva_to_gpa = nonpaging_gva_to_gpa;
1477 context->root_level = 0;
1478 } else if (is_long_mode(vcpu)) {
1479 context->gva_to_gpa = paging64_gva_to_gpa;
1480 context->root_level = PT64_ROOT_LEVEL;
1481 } else if (is_pae(vcpu)) {
1482 context->gva_to_gpa = paging64_gva_to_gpa;
1483 context->root_level = PT32E_ROOT_LEVEL;
1484 } else {
1485 context->gva_to_gpa = paging32_gva_to_gpa;
1486 context->root_level = PT32_ROOT_LEVEL;
1487 }
1488
1489 return 0;
1490}
1491
1492static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001493{
1494 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001495 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001496
1497 if (!is_paging(vcpu))
1498 return nonpaging_init_context(vcpu);
Avi Kivitya9058ec2006-12-29 16:49:37 -08001499 else if (is_long_mode(vcpu))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001500 return paging64_init_context(vcpu);
1501 else if (is_pae(vcpu))
1502 return paging32E_init_context(vcpu);
1503 else
1504 return paging32_init_context(vcpu);
1505}
1506
Joerg Roedelfb72d162008-02-07 13:47:44 +01001507static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1508{
Anthony Liguori35149e22008-04-02 14:46:56 -05001509 vcpu->arch.update_pte.pfn = bad_pfn;
1510
Joerg Roedelfb72d162008-02-07 13:47:44 +01001511 if (tdp_enabled)
1512 return init_kvm_tdp_mmu(vcpu);
1513 else
1514 return init_kvm_softmmu(vcpu);
1515}
1516
Avi Kivity6aa8b732006-12-10 02:21:36 -08001517static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1518{
1519 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001520 if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
1521 vcpu->arch.mmu.free(vcpu);
1522 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001523 }
1524}
1525
1526int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1527{
Avi Kivity17c3ba92007-06-04 15:58:30 +03001528 destroy_kvm_mmu(vcpu);
1529 return init_kvm_mmu(vcpu);
1530}
Eddie Dong8668a3c2007-10-10 14:26:45 +08001531EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001532
1533int kvm_mmu_load(struct kvm_vcpu *vcpu)
1534{
Avi Kivity714b93d2007-01-05 16:36:53 -08001535 int r;
1536
Avi Kivitye2dec932007-01-05 16:36:54 -08001537 r = mmu_topup_memory_caches(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001538 if (r)
1539 goto out;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001540 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivityeb787d12007-12-31 15:27:49 +02001541 kvm_mmu_free_some_pages(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001542 mmu_alloc_roots(vcpu);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001543 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001544 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001545 kvm_mmu_flush_tlb(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001546out:
1547 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001548}
Avi Kivity17c3ba92007-06-04 15:58:30 +03001549EXPORT_SYMBOL_GPL(kvm_mmu_load);
1550
1551void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1552{
1553 mmu_free_roots(vcpu);
1554}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001555
Avi Kivity09072da2007-05-01 14:16:52 +03001556static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +02001557 struct kvm_mmu_page *sp,
Avi Kivityac1b7142007-03-08 17:13:32 +02001558 u64 *spte)
1559{
1560 u64 pte;
1561 struct kvm_mmu_page *child;
1562
1563 pte = *spte;
Avi Kivityc7addb92007-09-16 18:58:32 +02001564 if (is_shadow_present_pte(pte)) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001565 if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
1566 is_large_pte(pte))
Izik Eidus290fc382007-09-27 14:11:22 +02001567 rmap_remove(vcpu->kvm, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02001568 else {
1569 child = page_header(pte & PT64_BASE_ADDR_MASK);
Avi Kivity90cb0522007-07-17 13:04:56 +03001570 mmu_page_remove_parent_pte(child, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02001571 }
1572 }
Avi Kivityc7addb92007-09-16 18:58:32 +02001573 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001574 if (is_large_pte(pte))
1575 --vcpu->kvm->stat.lpages;
Avi Kivityac1b7142007-03-08 17:13:32 +02001576}
1577
Avi Kivity00284252007-05-01 16:53:31 +03001578static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +02001579 struct kvm_mmu_page *sp,
Avi Kivity00284252007-05-01 16:53:31 +03001580 u64 *spte,
Dong, Eddie489f1d62008-01-07 11:14:20 +02001581 const void *new)
Avi Kivity00284252007-05-01 16:53:31 +03001582{
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001583 if ((sp->role.level != PT_PAGE_TABLE_LEVEL)
1584 && !vcpu->arch.update_pte.largepage) {
Avi Kivity4cee5762007-11-18 16:37:07 +02001585 ++vcpu->kvm->stat.mmu_pde_zapped;
Avi Kivity00284252007-05-01 16:53:31 +03001586 return;
Avi Kivity4cee5762007-11-18 16:37:07 +02001587 }
Avi Kivity00284252007-05-01 16:53:31 +03001588
Avi Kivity4cee5762007-11-18 16:37:07 +02001589 ++vcpu->kvm->stat.mmu_pte_updated;
Avi Kivity4db35312007-11-21 15:28:32 +02001590 if (sp->role.glevels == PT32_ROOT_LEVEL)
Dong, Eddie489f1d62008-01-07 11:14:20 +02001591 paging32_update_pte(vcpu, sp, spte, new);
Avi Kivity00284252007-05-01 16:53:31 +03001592 else
Dong, Eddie489f1d62008-01-07 11:14:20 +02001593 paging64_update_pte(vcpu, sp, spte, new);
Avi Kivity00284252007-05-01 16:53:31 +03001594}
1595
Avi Kivity79539ce2007-11-21 02:06:21 +02001596static bool need_remote_flush(u64 old, u64 new)
1597{
1598 if (!is_shadow_present_pte(old))
1599 return false;
1600 if (!is_shadow_present_pte(new))
1601 return true;
1602 if ((old ^ new) & PT64_BASE_ADDR_MASK)
1603 return true;
1604 old ^= PT64_NX_MASK;
1605 new ^= PT64_NX_MASK;
1606 return (old & ~new & PT64_PERM_MASK) != 0;
1607}
1608
1609static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
1610{
1611 if (need_remote_flush(old, new))
1612 kvm_flush_remote_tlbs(vcpu->kvm);
1613 else
1614 kvm_mmu_flush_tlb(vcpu);
1615}
1616
Avi Kivity12b7d282007-09-23 14:10:49 +02001617static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
1618{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001619 u64 *spte = vcpu->arch.last_pte_updated;
Avi Kivity12b7d282007-09-23 14:10:49 +02001620
Sheng Yang7b523452008-04-25 21:13:50 +08001621 return !!(spte && (*spte & shadow_accessed_mask));
Avi Kivity12b7d282007-09-23 14:10:49 +02001622}
1623
Avi Kivityd7824ff2007-12-30 12:29:05 +02001624static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1625 const u8 *new, int bytes)
1626{
1627 gfn_t gfn;
1628 int r;
1629 u64 gpte = 0;
Anthony Liguori35149e22008-04-02 14:46:56 -05001630 pfn_t pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02001631
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001632 vcpu->arch.update_pte.largepage = 0;
1633
Avi Kivityd7824ff2007-12-30 12:29:05 +02001634 if (bytes != 4 && bytes != 8)
1635 return;
1636
1637 /*
1638 * Assume that the pte write on a page table of the same type
1639 * as the current vcpu paging mode. This is nearly always true
1640 * (might be false while changing modes). Note it is verified later
1641 * by update_pte().
1642 */
1643 if (is_pae(vcpu)) {
1644 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
1645 if ((bytes == 4) && (gpa % 4 == 0)) {
1646 r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
1647 if (r)
1648 return;
1649 memcpy((void *)&gpte + (gpa % 8), new, 4);
1650 } else if ((bytes == 8) && (gpa % 8 == 0)) {
1651 memcpy((void *)&gpte, new, 8);
1652 }
1653 } else {
1654 if ((bytes == 4) && (gpa % 4 == 0))
1655 memcpy((void *)&gpte, new, 4);
1656 }
1657 if (!is_present_pte(gpte))
1658 return;
1659 gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
Izik Eidus72dc67a2008-02-10 18:04:15 +02001660
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001661 down_read(&current->mm->mmap_sem);
1662 if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
1663 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1664 vcpu->arch.update_pte.largepage = 1;
1665 }
Anthony Liguori35149e22008-04-02 14:46:56 -05001666 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001667 up_read(&current->mm->mmap_sem);
Izik Eidus72dc67a2008-02-10 18:04:15 +02001668
Anthony Liguori35149e22008-04-02 14:46:56 -05001669 if (is_error_pfn(pfn)) {
1670 kvm_release_pfn_clean(pfn);
Avi Kivityd196e342008-01-24 11:44:11 +02001671 return;
1672 }
Avi Kivityd7824ff2007-12-30 12:29:05 +02001673 vcpu->arch.update_pte.gfn = gfn;
Anthony Liguori35149e22008-04-02 14:46:56 -05001674 vcpu->arch.update_pte.pfn = pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02001675}
1676
Avi Kivity09072da2007-05-01 14:16:52 +03001677void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
Shaohua Life551882007-07-23 14:51:39 +08001678 const u8 *new, int bytes)
Avi Kivityda4a00f2007-01-05 16:36:44 -08001679{
Avi Kivity9b7a0322007-01-05 16:36:45 -08001680 gfn_t gfn = gpa >> PAGE_SHIFT;
Avi Kivity4db35312007-11-21 15:28:32 +02001681 struct kvm_mmu_page *sp;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001682 struct hlist_node *node, *n;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001683 struct hlist_head *bucket;
1684 unsigned index;
Dong, Eddie489f1d62008-01-07 11:14:20 +02001685 u64 entry, gentry;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001686 u64 *spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001687 unsigned offset = offset_in_page(gpa);
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001688 unsigned pte_size;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001689 unsigned page_offset;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001690 unsigned misaligned;
Avi Kivityfce06572007-05-01 16:44:05 +03001691 unsigned quadrant;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001692 int level;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001693 int flooded = 0;
Avi Kivityac1b7142007-03-08 17:13:32 +02001694 int npte;
Dong, Eddie489f1d62008-01-07 11:14:20 +02001695 int r;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001696
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001697 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
Avi Kivityd7824ff2007-12-30 12:29:05 +02001698 mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001699 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivityeb787d12007-12-31 15:27:49 +02001700 kvm_mmu_free_some_pages(vcpu);
Avi Kivity4cee5762007-11-18 16:37:07 +02001701 ++vcpu->kvm->stat.mmu_pte_write;
Avi Kivityc7addb92007-09-16 18:58:32 +02001702 kvm_mmu_audit(vcpu, "pre pte write");
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001703 if (gfn == vcpu->arch.last_pt_write_gfn
Avi Kivity12b7d282007-09-23 14:10:49 +02001704 && !last_updated_pte_accessed(vcpu)) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001705 ++vcpu->arch.last_pt_write_count;
1706 if (vcpu->arch.last_pt_write_count >= 3)
Avi Kivity86a5ba02007-01-05 16:36:50 -08001707 flooded = 1;
1708 } else {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001709 vcpu->arch.last_pt_write_gfn = gfn;
1710 vcpu->arch.last_pt_write_count = 1;
1711 vcpu->arch.last_pte_updated = NULL;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001712 }
Dong, Eddie1ae0a132008-01-07 13:20:25 +02001713 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001714 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +02001715 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
1716 if (sp->gfn != gfn || sp->role.metaphysical)
Avi Kivity9b7a0322007-01-05 16:36:45 -08001717 continue;
Avi Kivity4db35312007-11-21 15:28:32 +02001718 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001719 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
Avi Kivitye925c5b2007-04-30 14:47:02 +03001720 misaligned |= bytes < 4;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001721 if (misaligned || flooded) {
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001722 /*
1723 * Misaligned accesses are too much trouble to fix
1724 * up; also, they usually indicate a page is not used
1725 * as a page table.
Avi Kivity86a5ba02007-01-05 16:36:50 -08001726 *
1727 * If we're seeing too many writes to a page,
1728 * it may no longer be a page table, or we may be
1729 * forking, in which case it is better to unmap the
1730 * page.
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001731 */
1732 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
Avi Kivity4db35312007-11-21 15:28:32 +02001733 gpa, bytes, sp->role.word);
1734 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity4cee5762007-11-18 16:37:07 +02001735 ++vcpu->kvm->stat.mmu_flooded;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001736 continue;
1737 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001738 page_offset = offset;
Avi Kivity4db35312007-11-21 15:28:32 +02001739 level = sp->role.level;
Avi Kivityac1b7142007-03-08 17:13:32 +02001740 npte = 1;
Avi Kivity4db35312007-11-21 15:28:32 +02001741 if (sp->role.glevels == PT32_ROOT_LEVEL) {
Avi Kivityac1b7142007-03-08 17:13:32 +02001742 page_offset <<= 1; /* 32->64 */
1743 /*
1744 * A 32-bit pde maps 4MB while the shadow pdes map
1745 * only 2MB. So we need to double the offset again
1746 * and zap two pdes instead of one.
1747 */
1748 if (level == PT32_ROOT_LEVEL) {
Avi Kivity6b8d0f92007-04-18 11:18:18 +03001749 page_offset &= ~7; /* kill rounding error */
Avi Kivityac1b7142007-03-08 17:13:32 +02001750 page_offset <<= 1;
1751 npte = 2;
1752 }
Avi Kivityfce06572007-05-01 16:44:05 +03001753 quadrant = page_offset >> PAGE_SHIFT;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001754 page_offset &= ~PAGE_MASK;
Avi Kivity4db35312007-11-21 15:28:32 +02001755 if (quadrant != sp->role.quadrant)
Avi Kivityfce06572007-05-01 16:44:05 +03001756 continue;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001757 }
Avi Kivity4db35312007-11-21 15:28:32 +02001758 spte = &sp->spt[page_offset / sizeof(*spte)];
Dong, Eddie489f1d62008-01-07 11:14:20 +02001759 if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
1760 gentry = 0;
1761 r = kvm_read_guest_atomic(vcpu->kvm,
1762 gpa & ~(u64)(pte_size - 1),
1763 &gentry, pte_size);
1764 new = (const void *)&gentry;
1765 if (r < 0)
1766 new = NULL;
1767 }
Avi Kivityac1b7142007-03-08 17:13:32 +02001768 while (npte--) {
Avi Kivity79539ce2007-11-21 02:06:21 +02001769 entry = *spte;
Avi Kivity4db35312007-11-21 15:28:32 +02001770 mmu_pte_write_zap_pte(vcpu, sp, spte);
Dong, Eddie489f1d62008-01-07 11:14:20 +02001771 if (new)
1772 mmu_pte_write_new_pte(vcpu, sp, spte, new);
Avi Kivity79539ce2007-11-21 02:06:21 +02001773 mmu_pte_write_flush_tlb(vcpu, entry, *spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02001774 ++spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001775 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001776 }
Avi Kivityc7addb92007-09-16 18:58:32 +02001777 kvm_mmu_audit(vcpu, "post pte write");
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001778 spin_unlock(&vcpu->kvm->mmu_lock);
Anthony Liguori35149e22008-04-02 14:46:56 -05001779 if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
1780 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
1781 vcpu->arch.update_pte.pfn = bad_pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02001782 }
Avi Kivityda4a00f2007-01-05 16:36:44 -08001783}
1784
Avi Kivitya4360362007-01-05 16:36:45 -08001785int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1786{
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001787 gpa_t gpa;
1788 int r;
Avi Kivitya4360362007-01-05 16:36:45 -08001789
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001790 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001791
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001792 spin_lock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001793 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001794 spin_unlock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001795 return r;
Avi Kivitya4360362007-01-05 16:36:45 -08001796}
1797
Avi Kivity22d95b12007-09-14 20:26:06 +03001798void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
Avi Kivityebeace82007-01-05 16:36:47 -08001799{
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001800 while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
Avi Kivity4db35312007-11-21 15:28:32 +02001801 struct kvm_mmu_page *sp;
Avi Kivityebeace82007-01-05 16:36:47 -08001802
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001803 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
Avi Kivity4db35312007-11-21 15:28:32 +02001804 struct kvm_mmu_page, link);
1805 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity4cee5762007-11-18 16:37:07 +02001806 ++vcpu->kvm->stat.mmu_recycled;
Avi Kivityebeace82007-01-05 16:36:47 -08001807 }
1808}
Avi Kivityebeace82007-01-05 16:36:47 -08001809
Avi Kivity30677142007-10-28 18:48:59 +02001810int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
1811{
1812 int r;
1813 enum emulation_result er;
1814
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001815 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
Avi Kivity30677142007-10-28 18:48:59 +02001816 if (r < 0)
1817 goto out;
1818
1819 if (!r) {
1820 r = 1;
1821 goto out;
1822 }
1823
Avi Kivityb733bfb2007-10-28 18:52:05 +02001824 r = mmu_topup_memory_caches(vcpu);
1825 if (r)
1826 goto out;
1827
Avi Kivity30677142007-10-28 18:48:59 +02001828 er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
Avi Kivity30677142007-10-28 18:48:59 +02001829
1830 switch (er) {
1831 case EMULATE_DONE:
1832 return 1;
1833 case EMULATE_DO_MMIO:
1834 ++vcpu->stat.mmio_exits;
1835 return 0;
1836 case EMULATE_FAIL:
1837 kvm_report_emulation_failure(vcpu, "pagetable");
1838 return 1;
1839 default:
1840 BUG();
1841 }
1842out:
Avi Kivity30677142007-10-28 18:48:59 +02001843 return r;
1844}
1845EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
1846
Joerg Roedel18552672008-02-07 13:47:41 +01001847void kvm_enable_tdp(void)
1848{
1849 tdp_enabled = true;
1850}
1851EXPORT_SYMBOL_GPL(kvm_enable_tdp);
1852
Avi Kivity6aa8b732006-12-10 02:21:36 -08001853static void free_mmu_pages(struct kvm_vcpu *vcpu)
1854{
Avi Kivity4db35312007-11-21 15:28:32 +02001855 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001856
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001857 while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
1858 sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
Avi Kivity4db35312007-11-21 15:28:32 +02001859 struct kvm_mmu_page, link);
1860 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivityf51234c2007-01-05 16:36:52 -08001861 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001862 free_page((unsigned long)vcpu->arch.mmu.pae_root);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001863}
1864
1865static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1866{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001867 struct page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001868 int i;
1869
1870 ASSERT(vcpu);
1871
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001872 if (vcpu->kvm->arch.n_requested_mmu_pages)
1873 vcpu->kvm->arch.n_free_mmu_pages =
1874 vcpu->kvm->arch.n_requested_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02001875 else
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001876 vcpu->kvm->arch.n_free_mmu_pages =
1877 vcpu->kvm->arch.n_alloc_mmu_pages;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001878 /*
1879 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1880 * Therefore we need to allocate shadow page tables in the first
1881 * 4GB of memory, which happens to fit the DMA32 zone.
1882 */
1883 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1884 if (!page)
1885 goto error_1;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001886 vcpu->arch.mmu.pae_root = page_address(page);
Avi Kivity17ac10a2007-01-05 16:36:40 -08001887 for (i = 0; i < 4; ++i)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001888 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001889
Avi Kivity6aa8b732006-12-10 02:21:36 -08001890 return 0;
1891
1892error_1:
1893 free_mmu_pages(vcpu);
1894 return -ENOMEM;
1895}
1896
Ingo Molnar8018c272006-12-29 16:50:01 -08001897int kvm_mmu_create(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001898{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001899 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001900 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001901
Ingo Molnar8018c272006-12-29 16:50:01 -08001902 return alloc_mmu_pages(vcpu);
1903}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001904
Ingo Molnar8018c272006-12-29 16:50:01 -08001905int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1906{
1907 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001908 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity2c264952006-12-22 01:05:28 -08001909
Ingo Molnar8018c272006-12-29 16:50:01 -08001910 return init_kvm_mmu(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001911}
1912
1913void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1914{
1915 ASSERT(vcpu);
1916
1917 destroy_kvm_mmu(vcpu);
1918 free_mmu_pages(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001919 mmu_free_memory_caches(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001920}
1921
Avi Kivity90cb0522007-07-17 13:04:56 +03001922void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001923{
Avi Kivity4db35312007-11-21 15:28:32 +02001924 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001925
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001926 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001927 int i;
1928 u64 *pt;
1929
Avi Kivity4db35312007-11-21 15:28:32 +02001930 if (!test_bit(slot, &sp->slot_bitmap))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001931 continue;
1932
Avi Kivity4db35312007-11-21 15:28:32 +02001933 pt = sp->spt;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001934 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1935 /* avoid RMW */
Izik Eidus9647c142007-10-16 14:43:46 +02001936 if (pt[i] & PT_WRITABLE_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001937 pt[i] &= ~PT_WRITABLE_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001938 }
1939}
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001940
Avi Kivity90cb0522007-07-17 13:04:56 +03001941void kvm_mmu_zap_all(struct kvm *kvm)
Dor Laore0fa8262007-03-30 13:06:33 +03001942{
Avi Kivity4db35312007-11-21 15:28:32 +02001943 struct kvm_mmu_page *sp, *node;
Dor Laore0fa8262007-03-30 13:06:33 +03001944
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001945 spin_lock(&kvm->mmu_lock);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001946 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
Avi Kivity4db35312007-11-21 15:28:32 +02001947 kvm_mmu_zap_page(kvm, sp);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001948 spin_unlock(&kvm->mmu_lock);
Dor Laore0fa8262007-03-30 13:06:33 +03001949
Avi Kivity90cb0522007-07-17 13:04:56 +03001950 kvm_flush_remote_tlbs(kvm);
Dor Laore0fa8262007-03-30 13:06:33 +03001951}
1952
Izik Eidus3ee16c82008-03-30 15:17:21 +03001953void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
1954{
1955 struct kvm_mmu_page *page;
1956
1957 page = container_of(kvm->arch.active_mmu_pages.prev,
1958 struct kvm_mmu_page, link);
1959 kvm_mmu_zap_page(kvm, page);
1960}
1961
1962static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
1963{
1964 struct kvm *kvm;
1965 struct kvm *kvm_freed = NULL;
1966 int cache_count = 0;
1967
1968 spin_lock(&kvm_lock);
1969
1970 list_for_each_entry(kvm, &vm_list, vm_list) {
1971 int npages;
1972
1973 spin_lock(&kvm->mmu_lock);
1974 npages = kvm->arch.n_alloc_mmu_pages -
1975 kvm->arch.n_free_mmu_pages;
1976 cache_count += npages;
1977 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
1978 kvm_mmu_remove_one_alloc_mmu_page(kvm);
1979 cache_count--;
1980 kvm_freed = kvm;
1981 }
1982 nr_to_scan--;
1983
1984 spin_unlock(&kvm->mmu_lock);
1985 }
1986 if (kvm_freed)
1987 list_move_tail(&kvm_freed->vm_list, &vm_list);
1988
1989 spin_unlock(&kvm_lock);
1990
1991 return cache_count;
1992}
1993
1994static struct shrinker mmu_shrinker = {
1995 .shrink = mmu_shrink,
1996 .seeks = DEFAULT_SEEKS * 10,
1997};
1998
Ingo Molnar2ddfd202008-05-22 10:37:48 +02001999static void mmu_destroy_caches(void)
Avi Kivityb5a33a72007-04-15 16:31:09 +03002000{
2001 if (pte_chain_cache)
2002 kmem_cache_destroy(pte_chain_cache);
2003 if (rmap_desc_cache)
2004 kmem_cache_destroy(rmap_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +03002005 if (mmu_page_header_cache)
2006 kmem_cache_destroy(mmu_page_header_cache);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002007}
2008
Izik Eidus3ee16c82008-03-30 15:17:21 +03002009void kvm_mmu_module_exit(void)
2010{
2011 mmu_destroy_caches();
2012 unregister_shrinker(&mmu_shrinker);
2013}
2014
Avi Kivityb5a33a72007-04-15 16:31:09 +03002015int kvm_mmu_module_init(void)
2016{
2017 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
2018 sizeof(struct kvm_pte_chain),
Paul Mundt20c2df82007-07-20 10:11:58 +09002019 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002020 if (!pte_chain_cache)
2021 goto nomem;
2022 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
2023 sizeof(struct kvm_rmap_desc),
Paul Mundt20c2df82007-07-20 10:11:58 +09002024 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002025 if (!rmap_desc_cache)
2026 goto nomem;
2027
Avi Kivityd3d25b02007-05-30 12:34:53 +03002028 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
2029 sizeof(struct kvm_mmu_page),
Paul Mundt20c2df82007-07-20 10:11:58 +09002030 0, 0, NULL);
Avi Kivityd3d25b02007-05-30 12:34:53 +03002031 if (!mmu_page_header_cache)
2032 goto nomem;
2033
Izik Eidus3ee16c82008-03-30 15:17:21 +03002034 register_shrinker(&mmu_shrinker);
2035
Avi Kivityb5a33a72007-04-15 16:31:09 +03002036 return 0;
2037
2038nomem:
Izik Eidus3ee16c82008-03-30 15:17:21 +03002039 mmu_destroy_caches();
Avi Kivityb5a33a72007-04-15 16:31:09 +03002040 return -ENOMEM;
2041}
2042
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08002043/*
2044 * Caculate mmu pages needed for kvm.
2045 */
2046unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
2047{
2048 int i;
2049 unsigned int nr_mmu_pages;
2050 unsigned int nr_pages = 0;
2051
2052 for (i = 0; i < kvm->nmemslots; i++)
2053 nr_pages += kvm->memslots[i].npages;
2054
2055 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
2056 nr_mmu_pages = max(nr_mmu_pages,
2057 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
2058
2059 return nr_mmu_pages;
2060}
2061
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002062static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2063 unsigned len)
2064{
2065 if (len > buffer->len)
2066 return NULL;
2067 return buffer->ptr;
2068}
2069
2070static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2071 unsigned len)
2072{
2073 void *ret;
2074
2075 ret = pv_mmu_peek_buffer(buffer, len);
2076 if (!ret)
2077 return ret;
2078 buffer->ptr += len;
2079 buffer->len -= len;
2080 buffer->processed += len;
2081 return ret;
2082}
2083
2084static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
2085 gpa_t addr, gpa_t value)
2086{
2087 int bytes = 8;
2088 int r;
2089
2090 if (!is_long_mode(vcpu) && !is_pae(vcpu))
2091 bytes = 4;
2092
2093 r = mmu_topup_memory_caches(vcpu);
2094 if (r)
2095 return r;
2096
Marcelo Tosatti3200f402008-03-29 20:17:59 -03002097 if (!emulator_write_phys(vcpu, addr, &value, bytes))
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002098 return -EFAULT;
2099
2100 return 1;
2101}
2102
2103static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2104{
2105 kvm_x86_ops->tlb_flush(vcpu);
2106 return 1;
2107}
2108
2109static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
2110{
2111 spin_lock(&vcpu->kvm->mmu_lock);
2112 mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
2113 spin_unlock(&vcpu->kvm->mmu_lock);
2114 return 1;
2115}
2116
2117static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
2118 struct kvm_pv_mmu_op_buffer *buffer)
2119{
2120 struct kvm_mmu_op_header *header;
2121
2122 header = pv_mmu_peek_buffer(buffer, sizeof *header);
2123 if (!header)
2124 return 0;
2125 switch (header->op) {
2126 case KVM_MMU_OP_WRITE_PTE: {
2127 struct kvm_mmu_op_write_pte *wpte;
2128
2129 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
2130 if (!wpte)
2131 return 0;
2132 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
2133 wpte->pte_val);
2134 }
2135 case KVM_MMU_OP_FLUSH_TLB: {
2136 struct kvm_mmu_op_flush_tlb *ftlb;
2137
2138 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
2139 if (!ftlb)
2140 return 0;
2141 return kvm_pv_mmu_flush_tlb(vcpu);
2142 }
2143 case KVM_MMU_OP_RELEASE_PT: {
2144 struct kvm_mmu_op_release_pt *rpt;
2145
2146 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
2147 if (!rpt)
2148 return 0;
2149 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
2150 }
2151 default: return 0;
2152 }
2153}
2154
2155int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
2156 gpa_t addr, unsigned long *ret)
2157{
2158 int r;
2159 struct kvm_pv_mmu_op_buffer buffer;
2160
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002161 buffer.ptr = buffer.buf;
2162 buffer.len = min_t(unsigned long, bytes, sizeof buffer.buf);
2163 buffer.processed = 0;
2164
2165 r = kvm_read_guest(vcpu->kvm, addr, buffer.buf, buffer.len);
2166 if (r)
2167 goto out;
2168
2169 while (buffer.len) {
2170 r = kvm_pv_mmu_op_one(vcpu, &buffer);
2171 if (r < 0)
2172 goto out;
2173 if (r == 0)
2174 break;
2175 }
2176
2177 r = 1;
2178out:
2179 *ret = buffer.processed;
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002180 return r;
2181}
2182
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002183#ifdef AUDIT
2184
2185static const char *audit_msg;
2186
2187static gva_t canonicalize(gva_t gva)
2188{
2189#ifdef CONFIG_X86_64
2190 gva = (long long)(gva << 16) >> 16;
2191#endif
2192 return gva;
2193}
2194
2195static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
2196 gva_t va, int level)
2197{
2198 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
2199 int i;
2200 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
2201
2202 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
2203 u64 ent = pt[i];
2204
Avi Kivityc7addb92007-09-16 18:58:32 +02002205 if (ent == shadow_trap_nonpresent_pte)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002206 continue;
2207
2208 va = canonicalize(va);
Avi Kivityc7addb92007-09-16 18:58:32 +02002209 if (level > 1) {
2210 if (ent == shadow_notrap_nonpresent_pte)
2211 printk(KERN_ERR "audit: (%s) nontrapping pte"
2212 " in nonleaf level: levels %d gva %lx"
2213 " level %d pte %llx\n", audit_msg,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002214 vcpu->arch.mmu.root_level, va, level, ent);
Avi Kivityc7addb92007-09-16 18:58:32 +02002215
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002216 audit_mappings_page(vcpu, ent, va, level - 1);
Avi Kivityc7addb92007-09-16 18:58:32 +02002217 } else {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002218 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
Anthony Liguori35149e22008-04-02 14:46:56 -05002219 hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002220
Avi Kivityc7addb92007-09-16 18:58:32 +02002221 if (is_shadow_present_pte(ent)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002222 && (ent & PT64_BASE_ADDR_MASK) != hpa)
Avi Kivityc7addb92007-09-16 18:58:32 +02002223 printk(KERN_ERR "xx audit error: (%s) levels %d"
2224 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002225 audit_msg, vcpu->arch.mmu.root_level,
Mike Dayd77c26f2007-10-08 09:02:08 -04002226 va, gpa, hpa, ent,
2227 is_shadow_present_pte(ent));
Avi Kivityc7addb92007-09-16 18:58:32 +02002228 else if (ent == shadow_notrap_nonpresent_pte
2229 && !is_error_hpa(hpa))
2230 printk(KERN_ERR "audit: (%s) notrap shadow,"
2231 " valid guest gva %lx\n", audit_msg, va);
Anthony Liguori35149e22008-04-02 14:46:56 -05002232 kvm_release_pfn_clean(pfn);
Avi Kivityc7addb92007-09-16 18:58:32 +02002233
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002234 }
2235 }
2236}
2237
2238static void audit_mappings(struct kvm_vcpu *vcpu)
2239{
Avi Kivity1ea252a2007-03-08 11:48:09 +02002240 unsigned i;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002241
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002242 if (vcpu->arch.mmu.root_level == 4)
2243 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002244 else
2245 for (i = 0; i < 4; ++i)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002246 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002247 audit_mappings_page(vcpu,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002248 vcpu->arch.mmu.pae_root[i],
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002249 i << 30,
2250 2);
2251}
2252
2253static int count_rmaps(struct kvm_vcpu *vcpu)
2254{
2255 int nmaps = 0;
2256 int i, j, k;
2257
2258 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
2259 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
2260 struct kvm_rmap_desc *d;
2261
2262 for (j = 0; j < m->npages; ++j) {
Izik Eidus290fc382007-09-27 14:11:22 +02002263 unsigned long *rmapp = &m->rmap[j];
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002264
Izik Eidus290fc382007-09-27 14:11:22 +02002265 if (!*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002266 continue;
Izik Eidus290fc382007-09-27 14:11:22 +02002267 if (!(*rmapp & 1)) {
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002268 ++nmaps;
2269 continue;
2270 }
Izik Eidus290fc382007-09-27 14:11:22 +02002271 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002272 while (d) {
2273 for (k = 0; k < RMAP_EXT; ++k)
2274 if (d->shadow_ptes[k])
2275 ++nmaps;
2276 else
2277 break;
2278 d = d->more;
2279 }
2280 }
2281 }
2282 return nmaps;
2283}
2284
2285static int count_writable_mappings(struct kvm_vcpu *vcpu)
2286{
2287 int nmaps = 0;
Avi Kivity4db35312007-11-21 15:28:32 +02002288 struct kvm_mmu_page *sp;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002289 int i;
2290
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002291 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
Avi Kivity4db35312007-11-21 15:28:32 +02002292 u64 *pt = sp->spt;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002293
Avi Kivity4db35312007-11-21 15:28:32 +02002294 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002295 continue;
2296
2297 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
2298 u64 ent = pt[i];
2299
2300 if (!(ent & PT_PRESENT_MASK))
2301 continue;
2302 if (!(ent & PT_WRITABLE_MASK))
2303 continue;
2304 ++nmaps;
2305 }
2306 }
2307 return nmaps;
2308}
2309
2310static void audit_rmap(struct kvm_vcpu *vcpu)
2311{
2312 int n_rmap = count_rmaps(vcpu);
2313 int n_actual = count_writable_mappings(vcpu);
2314
2315 if (n_rmap != n_actual)
2316 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002317 __func__, audit_msg, n_rmap, n_actual);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002318}
2319
2320static void audit_write_protection(struct kvm_vcpu *vcpu)
2321{
Avi Kivity4db35312007-11-21 15:28:32 +02002322 struct kvm_mmu_page *sp;
Izik Eidus290fc382007-09-27 14:11:22 +02002323 struct kvm_memory_slot *slot;
2324 unsigned long *rmapp;
2325 gfn_t gfn;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002326
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002327 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
Avi Kivity4db35312007-11-21 15:28:32 +02002328 if (sp->role.metaphysical)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002329 continue;
2330
Avi Kivity4db35312007-11-21 15:28:32 +02002331 slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
2332 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
Izik Eidus290fc382007-09-27 14:11:22 +02002333 rmapp = &slot->rmap[gfn - slot->base_gfn];
2334 if (*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002335 printk(KERN_ERR "%s: (%s) shadow page has writable"
2336 " mappings: gfn %lx role %x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002337 __func__, audit_msg, sp->gfn,
Avi Kivity4db35312007-11-21 15:28:32 +02002338 sp->role.word);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002339 }
2340}
2341
2342static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
2343{
2344 int olddbg = dbg;
2345
2346 dbg = 0;
2347 audit_msg = msg;
2348 audit_rmap(vcpu);
2349 audit_write_protection(vcpu);
2350 audit_mappings(vcpu);
2351 dbg = olddbg;
2352}
2353
2354#endif