blob: 2a36f7f7c4c74918e31c3854f5b91da86a503168 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
Avi Kivity6aa8b732006-12-10 02:21:36 -080019
Zhang Xiantao1d737c82007-12-14 09:35:10 +080020#include "mmu.h"
Avi Kivity6aa8b732006-12-10 02:21:36 -080021
Avi Kivityedf88412007-12-16 11:02:48 +020022#include <linux/kvm_host.h>
Avi Kivitye4956062007-06-28 14:15:57 -040023#include <linux/types.h>
24#include <linux/string.h>
25#include <linux/mm.h>
26#include <linux/highmem.h>
27#include <linux/module.h>
Izik Eidus448353c2007-11-26 14:08:14 +020028#include <linux/swap.h>
Marcelo Tosatti05da4552008-02-23 11:44:30 -030029#include <linux/hugetlb.h>
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050030#include <linux/compiler.h>
Avi Kivitye4956062007-06-28 14:15:57 -040031
32#include <asm/page.h>
33#include <asm/cmpxchg.h>
Avi Kivity4e542372007-11-21 14:08:40 +020034#include <asm/io.h>
Eduardo Habkost13673a92008-11-17 19:03:13 -020035#include <asm/vmx.h>
Avi Kivitye4956062007-06-28 14:15:57 -040036
Joerg Roedel18552672008-02-07 13:47:41 +010037/*
38 * When setting this variable to true it enables Two-Dimensional-Paging
39 * where the hardware walks 2 page tables:
40 * 1. the guest-virtual to guest-physical
41 * 2. while doing 1. it walks guest-physical to host-physical
42 * If the hardware supports that we don't need to do shadow paging.
43 */
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -050044bool tdp_enabled = false;
Joerg Roedel18552672008-02-07 13:47:41 +010045
Avi Kivity37a7d8b2007-01-05 16:36:56 -080046#undef MMU_DEBUG
47
48#undef AUDIT
49
50#ifdef AUDIT
51static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
52#else
53static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
54#endif
55
56#ifdef MMU_DEBUG
57
58#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
59#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
60
61#else
62
63#define pgprintk(x...) do { } while (0)
64#define rmap_printk(x...) do { } while (0)
65
66#endif
67
68#if defined(MMU_DEBUG) || defined(AUDIT)
Avi Kivity6ada8cc2008-06-22 16:45:24 +030069static int dbg = 0;
70module_param(dbg, bool, 0644);
Avi Kivity37a7d8b2007-01-05 16:36:56 -080071#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080072
Marcelo Tosatti582801a2008-09-23 13:18:41 -030073static int oos_shadow = 1;
74module_param(oos_shadow, bool, 0644);
75
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080076#ifndef MMU_DEBUG
77#define ASSERT(x) do { } while (0)
78#else
Avi Kivity6aa8b732006-12-10 02:21:36 -080079#define ASSERT(x) \
80 if (!(x)) { \
81 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
82 __FILE__, __LINE__, #x); \
83 }
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080084#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080085
Avi Kivity6aa8b732006-12-10 02:21:36 -080086#define PT_FIRST_AVAIL_BITS_SHIFT 9
87#define PT64_SECOND_AVAIL_BITS_SHIFT 52
88
Avi Kivity6aa8b732006-12-10 02:21:36 -080089#define VALID_PAGE(x) ((x) != INVALID_PAGE)
90
91#define PT64_LEVEL_BITS 9
92
93#define PT64_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -040094 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080095
96#define PT64_LEVEL_MASK(level) \
97 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
98
99#define PT64_INDEX(address, level)\
100 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
101
102
103#define PT32_LEVEL_BITS 10
104
105#define PT32_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -0400106 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800107
108#define PT32_LEVEL_MASK(level) \
109 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
110
111#define PT32_INDEX(address, level)\
112 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
113
114
Avi Kivity27aba762007-03-09 13:04:31 +0200115#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800116#define PT64_DIR_BASE_ADDR_MASK \
117 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
118
119#define PT32_BASE_ADDR_MASK PAGE_MASK
120#define PT32_DIR_BASE_ADDR_MASK \
121 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
122
Avi Kivity79539ce2007-11-21 02:06:21 +0200123#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
124 | PT64_NX_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800125
126#define PFERR_PRESENT_MASK (1U << 0)
127#define PFERR_WRITE_MASK (1U << 1)
128#define PFERR_USER_MASK (1U << 2)
Avi Kivity73b10872007-01-26 00:56:41 -0800129#define PFERR_FETCH_MASK (1U << 4)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800130
Avi Kivity6aa8b732006-12-10 02:21:36 -0800131#define PT_DIRECTORY_LEVEL 2
132#define PT_PAGE_TABLE_LEVEL 1
133
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800134#define RMAP_EXT 4
135
Avi Kivityfe135d22007-12-09 16:15:46 +0200136#define ACC_EXEC_MASK 1
137#define ACC_WRITE_MASK PT_WRITABLE_MASK
138#define ACC_USER_MASK PT_USER_MASK
139#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
140
Avi Kivity135f8c22008-08-21 17:49:56 +0300141#define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
142
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800143struct kvm_rmap_desc {
144 u64 *shadow_ptes[RMAP_EXT];
145 struct kvm_rmap_desc *more;
146};
147
Avi Kivity2d111232008-12-25 14:39:47 +0200148struct kvm_shadow_walk_iterator {
149 u64 addr;
150 hpa_t shadow_addr;
151 int level;
152 u64 *sptep;
153 unsigned index;
154};
155
156#define for_each_shadow_entry(_vcpu, _addr, _walker) \
157 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
158 shadow_walk_okay(&(_walker)); \
159 shadow_walk_next(&(_walker)))
160
161
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -0300162struct kvm_unsync_walk {
163 int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk);
164};
165
Marcelo Tosattiad8cfbe2008-09-23 13:18:36 -0300166typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp);
167
Avi Kivityb5a33a72007-04-15 16:31:09 +0300168static struct kmem_cache *pte_chain_cache;
169static struct kmem_cache *rmap_desc_cache;
Avi Kivityd3d25b02007-05-30 12:34:53 +0300170static struct kmem_cache *mmu_page_header_cache;
Avi Kivityb5a33a72007-04-15 16:31:09 +0300171
Avi Kivityc7addb92007-09-16 18:58:32 +0200172static u64 __read_mostly shadow_trap_nonpresent_pte;
173static u64 __read_mostly shadow_notrap_nonpresent_pte;
Sheng Yang7b523452008-04-25 21:13:50 +0800174static u64 __read_mostly shadow_base_present_pte;
175static u64 __read_mostly shadow_nx_mask;
176static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
177static u64 __read_mostly shadow_user_mask;
178static u64 __read_mostly shadow_accessed_mask;
179static u64 __read_mostly shadow_dirty_mask;
Sheng Yang64d4d522008-10-09 16:01:57 +0800180static u64 __read_mostly shadow_mt_mask;
Avi Kivityc7addb92007-09-16 18:58:32 +0200181
182void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
183{
184 shadow_trap_nonpresent_pte = trap_pte;
185 shadow_notrap_nonpresent_pte = notrap_pte;
186}
187EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
188
Sheng Yang7b523452008-04-25 21:13:50 +0800189void kvm_mmu_set_base_ptes(u64 base_pte)
190{
191 shadow_base_present_pte = base_pte;
192}
193EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
194
195void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
Sheng Yang64d4d522008-10-09 16:01:57 +0800196 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 mt_mask)
Sheng Yang7b523452008-04-25 21:13:50 +0800197{
198 shadow_user_mask = user_mask;
199 shadow_accessed_mask = accessed_mask;
200 shadow_dirty_mask = dirty_mask;
201 shadow_nx_mask = nx_mask;
202 shadow_x_mask = x_mask;
Sheng Yang64d4d522008-10-09 16:01:57 +0800203 shadow_mt_mask = mt_mask;
Sheng Yang7b523452008-04-25 21:13:50 +0800204}
205EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
206
Avi Kivity6aa8b732006-12-10 02:21:36 -0800207static int is_write_protection(struct kvm_vcpu *vcpu)
208{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800209 return vcpu->arch.cr0 & X86_CR0_WP;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800210}
211
212static int is_cpuid_PSE36(void)
213{
214 return 1;
215}
216
Avi Kivity73b10872007-01-26 00:56:41 -0800217static int is_nx(struct kvm_vcpu *vcpu)
218{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800219 return vcpu->arch.shadow_efer & EFER_NX;
Avi Kivity73b10872007-01-26 00:56:41 -0800220}
221
Avi Kivity6aa8b732006-12-10 02:21:36 -0800222static int is_present_pte(unsigned long pte)
223{
224 return pte & PT_PRESENT_MASK;
225}
226
Avi Kivityc7addb92007-09-16 18:58:32 +0200227static int is_shadow_present_pte(u64 pte)
228{
Avi Kivityc7addb92007-09-16 18:58:32 +0200229 return pte != shadow_trap_nonpresent_pte
230 && pte != shadow_notrap_nonpresent_pte;
231}
232
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300233static int is_large_pte(u64 pte)
234{
235 return pte & PT_PAGE_SIZE_MASK;
236}
237
Avi Kivity6aa8b732006-12-10 02:21:36 -0800238static int is_writeble_pte(unsigned long pte)
239{
240 return pte & PT_WRITABLE_MASK;
241}
242
Avi Kivitye3c5e7ec2007-10-11 12:32:30 +0200243static int is_dirty_pte(unsigned long pte)
244{
Sheng Yang7b523452008-04-25 21:13:50 +0800245 return pte & shadow_dirty_mask;
Avi Kivitye3c5e7ec2007-10-11 12:32:30 +0200246}
247
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800248static int is_rmap_pte(u64 pte)
249{
Avi Kivity4b1a80f2008-03-23 12:18:19 +0200250 return is_shadow_present_pte(pte);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800251}
252
Anthony Liguori35149e22008-04-02 14:46:56 -0500253static pfn_t spte_to_pfn(u64 pte)
Avi Kivity0b49ea82008-03-23 15:06:23 +0200254{
Anthony Liguori35149e22008-04-02 14:46:56 -0500255 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
Avi Kivity0b49ea82008-03-23 15:06:23 +0200256}
257
Avi Kivityda928522007-11-21 13:54:47 +0200258static gfn_t pse36_gfn_delta(u32 gpte)
259{
260 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
261
262 return (gpte & PT32_DIR_PSE36_MASK) << shift;
263}
264
Avi Kivitye663ee62007-05-31 15:46:04 +0300265static void set_shadow_pte(u64 *sptep, u64 spte)
266{
267#ifdef CONFIG_X86_64
268 set_64bit((unsigned long *)sptep, spte);
269#else
270 set_64bit((unsigned long long *)sptep, spte);
271#endif
272}
273
Avi Kivitye2dec932007-01-05 16:36:54 -0800274static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300275 struct kmem_cache *base_cache, int min)
Avi Kivity714b93d2007-01-05 16:36:53 -0800276{
277 void *obj;
278
279 if (cache->nobjs >= min)
Avi Kivitye2dec932007-01-05 16:36:54 -0800280 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800281 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300282 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
Avi Kivity714b93d2007-01-05 16:36:53 -0800283 if (!obj)
Avi Kivitye2dec932007-01-05 16:36:54 -0800284 return -ENOMEM;
Avi Kivity714b93d2007-01-05 16:36:53 -0800285 cache->objects[cache->nobjs++] = obj;
286 }
Avi Kivitye2dec932007-01-05 16:36:54 -0800287 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800288}
289
290static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
291{
292 while (mc->nobjs)
293 kfree(mc->objects[--mc->nobjs]);
294}
295
Avi Kivityc1158e62007-07-20 08:18:27 +0300296static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300297 int min)
Avi Kivityc1158e62007-07-20 08:18:27 +0300298{
299 struct page *page;
300
301 if (cache->nobjs >= min)
302 return 0;
303 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300304 page = alloc_page(GFP_KERNEL);
Avi Kivityc1158e62007-07-20 08:18:27 +0300305 if (!page)
306 return -ENOMEM;
307 set_page_private(page, 0);
308 cache->objects[cache->nobjs++] = page_address(page);
309 }
310 return 0;
311}
312
313static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
314{
315 while (mc->nobjs)
Avi Kivityc4d198d2007-07-21 09:06:46 +0300316 free_page((unsigned long)mc->objects[--mc->nobjs]);
Avi Kivityc1158e62007-07-20 08:18:27 +0300317}
318
Avi Kivity8c438502007-04-16 11:53:17 +0300319static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
320{
321 int r;
322
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800323 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300324 pte_chain_cache, 4);
325 if (r)
326 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800327 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
Marcelo Tosattic41ef342008-10-28 18:16:58 -0200328 rmap_desc_cache, 4);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300329 if (r)
330 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800331 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300332 if (r)
333 goto out;
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800334 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300335 mmu_page_header_cache, 4);
336out:
Avi Kivity8c438502007-04-16 11:53:17 +0300337 return r;
338}
339
Avi Kivity714b93d2007-01-05 16:36:53 -0800340static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
341{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800342 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
343 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
344 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
345 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800346}
347
348static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
349 size_t size)
350{
351 void *p;
352
353 BUG_ON(!mc->nobjs);
354 p = mc->objects[--mc->nobjs];
Avi Kivity714b93d2007-01-05 16:36:53 -0800355 return p;
356}
357
Avi Kivity714b93d2007-01-05 16:36:53 -0800358static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
359{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800360 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
Avi Kivity714b93d2007-01-05 16:36:53 -0800361 sizeof(struct kvm_pte_chain));
362}
363
Avi Kivity90cb0522007-07-17 13:04:56 +0300364static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
Avi Kivity714b93d2007-01-05 16:36:53 -0800365{
Avi Kivity90cb0522007-07-17 13:04:56 +0300366 kfree(pc);
Avi Kivity714b93d2007-01-05 16:36:53 -0800367}
368
369static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
370{
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800371 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
Avi Kivity714b93d2007-01-05 16:36:53 -0800372 sizeof(struct kvm_rmap_desc));
373}
374
Avi Kivity90cb0522007-07-17 13:04:56 +0300375static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
Avi Kivity714b93d2007-01-05 16:36:53 -0800376{
Avi Kivity90cb0522007-07-17 13:04:56 +0300377 kfree(rd);
Avi Kivity714b93d2007-01-05 16:36:53 -0800378}
379
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800380/*
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300381 * Return the pointer to the largepage write count for a given
382 * gfn, handling slots that are not large page aligned.
383 */
384static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
385{
386 unsigned long idx;
387
388 idx = (gfn / KVM_PAGES_PER_HPAGE) -
389 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
390 return &slot->lpage_info[idx].write_count;
391}
392
393static void account_shadowed(struct kvm *kvm, gfn_t gfn)
394{
395 int *write_count;
396
Izik Eidus28430992008-10-03 17:40:32 +0300397 gfn = unalias_gfn(kvm, gfn);
398 write_count = slot_largepage_idx(gfn,
399 gfn_to_memslot_unaliased(kvm, gfn));
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300400 *write_count += 1;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300401}
402
403static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
404{
405 int *write_count;
406
Izik Eidus28430992008-10-03 17:40:32 +0300407 gfn = unalias_gfn(kvm, gfn);
408 write_count = slot_largepage_idx(gfn,
409 gfn_to_memslot_unaliased(kvm, gfn));
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300410 *write_count -= 1;
411 WARN_ON(*write_count < 0);
412}
413
414static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
415{
Izik Eidus28430992008-10-03 17:40:32 +0300416 struct kvm_memory_slot *slot;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300417 int *largepage_idx;
418
Izik Eidus28430992008-10-03 17:40:32 +0300419 gfn = unalias_gfn(kvm, gfn);
420 slot = gfn_to_memslot_unaliased(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300421 if (slot) {
422 largepage_idx = slot_largepage_idx(gfn, slot);
423 return *largepage_idx;
424 }
425
426 return 1;
427}
428
429static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
430{
431 struct vm_area_struct *vma;
432 unsigned long addr;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300433 int ret = 0;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300434
435 addr = gfn_to_hva(kvm, gfn);
436 if (kvm_is_error_hva(addr))
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300437 return ret;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300438
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300439 down_read(&current->mm->mmap_sem);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300440 vma = find_vma(current->mm, addr);
441 if (vma && is_vm_hugetlb_page(vma))
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300442 ret = 1;
443 up_read(&current->mm->mmap_sem);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300444
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300445 return ret;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300446}
447
448static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
449{
450 struct kvm_memory_slot *slot;
451
452 if (has_wrprotected_page(vcpu->kvm, large_gfn))
453 return 0;
454
455 if (!host_largepage_backed(vcpu->kvm, large_gfn))
456 return 0;
457
458 slot = gfn_to_memslot(vcpu->kvm, large_gfn);
459 if (slot && slot->dirty_bitmap)
460 return 0;
461
462 return 1;
463}
464
465/*
Izik Eidus290fc382007-09-27 14:11:22 +0200466 * Take gfn and return the reverse mapping to it.
467 * Note: gfn must be unaliased before this function get called
468 */
469
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300470static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
Izik Eidus290fc382007-09-27 14:11:22 +0200471{
472 struct kvm_memory_slot *slot;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300473 unsigned long idx;
Izik Eidus290fc382007-09-27 14:11:22 +0200474
475 slot = gfn_to_memslot(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300476 if (!lpage)
477 return &slot->rmap[gfn - slot->base_gfn];
478
479 idx = (gfn / KVM_PAGES_PER_HPAGE) -
480 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
481
482 return &slot->lpage_info[idx].rmap_pde;
Izik Eidus290fc382007-09-27 14:11:22 +0200483}
484
485/*
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800486 * Reverse mapping data structures:
487 *
Izik Eidus290fc382007-09-27 14:11:22 +0200488 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
489 * that points to page_address(page).
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800490 *
Izik Eidus290fc382007-09-27 14:11:22 +0200491 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
492 * containing more mappings.
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800493 */
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300494static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800495{
Avi Kivity4db35312007-11-21 15:28:32 +0200496 struct kvm_mmu_page *sp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800497 struct kvm_rmap_desc *desc;
Izik Eidus290fc382007-09-27 14:11:22 +0200498 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800499 int i;
500
501 if (!is_rmap_pte(*spte))
502 return;
Izik Eidus290fc382007-09-27 14:11:22 +0200503 gfn = unalias_gfn(vcpu->kvm, gfn);
Avi Kivity4db35312007-11-21 15:28:32 +0200504 sp = page_header(__pa(spte));
505 sp->gfns[spte - sp->spt] = gfn;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300506 rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
Izik Eidus290fc382007-09-27 14:11:22 +0200507 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800508 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200509 *rmapp = (unsigned long)spte;
510 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800511 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
Avi Kivity714b93d2007-01-05 16:36:53 -0800512 desc = mmu_alloc_rmap_desc(vcpu);
Izik Eidus290fc382007-09-27 14:11:22 +0200513 desc->shadow_ptes[0] = (u64 *)*rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800514 desc->shadow_ptes[1] = spte;
Izik Eidus290fc382007-09-27 14:11:22 +0200515 *rmapp = (unsigned long)desc | 1;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800516 } else {
517 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200518 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800519 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
520 desc = desc->more;
521 if (desc->shadow_ptes[RMAP_EXT-1]) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800522 desc->more = mmu_alloc_rmap_desc(vcpu);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800523 desc = desc->more;
524 }
525 for (i = 0; desc->shadow_ptes[i]; ++i)
526 ;
527 desc->shadow_ptes[i] = spte;
528 }
529}
530
Izik Eidus290fc382007-09-27 14:11:22 +0200531static void rmap_desc_remove_entry(unsigned long *rmapp,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800532 struct kvm_rmap_desc *desc,
533 int i,
534 struct kvm_rmap_desc *prev_desc)
535{
536 int j;
537
538 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
539 ;
540 desc->shadow_ptes[i] = desc->shadow_ptes[j];
Al Viro11718b4d2007-02-09 16:39:20 +0000541 desc->shadow_ptes[j] = NULL;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800542 if (j != 0)
543 return;
544 if (!prev_desc && !desc->more)
Izik Eidus290fc382007-09-27 14:11:22 +0200545 *rmapp = (unsigned long)desc->shadow_ptes[0];
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800546 else
547 if (prev_desc)
548 prev_desc->more = desc->more;
549 else
Izik Eidus290fc382007-09-27 14:11:22 +0200550 *rmapp = (unsigned long)desc->more | 1;
Avi Kivity90cb0522007-07-17 13:04:56 +0300551 mmu_free_rmap_desc(desc);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800552}
553
Izik Eidus290fc382007-09-27 14:11:22 +0200554static void rmap_remove(struct kvm *kvm, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800555{
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800556 struct kvm_rmap_desc *desc;
557 struct kvm_rmap_desc *prev_desc;
Avi Kivity4db35312007-11-21 15:28:32 +0200558 struct kvm_mmu_page *sp;
Anthony Liguori35149e22008-04-02 14:46:56 -0500559 pfn_t pfn;
Izik Eidus290fc382007-09-27 14:11:22 +0200560 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800561 int i;
562
563 if (!is_rmap_pte(*spte))
564 return;
Avi Kivity4db35312007-11-21 15:28:32 +0200565 sp = page_header(__pa(spte));
Anthony Liguori35149e22008-04-02 14:46:56 -0500566 pfn = spte_to_pfn(*spte);
Sheng Yang7b523452008-04-25 21:13:50 +0800567 if (*spte & shadow_accessed_mask)
Anthony Liguori35149e22008-04-02 14:46:56 -0500568 kvm_set_pfn_accessed(pfn);
Izik Eidusb4231d62007-11-20 11:49:33 +0200569 if (is_writeble_pte(*spte))
Anthony Liguori35149e22008-04-02 14:46:56 -0500570 kvm_release_pfn_dirty(pfn);
Izik Eidusb4231d62007-11-20 11:49:33 +0200571 else
Anthony Liguori35149e22008-04-02 14:46:56 -0500572 kvm_release_pfn_clean(pfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300573 rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
Izik Eidus290fc382007-09-27 14:11:22 +0200574 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800575 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
576 BUG();
Izik Eidus290fc382007-09-27 14:11:22 +0200577 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800578 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200579 if ((u64 *)*rmapp != spte) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800580 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
581 spte, *spte);
582 BUG();
583 }
Izik Eidus290fc382007-09-27 14:11:22 +0200584 *rmapp = 0;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800585 } else {
586 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200587 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800588 prev_desc = NULL;
589 while (desc) {
590 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
591 if (desc->shadow_ptes[i] == spte) {
Izik Eidus290fc382007-09-27 14:11:22 +0200592 rmap_desc_remove_entry(rmapp,
Avi Kivity714b93d2007-01-05 16:36:53 -0800593 desc, i,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800594 prev_desc);
595 return;
596 }
597 prev_desc = desc;
598 desc = desc->more;
599 }
600 BUG();
601 }
602}
603
Izik Eidus98348e92007-10-16 14:42:30 +0200604static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
Avi Kivity374cbac2007-01-05 16:36:43 -0800605{
Avi Kivity374cbac2007-01-05 16:36:43 -0800606 struct kvm_rmap_desc *desc;
Izik Eidus98348e92007-10-16 14:42:30 +0200607 struct kvm_rmap_desc *prev_desc;
608 u64 *prev_spte;
609 int i;
610
611 if (!*rmapp)
612 return NULL;
613 else if (!(*rmapp & 1)) {
614 if (!spte)
615 return (u64 *)*rmapp;
616 return NULL;
617 }
618 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
619 prev_desc = NULL;
620 prev_spte = NULL;
621 while (desc) {
622 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
623 if (prev_spte == spte)
624 return desc->shadow_ptes[i];
625 prev_spte = desc->shadow_ptes[i];
626 }
627 desc = desc->more;
628 }
629 return NULL;
630}
631
Marcelo Tosattib1a36822008-12-01 22:32:03 -0200632static int rmap_write_protect(struct kvm *kvm, u64 gfn)
Izik Eidus98348e92007-10-16 14:42:30 +0200633{
Izik Eidus290fc382007-09-27 14:11:22 +0200634 unsigned long *rmapp;
Avi Kivity374cbac2007-01-05 16:36:43 -0800635 u64 *spte;
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800636 int write_protected = 0;
Avi Kivity374cbac2007-01-05 16:36:43 -0800637
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500638 gfn = unalias_gfn(kvm, gfn);
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300639 rmapp = gfn_to_rmap(kvm, gfn, 0);
Avi Kivity374cbac2007-01-05 16:36:43 -0800640
Izik Eidus98348e92007-10-16 14:42:30 +0200641 spte = rmap_next(kvm, rmapp, NULL);
642 while (spte) {
Avi Kivity374cbac2007-01-05 16:36:43 -0800643 BUG_ON(!spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800644 BUG_ON(!(*spte & PT_PRESENT_MASK));
Avi Kivity374cbac2007-01-05 16:36:43 -0800645 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800646 if (is_writeble_pte(*spte)) {
Izik Eidus9647c142007-10-16 14:43:46 +0200647 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
Eddie Dongcaa5b8a2007-12-18 06:08:27 +0800648 write_protected = 1;
649 }
Izik Eidus9647c142007-10-16 14:43:46 +0200650 spte = rmap_next(kvm, rmapp, spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800651 }
Izik Eidus855149a2008-03-20 18:17:24 +0200652 if (write_protected) {
Anthony Liguori35149e22008-04-02 14:46:56 -0500653 pfn_t pfn;
Izik Eidus855149a2008-03-20 18:17:24 +0200654
655 spte = rmap_next(kvm, rmapp, NULL);
Anthony Liguori35149e22008-04-02 14:46:56 -0500656 pfn = spte_to_pfn(*spte);
657 kvm_set_pfn_dirty(pfn);
Izik Eidus855149a2008-03-20 18:17:24 +0200658 }
659
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300660 /* check for huge page mappings */
661 rmapp = gfn_to_rmap(kvm, gfn, 1);
662 spte = rmap_next(kvm, rmapp, NULL);
663 while (spte) {
664 BUG_ON(!spte);
665 BUG_ON(!(*spte & PT_PRESENT_MASK));
666 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
667 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
668 if (is_writeble_pte(*spte)) {
669 rmap_remove(kvm, spte);
670 --kvm->stat.lpages;
671 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
Marcelo Tosatti6597ca02008-06-08 01:48:53 -0300672 spte = NULL;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300673 write_protected = 1;
674 }
675 spte = rmap_next(kvm, rmapp, spte);
676 }
677
Marcelo Tosattib1a36822008-12-01 22:32:03 -0200678 return write_protected;
Avi Kivity374cbac2007-01-05 16:36:43 -0800679}
680
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200681static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
682{
683 u64 *spte;
684 int need_tlb_flush = 0;
685
686 while ((spte = rmap_next(kvm, rmapp, NULL))) {
687 BUG_ON(!(*spte & PT_PRESENT_MASK));
688 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
689 rmap_remove(kvm, spte);
690 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
691 need_tlb_flush = 1;
692 }
693 return need_tlb_flush;
694}
695
696static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
697 int (*handler)(struct kvm *kvm, unsigned long *rmapp))
698{
699 int i;
700 int retval = 0;
701
702 /*
703 * If mmap_sem isn't taken, we can look the memslots with only
704 * the mmu_lock by skipping over the slots with userspace_addr == 0.
705 */
706 for (i = 0; i < kvm->nmemslots; i++) {
707 struct kvm_memory_slot *memslot = &kvm->memslots[i];
708 unsigned long start = memslot->userspace_addr;
709 unsigned long end;
710
711 /* mmu_lock protects userspace_addr */
712 if (!start)
713 continue;
714
715 end = start + (memslot->npages << PAGE_SHIFT);
716 if (hva >= start && hva < end) {
717 gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
718 retval |= handler(kvm, &memslot->rmap[gfn_offset]);
719 retval |= handler(kvm,
720 &memslot->lpage_info[
721 gfn_offset /
722 KVM_PAGES_PER_HPAGE].rmap_pde);
723 }
724 }
725
726 return retval;
727}
728
729int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
730{
731 return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
732}
733
734static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
735{
736 u64 *spte;
737 int young = 0;
738
Sheng Yang534e38b2008-09-08 15:12:30 +0800739 /* always return old for EPT */
740 if (!shadow_accessed_mask)
741 return 0;
742
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200743 spte = rmap_next(kvm, rmapp, NULL);
744 while (spte) {
745 int _young;
746 u64 _spte = *spte;
747 BUG_ON(!(_spte & PT_PRESENT_MASK));
748 _young = _spte & PT_ACCESSED_MASK;
749 if (_young) {
750 young = 1;
751 clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
752 }
753 spte = rmap_next(kvm, rmapp, spte);
754 }
755 return young;
756}
757
758int kvm_age_hva(struct kvm *kvm, unsigned long hva)
759{
760 return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
761}
762
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800763#ifdef MMU_DEBUG
Avi Kivity47ad8e62007-05-06 15:50:58 +0300764static int is_empty_shadow_page(u64 *spt)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800765{
Avi Kivity139bdb22007-01-05 16:36:50 -0800766 u64 *pos;
767 u64 *end;
768
Avi Kivity47ad8e62007-05-06 15:50:58 +0300769 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
Avi Kivity3c915512008-05-20 16:21:13 +0300770 if (is_shadow_present_pte(*pos)) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -0800771 printk(KERN_ERR "%s: %p %llx\n", __func__,
Avi Kivity139bdb22007-01-05 16:36:50 -0800772 pos, *pos);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800773 return 0;
Avi Kivity139bdb22007-01-05 16:36:50 -0800774 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800775 return 1;
776}
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800777#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800778
Avi Kivity4db35312007-11-21 15:28:32 +0200779static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivity260746c2007-01-05 16:36:49 -0800780{
Avi Kivity4db35312007-11-21 15:28:32 +0200781 ASSERT(is_empty_shadow_page(sp->spt));
782 list_del(&sp->link);
783 __free_page(virt_to_page(sp->spt));
784 __free_page(virt_to_page(sp->gfns));
785 kfree(sp);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800786 ++kvm->arch.n_free_mmu_pages;
Avi Kivity260746c2007-01-05 16:36:49 -0800787}
788
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800789static unsigned kvm_page_table_hashfn(gfn_t gfn)
790{
Dong, Eddie1ae0a132008-01-07 13:20:25 +0200791 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800792}
793
Avi Kivity25c0de22007-01-05 16:36:42 -0800794static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
795 u64 *parent_pte)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800796{
Avi Kivity4db35312007-11-21 15:28:32 +0200797 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800798
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800799 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
800 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
801 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
Avi Kivity4db35312007-11-21 15:28:32 +0200802 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800803 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -0200804 INIT_LIST_HEAD(&sp->oos_link);
Sheng Yang291f26b2008-10-16 17:30:57 +0800805 bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
Avi Kivity4db35312007-11-21 15:28:32 +0200806 sp->multimapped = 0;
807 sp->parent_pte = parent_pte;
Zhang Xiantaof05e70a2007-12-14 10:01:48 +0800808 --vcpu->kvm->arch.n_free_mmu_pages;
Avi Kivity4db35312007-11-21 15:28:32 +0200809 return sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800810}
811
Avi Kivity714b93d2007-01-05 16:36:53 -0800812static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +0200813 struct kvm_mmu_page *sp, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800814{
815 struct kvm_pte_chain *pte_chain;
816 struct hlist_node *node;
817 int i;
818
819 if (!parent_pte)
820 return;
Avi Kivity4db35312007-11-21 15:28:32 +0200821 if (!sp->multimapped) {
822 u64 *old = sp->parent_pte;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800823
824 if (!old) {
Avi Kivity4db35312007-11-21 15:28:32 +0200825 sp->parent_pte = parent_pte;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800826 return;
827 }
Avi Kivity4db35312007-11-21 15:28:32 +0200828 sp->multimapped = 1;
Avi Kivity714b93d2007-01-05 16:36:53 -0800829 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivity4db35312007-11-21 15:28:32 +0200830 INIT_HLIST_HEAD(&sp->parent_ptes);
831 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800832 pte_chain->parent_ptes[0] = old;
833 }
Avi Kivity4db35312007-11-21 15:28:32 +0200834 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800835 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
836 continue;
837 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
838 if (!pte_chain->parent_ptes[i]) {
839 pte_chain->parent_ptes[i] = parent_pte;
840 return;
841 }
842 }
Avi Kivity714b93d2007-01-05 16:36:53 -0800843 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800844 BUG_ON(!pte_chain);
Avi Kivity4db35312007-11-21 15:28:32 +0200845 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800846 pte_chain->parent_ptes[0] = parent_pte;
847}
848
Avi Kivity4db35312007-11-21 15:28:32 +0200849static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800850 u64 *parent_pte)
851{
852 struct kvm_pte_chain *pte_chain;
853 struct hlist_node *node;
854 int i;
855
Avi Kivity4db35312007-11-21 15:28:32 +0200856 if (!sp->multimapped) {
857 BUG_ON(sp->parent_pte != parent_pte);
858 sp->parent_pte = NULL;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800859 return;
860 }
Avi Kivity4db35312007-11-21 15:28:32 +0200861 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800862 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
863 if (!pte_chain->parent_ptes[i])
864 break;
865 if (pte_chain->parent_ptes[i] != parent_pte)
866 continue;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800867 while (i + 1 < NR_PTE_CHAIN_ENTRIES
868 && pte_chain->parent_ptes[i + 1]) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800869 pte_chain->parent_ptes[i]
870 = pte_chain->parent_ptes[i + 1];
871 ++i;
872 }
873 pte_chain->parent_ptes[i] = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800874 if (i == 0) {
875 hlist_del(&pte_chain->link);
Avi Kivity90cb0522007-07-17 13:04:56 +0300876 mmu_free_pte_chain(pte_chain);
Avi Kivity4db35312007-11-21 15:28:32 +0200877 if (hlist_empty(&sp->parent_ptes)) {
878 sp->multimapped = 0;
879 sp->parent_pte = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800880 }
881 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800882 return;
883 }
884 BUG();
885}
886
Marcelo Tosattiad8cfbe2008-09-23 13:18:36 -0300887
888static void mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
889 mmu_parent_walk_fn fn)
890{
891 struct kvm_pte_chain *pte_chain;
892 struct hlist_node *node;
893 struct kvm_mmu_page *parent_sp;
894 int i;
895
896 if (!sp->multimapped && sp->parent_pte) {
897 parent_sp = page_header(__pa(sp->parent_pte));
898 fn(vcpu, parent_sp);
899 mmu_parent_walk(vcpu, parent_sp, fn);
900 return;
901 }
902 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
903 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
904 if (!pte_chain->parent_ptes[i])
905 break;
906 parent_sp = page_header(__pa(pte_chain->parent_ptes[i]));
907 fn(vcpu, parent_sp);
908 mmu_parent_walk(vcpu, parent_sp, fn);
909 }
910}
911
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300912static void kvm_mmu_update_unsync_bitmap(u64 *spte)
913{
914 unsigned int index;
915 struct kvm_mmu_page *sp = page_header(__pa(spte));
916
917 index = spte - sp->spt;
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200918 if (!__test_and_set_bit(index, sp->unsync_child_bitmap))
919 sp->unsync_children++;
920 WARN_ON(!sp->unsync_children);
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300921}
922
923static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
924{
925 struct kvm_pte_chain *pte_chain;
926 struct hlist_node *node;
927 int i;
928
929 if (!sp->parent_pte)
930 return;
931
932 if (!sp->multimapped) {
933 kvm_mmu_update_unsync_bitmap(sp->parent_pte);
934 return;
935 }
936
937 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
938 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
939 if (!pte_chain->parent_ptes[i])
940 break;
941 kvm_mmu_update_unsync_bitmap(pte_chain->parent_ptes[i]);
942 }
943}
944
945static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
946{
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300947 kvm_mmu_update_parents_unsync(sp);
948 return 1;
949}
950
951static void kvm_mmu_mark_parents_unsync(struct kvm_vcpu *vcpu,
952 struct kvm_mmu_page *sp)
953{
954 mmu_parent_walk(vcpu, sp, unsync_walk_fn);
955 kvm_mmu_update_parents_unsync(sp);
956}
957
Avi Kivityd761a502008-05-29 14:55:03 +0300958static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
959 struct kvm_mmu_page *sp)
960{
961 int i;
962
963 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
964 sp->spt[i] = shadow_trap_nonpresent_pte;
965}
966
Marcelo Tosattie8bc2172008-09-23 13:18:33 -0300967static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
968 struct kvm_mmu_page *sp)
969{
970 return 1;
971}
972
Marcelo Tosattia7052892008-09-23 13:18:35 -0300973static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
974{
975}
976
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200977#define KVM_PAGE_ARRAY_NR 16
978
979struct kvm_mmu_pages {
980 struct mmu_page_and_offset {
981 struct kvm_mmu_page *sp;
982 unsigned int idx;
983 } page[KVM_PAGE_ARRAY_NR];
984 unsigned int nr;
985};
986
Marcelo Tosatti0074ff62008-09-23 13:18:40 -0300987#define for_each_unsync_children(bitmap, idx) \
988 for (idx = find_first_bit(bitmap, 512); \
989 idx < 512; \
990 idx = find_next_bit(bitmap, 512, idx+1))
991
Hannes Edercded19f2009-02-21 02:19:13 +0100992static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
993 int idx)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -0300994{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200995 int i;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -0300996
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -0200997 if (sp->unsync)
998 for (i=0; i < pvec->nr; i++)
999 if (pvec->page[i].sp == sp)
1000 return 0;
1001
1002 pvec->page[pvec->nr].sp = sp;
1003 pvec->page[pvec->nr].idx = idx;
1004 pvec->nr++;
1005 return (pvec->nr == KVM_PAGE_ARRAY_NR);
1006}
1007
1008static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1009 struct kvm_mmu_pages *pvec)
1010{
1011 int i, ret, nr_unsync_leaf = 0;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001012
Marcelo Tosatti0074ff62008-09-23 13:18:40 -03001013 for_each_unsync_children(sp->unsync_child_bitmap, i) {
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001014 u64 ent = sp->spt[i];
1015
Marcelo Tosatti87917232008-12-22 18:49:30 -02001016 if (is_shadow_present_pte(ent) && !is_large_pte(ent)) {
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001017 struct kvm_mmu_page *child;
1018 child = page_header(ent & PT64_BASE_ADDR_MASK);
1019
1020 if (child->unsync_children) {
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001021 if (mmu_pages_add(pvec, child, i))
1022 return -ENOSPC;
1023
1024 ret = __mmu_unsync_walk(child, pvec);
1025 if (!ret)
1026 __clear_bit(i, sp->unsync_child_bitmap);
1027 else if (ret > 0)
1028 nr_unsync_leaf += ret;
1029 else
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001030 return ret;
1031 }
1032
1033 if (child->unsync) {
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001034 nr_unsync_leaf++;
1035 if (mmu_pages_add(pvec, child, i))
1036 return -ENOSPC;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001037 }
1038 }
1039 }
1040
Marcelo Tosatti0074ff62008-09-23 13:18:40 -03001041 if (find_first_bit(sp->unsync_child_bitmap, 512) == 512)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001042 sp->unsync_children = 0;
1043
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001044 return nr_unsync_leaf;
1045}
1046
1047static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1048 struct kvm_mmu_pages *pvec)
1049{
1050 if (!sp->unsync_children)
1051 return 0;
1052
1053 mmu_pages_add(pvec, sp, 0);
1054 return __mmu_unsync_walk(sp, pvec);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001055}
1056
Avi Kivity4db35312007-11-21 15:28:32 +02001057static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001058{
1059 unsigned index;
1060 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +02001061 struct kvm_mmu_page *sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001062 struct hlist_node *node;
1063
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001064 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
Dong, Eddie1ae0a132008-01-07 13:20:25 +02001065 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001066 bucket = &kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +02001067 hlist_for_each_entry(sp, node, bucket, hash_link)
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001068 if (sp->gfn == gfn && !sp->role.direct
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001069 && !sp->role.invalid) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001070 pgprintk("%s: found role %x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001071 __func__, sp->role.word);
Avi Kivity4db35312007-11-21 15:28:32 +02001072 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001073 }
1074 return NULL;
1075}
1076
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001077static void kvm_unlink_unsync_global(struct kvm *kvm, struct kvm_mmu_page *sp)
1078{
1079 list_del(&sp->oos_link);
1080 --kvm->stat.mmu_unsync_global;
1081}
1082
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001083static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1084{
1085 WARN_ON(!sp->unsync);
1086 sp->unsync = 0;
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001087 if (sp->global)
1088 kvm_unlink_unsync_global(kvm, sp);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001089 --kvm->stat.mmu_unsync;
1090}
1091
1092static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp);
1093
1094static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1095{
1096 if (sp->role.glevels != vcpu->arch.mmu.root_level) {
1097 kvm_mmu_zap_page(vcpu->kvm, sp);
1098 return 1;
1099 }
1100
Marcelo Tosattib1a36822008-12-01 22:32:03 -02001101 if (rmap_write_protect(vcpu->kvm, sp->gfn))
1102 kvm_flush_remote_tlbs(vcpu->kvm);
Marcelo Tosatti0c0f40b2008-11-21 19:13:58 +01001103 kvm_unlink_unsync_page(vcpu->kvm, sp);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001104 if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1105 kvm_mmu_zap_page(vcpu->kvm, sp);
1106 return 1;
1107 }
1108
1109 kvm_mmu_flush_tlb(vcpu);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001110 return 0;
1111}
1112
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001113struct mmu_page_path {
1114 struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1115 unsigned int idx[PT64_ROOT_LEVEL-1];
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001116};
1117
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001118#define for_each_sp(pvec, sp, parents, i) \
1119 for (i = mmu_pages_next(&pvec, &parents, -1), \
1120 sp = pvec.page[i].sp; \
1121 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
1122 i = mmu_pages_next(&pvec, &parents, i))
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001123
Hannes Edercded19f2009-02-21 02:19:13 +01001124static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1125 struct mmu_page_path *parents,
1126 int i)
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001127{
1128 int n;
1129
1130 for (n = i+1; n < pvec->nr; n++) {
1131 struct kvm_mmu_page *sp = pvec->page[n].sp;
1132
1133 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1134 parents->idx[0] = pvec->page[n].idx;
1135 return n;
1136 }
1137
1138 parents->parent[sp->role.level-2] = sp;
1139 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1140 }
1141
1142 return n;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001143}
1144
Hannes Edercded19f2009-02-21 02:19:13 +01001145static void mmu_pages_clear_parents(struct mmu_page_path *parents)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001146{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001147 struct kvm_mmu_page *sp;
1148 unsigned int level = 0;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001149
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001150 do {
1151 unsigned int idx = parents->idx[level];
1152
1153 sp = parents->parent[level];
1154 if (!sp)
1155 return;
1156
1157 --sp->unsync_children;
1158 WARN_ON((int)sp->unsync_children < 0);
1159 __clear_bit(idx, sp->unsync_child_bitmap);
1160 level++;
1161 } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1162}
1163
1164static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1165 struct mmu_page_path *parents,
1166 struct kvm_mmu_pages *pvec)
1167{
1168 parents->parent[parent->role.level-1] = NULL;
1169 pvec->nr = 0;
1170}
1171
1172static void mmu_sync_children(struct kvm_vcpu *vcpu,
1173 struct kvm_mmu_page *parent)
1174{
1175 int i;
1176 struct kvm_mmu_page *sp;
1177 struct mmu_page_path parents;
1178 struct kvm_mmu_pages pages;
1179
1180 kvm_mmu_pages_init(parent, &parents, &pages);
1181 while (mmu_unsync_walk(parent, &pages)) {
Marcelo Tosattib1a36822008-12-01 22:32:03 -02001182 int protected = 0;
1183
1184 for_each_sp(pages, sp, parents, i)
1185 protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1186
1187 if (protected)
1188 kvm_flush_remote_tlbs(vcpu->kvm);
1189
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001190 for_each_sp(pages, sp, parents, i) {
1191 kvm_sync_page(vcpu, sp);
1192 mmu_pages_clear_parents(&parents);
1193 }
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001194 cond_resched_lock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001195 kvm_mmu_pages_init(parent, &parents, &pages);
1196 }
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001197}
1198
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001199static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1200 gfn_t gfn,
1201 gva_t gaddr,
1202 unsigned level,
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001203 int direct,
Avi Kivity41074d02007-12-09 17:00:02 +02001204 unsigned access,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02001205 u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001206{
1207 union kvm_mmu_page_role role;
1208 unsigned index;
1209 unsigned quadrant;
1210 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +02001211 struct kvm_mmu_page *sp;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001212 struct hlist_node *node, *tmp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001213
Avi Kivitya770f6f2008-12-21 19:20:09 +02001214 role = vcpu->arch.mmu.base_role;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001215 role.level = level;
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001216 role.direct = direct;
Avi Kivity41074d02007-12-09 17:00:02 +02001217 role.access = access;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001218 if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001219 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1220 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1221 role.quadrant = quadrant;
1222 }
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001223 pgprintk("%s: looking gfn %lx role %x\n", __func__,
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001224 gfn, role.word);
Dong, Eddie1ae0a132008-01-07 13:20:25 +02001225 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001226 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001227 hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
1228 if (sp->gfn == gfn) {
1229 if (sp->unsync)
1230 if (kvm_sync_page(vcpu, sp))
1231 continue;
1232
1233 if (sp->role.word != role.word)
1234 continue;
1235
Avi Kivity4db35312007-11-21 15:28:32 +02001236 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
Marcelo Tosatti0074ff62008-09-23 13:18:40 -03001237 if (sp->unsync_children) {
1238 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
1239 kvm_mmu_mark_parents_unsync(vcpu, sp);
1240 }
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001241 pgprintk("%s: found\n", __func__);
Avi Kivity4db35312007-11-21 15:28:32 +02001242 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001243 }
Avi Kivitydfc5aa02007-12-18 19:47:18 +02001244 ++vcpu->kvm->stat.mmu_cache_miss;
Avi Kivity4db35312007-11-21 15:28:32 +02001245 sp = kvm_mmu_alloc_page(vcpu, parent_pte);
1246 if (!sp)
1247 return sp;
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001248 pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
Avi Kivity4db35312007-11-21 15:28:32 +02001249 sp->gfn = gfn;
1250 sp->role = role;
Avi Kivitye2078312008-12-21 19:36:59 +02001251 sp->global = role.cr4_pge;
Avi Kivity4db35312007-11-21 15:28:32 +02001252 hlist_add_head(&sp->hash_link, bucket);
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001253 if (!direct) {
Marcelo Tosattib1a36822008-12-01 22:32:03 -02001254 if (rmap_write_protect(vcpu->kvm, gfn))
1255 kvm_flush_remote_tlbs(vcpu->kvm);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001256 account_shadowed(vcpu->kvm, gfn);
1257 }
Avi Kivity131d8272008-05-29 14:56:28 +03001258 if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1259 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1260 else
1261 nonpaging_prefetch_page(vcpu, sp);
Avi Kivity4db35312007-11-21 15:28:32 +02001262 return sp;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001263}
1264
Avi Kivity2d111232008-12-25 14:39:47 +02001265static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1266 struct kvm_vcpu *vcpu, u64 addr)
1267{
1268 iterator->addr = addr;
1269 iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1270 iterator->level = vcpu->arch.mmu.shadow_root_level;
1271 if (iterator->level == PT32E_ROOT_LEVEL) {
1272 iterator->shadow_addr
1273 = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1274 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1275 --iterator->level;
1276 if (!iterator->shadow_addr)
1277 iterator->level = 0;
1278 }
1279}
1280
1281static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
1282{
1283 if (iterator->level < PT_PAGE_TABLE_LEVEL)
1284 return false;
1285 iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
1286 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
1287 return true;
1288}
1289
1290static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
1291{
1292 iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
1293 --iterator->level;
1294}
1295
Avi Kivity90cb0522007-07-17 13:04:56 +03001296static void kvm_mmu_page_unlink_children(struct kvm *kvm,
Avi Kivity4db35312007-11-21 15:28:32 +02001297 struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -08001298{
Avi Kivity697fe2e2007-01-05 16:36:46 -08001299 unsigned i;
1300 u64 *pt;
1301 u64 ent;
1302
Avi Kivity4db35312007-11-21 15:28:32 +02001303 pt = sp->spt;
Avi Kivity697fe2e2007-01-05 16:36:46 -08001304
Avi Kivity4db35312007-11-21 15:28:32 +02001305 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
Avi Kivity697fe2e2007-01-05 16:36:46 -08001306 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
Avi Kivityc7addb92007-09-16 18:58:32 +02001307 if (is_shadow_present_pte(pt[i]))
Izik Eidus290fc382007-09-27 14:11:22 +02001308 rmap_remove(kvm, &pt[i]);
Avi Kivityc7addb92007-09-16 18:58:32 +02001309 pt[i] = shadow_trap_nonpresent_pte;
Avi Kivity697fe2e2007-01-05 16:36:46 -08001310 }
1311 return;
1312 }
1313
1314 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1315 ent = pt[i];
1316
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001317 if (is_shadow_present_pte(ent)) {
1318 if (!is_large_pte(ent)) {
1319 ent &= PT64_BASE_ADDR_MASK;
1320 mmu_page_remove_parent_pte(page_header(ent),
1321 &pt[i]);
1322 } else {
1323 --kvm->stat.lpages;
1324 rmap_remove(kvm, &pt[i]);
1325 }
1326 }
Avi Kivityc7addb92007-09-16 18:58:32 +02001327 pt[i] = shadow_trap_nonpresent_pte;
Avi Kivity697fe2e2007-01-05 16:36:46 -08001328 }
Avi Kivitya4360362007-01-05 16:36:45 -08001329}
1330
Avi Kivity4db35312007-11-21 15:28:32 +02001331static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001332{
Avi Kivity4db35312007-11-21 15:28:32 +02001333 mmu_page_remove_parent_pte(sp, parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -08001334}
1335
Avi Kivity12b7d282007-09-23 14:10:49 +02001336static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1337{
1338 int i;
1339
1340 for (i = 0; i < KVM_MAX_VCPUS; ++i)
1341 if (kvm->vcpus[i])
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001342 kvm->vcpus[i]->arch.last_pte_updated = NULL;
Avi Kivity12b7d282007-09-23 14:10:49 +02001343}
1344
Avi Kivity31aa2b42008-07-11 17:59:46 +03001345static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivitya4360362007-01-05 16:36:45 -08001346{
1347 u64 *parent_pte;
1348
Avi Kivity4db35312007-11-21 15:28:32 +02001349 while (sp->multimapped || sp->parent_pte) {
1350 if (!sp->multimapped)
1351 parent_pte = sp->parent_pte;
Avi Kivitya4360362007-01-05 16:36:45 -08001352 else {
1353 struct kvm_pte_chain *chain;
1354
Avi Kivity4db35312007-11-21 15:28:32 +02001355 chain = container_of(sp->parent_ptes.first,
Avi Kivitya4360362007-01-05 16:36:45 -08001356 struct kvm_pte_chain, link);
1357 parent_pte = chain->parent_ptes[0];
1358 }
Avi Kivity697fe2e2007-01-05 16:36:46 -08001359 BUG_ON(!parent_pte);
Avi Kivity4db35312007-11-21 15:28:32 +02001360 kvm_mmu_put_page(sp, parent_pte);
Avi Kivityc7addb92007-09-16 18:58:32 +02001361 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -08001362 }
Avi Kivity31aa2b42008-07-11 17:59:46 +03001363}
1364
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001365static int mmu_zap_unsync_children(struct kvm *kvm,
1366 struct kvm_mmu_page *parent)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001367{
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001368 int i, zapped = 0;
1369 struct mmu_page_path parents;
1370 struct kvm_mmu_pages pages;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001371
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001372 if (parent->role.level == PT_PAGE_TABLE_LEVEL)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001373 return 0;
Marcelo Tosatti60c8aec2008-12-01 22:32:02 -02001374
1375 kvm_mmu_pages_init(parent, &parents, &pages);
1376 while (mmu_unsync_walk(parent, &pages)) {
1377 struct kvm_mmu_page *sp;
1378
1379 for_each_sp(pages, sp, parents, i) {
1380 kvm_mmu_zap_page(kvm, sp);
1381 mmu_pages_clear_parents(&parents);
1382 }
1383 zapped += pages.nr;
1384 kvm_mmu_pages_init(parent, &parents, &pages);
1385 }
1386
1387 return zapped;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001388}
1389
Marcelo Tosatti07385412008-09-23 13:18:37 -03001390static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
Avi Kivity31aa2b42008-07-11 17:59:46 +03001391{
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001392 int ret;
Avi Kivity31aa2b42008-07-11 17:59:46 +03001393 ++kvm->stat.mmu_shadow_zapped;
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001394 ret = mmu_zap_unsync_children(kvm, sp);
Avi Kivity4db35312007-11-21 15:28:32 +02001395 kvm_mmu_page_unlink_children(kvm, sp);
Avi Kivity31aa2b42008-07-11 17:59:46 +03001396 kvm_mmu_unlink_parents(kvm, sp);
Avi Kivity5b5c6a52008-07-11 18:07:26 +03001397 kvm_flush_remote_tlbs(kvm);
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001398 if (!sp->role.invalid && !sp->role.direct)
Avi Kivity5b5c6a52008-07-11 18:07:26 +03001399 unaccount_shadowed(kvm, sp->gfn);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001400 if (sp->unsync)
1401 kvm_unlink_unsync_page(kvm, sp);
Avi Kivity4db35312007-11-21 15:28:32 +02001402 if (!sp->root_count) {
1403 hlist_del(&sp->hash_link);
1404 kvm_mmu_free_page(kvm, sp);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001405 } else {
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001406 sp->role.invalid = 1;
Avi Kivity5b5c6a52008-07-11 18:07:26 +03001407 list_move(&sp->link, &kvm->arch.active_mmu_pages);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001408 kvm_reload_remote_mmus(kvm);
1409 }
Avi Kivity12b7d282007-09-23 14:10:49 +02001410 kvm_mmu_reset_last_pte_updated(kvm);
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001411 return ret;
Avi Kivitya4360362007-01-05 16:36:45 -08001412}
1413
Izik Eidus82ce2c92007-10-02 18:52:55 +02001414/*
1415 * Changing the number of mmu pages allocated to the vm
1416 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
1417 */
1418void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
1419{
1420 /*
1421 * If we set the number of mmu pages to be smaller be than the
1422 * number of actived pages , we must to free some mmu pages before we
1423 * change the value
1424 */
1425
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001426 if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
Izik Eidus82ce2c92007-10-02 18:52:55 +02001427 kvm_nr_mmu_pages) {
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001428 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
1429 - kvm->arch.n_free_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02001430
1431 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
1432 struct kvm_mmu_page *page;
1433
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001434 page = container_of(kvm->arch.active_mmu_pages.prev,
Izik Eidus82ce2c92007-10-02 18:52:55 +02001435 struct kvm_mmu_page, link);
1436 kvm_mmu_zap_page(kvm, page);
1437 n_used_mmu_pages--;
1438 }
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001439 kvm->arch.n_free_mmu_pages = 0;
Izik Eidus82ce2c92007-10-02 18:52:55 +02001440 }
1441 else
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001442 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
1443 - kvm->arch.n_alloc_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02001444
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001445 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02001446}
1447
Anthony Liguorif67a46f2007-10-10 19:25:50 -05001448static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitya4360362007-01-05 16:36:45 -08001449{
1450 unsigned index;
1451 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +02001452 struct kvm_mmu_page *sp;
Avi Kivitya4360362007-01-05 16:36:45 -08001453 struct hlist_node *node, *n;
1454 int r;
1455
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001456 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
Avi Kivitya4360362007-01-05 16:36:45 -08001457 r = 0;
Dong, Eddie1ae0a132008-01-07 13:20:25 +02001458 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08001459 bucket = &kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +02001460 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001461 if (sp->gfn == gfn && !sp->role.direct) {
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001462 pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
Avi Kivity4db35312007-11-21 15:28:32 +02001463 sp->role.word);
Avi Kivitya4360362007-01-05 16:36:45 -08001464 r = 1;
Marcelo Tosatti07385412008-09-23 13:18:37 -03001465 if (kvm_mmu_zap_page(kvm, sp))
1466 n = bucket->first;
Avi Kivitya4360362007-01-05 16:36:45 -08001467 }
1468 return r;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001469}
1470
Anthony Liguorif67a46f2007-10-10 19:25:50 -05001471static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
Avi Kivity97a0a012007-05-31 15:08:29 +03001472{
Avi Kivity4677a3b2009-01-06 13:00:27 +02001473 unsigned index;
1474 struct hlist_head *bucket;
Avi Kivity4db35312007-11-21 15:28:32 +02001475 struct kvm_mmu_page *sp;
Avi Kivity4677a3b2009-01-06 13:00:27 +02001476 struct hlist_node *node, *nn;
Avi Kivity97a0a012007-05-31 15:08:29 +03001477
Avi Kivity4677a3b2009-01-06 13:00:27 +02001478 index = kvm_page_table_hashfn(gfn);
1479 bucket = &kvm->arch.mmu_page_hash[index];
1480 hlist_for_each_entry_safe(sp, node, nn, bucket, hash_link) {
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001481 if (sp->gfn == gfn && !sp->role.direct
Avi Kivity4677a3b2009-01-06 13:00:27 +02001482 && !sp->role.invalid) {
1483 pgprintk("%s: zap %lx %x\n",
1484 __func__, gfn, sp->role.word);
1485 kvm_mmu_zap_page(kvm, sp);
1486 }
Avi Kivity97a0a012007-05-31 15:08:29 +03001487 }
1488}
1489
Avi Kivity38c335f2007-11-21 14:20:22 +02001490static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001491{
Avi Kivity38c335f2007-11-21 14:20:22 +02001492 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
Avi Kivity4db35312007-11-21 15:28:32 +02001493 struct kvm_mmu_page *sp = page_header(__pa(pte));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001494
Sheng Yang291f26b2008-10-16 17:30:57 +08001495 __set_bit(slot, sp->slot_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001496}
1497
Marcelo Tosatti6844dec2008-09-23 13:18:38 -03001498static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1499{
1500 int i;
1501 u64 *pt = sp->spt;
1502
1503 if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1504 return;
1505
1506 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1507 if (pt[i] == shadow_notrap_nonpresent_pte)
1508 set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
1509 }
1510}
1511
Avi Kivity039576c2007-03-20 12:46:50 +02001512struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
1513{
Izik Eidus72dc67a2008-02-10 18:04:15 +02001514 struct page *page;
1515
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001516 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
Avi Kivity039576c2007-03-20 12:46:50 +02001517
1518 if (gpa == UNMAPPED_GVA)
1519 return NULL;
Izik Eidus72dc67a2008-02-10 18:04:15 +02001520
Izik Eidus72dc67a2008-02-10 18:04:15 +02001521 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
Izik Eidus72dc67a2008-02-10 18:04:15 +02001522
1523 return page;
Avi Kivity039576c2007-03-20 12:46:50 +02001524}
1525
Sheng Yang74be52e2008-10-09 16:01:56 +08001526/*
1527 * The function is based on mtrr_type_lookup() in
1528 * arch/x86/kernel/cpu/mtrr/generic.c
1529 */
1530static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1531 u64 start, u64 end)
1532{
1533 int i;
1534 u64 base, mask;
1535 u8 prev_match, curr_match;
1536 int num_var_ranges = KVM_NR_VAR_MTRR;
1537
1538 if (!mtrr_state->enabled)
1539 return 0xFF;
1540
1541 /* Make end inclusive end, instead of exclusive */
1542 end--;
1543
1544 /* Look in fixed ranges. Just return the type as per start */
1545 if (mtrr_state->have_fixed && (start < 0x100000)) {
1546 int idx;
1547
1548 if (start < 0x80000) {
1549 idx = 0;
1550 idx += (start >> 16);
1551 return mtrr_state->fixed_ranges[idx];
1552 } else if (start < 0xC0000) {
1553 idx = 1 * 8;
1554 idx += ((start - 0x80000) >> 14);
1555 return mtrr_state->fixed_ranges[idx];
1556 } else if (start < 0x1000000) {
1557 idx = 3 * 8;
1558 idx += ((start - 0xC0000) >> 12);
1559 return mtrr_state->fixed_ranges[idx];
1560 }
1561 }
1562
1563 /*
1564 * Look in variable ranges
1565 * Look of multiple ranges matching this address and pick type
1566 * as per MTRR precedence
1567 */
1568 if (!(mtrr_state->enabled & 2))
1569 return mtrr_state->def_type;
1570
1571 prev_match = 0xFF;
1572 for (i = 0; i < num_var_ranges; ++i) {
1573 unsigned short start_state, end_state;
1574
1575 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1576 continue;
1577
1578 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1579 (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1580 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1581 (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1582
1583 start_state = ((start & mask) == (base & mask));
1584 end_state = ((end & mask) == (base & mask));
1585 if (start_state != end_state)
1586 return 0xFE;
1587
1588 if ((start & mask) != (base & mask))
1589 continue;
1590
1591 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1592 if (prev_match == 0xFF) {
1593 prev_match = curr_match;
1594 continue;
1595 }
1596
1597 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1598 curr_match == MTRR_TYPE_UNCACHABLE)
1599 return MTRR_TYPE_UNCACHABLE;
1600
1601 if ((prev_match == MTRR_TYPE_WRBACK &&
1602 curr_match == MTRR_TYPE_WRTHROUGH) ||
1603 (prev_match == MTRR_TYPE_WRTHROUGH &&
1604 curr_match == MTRR_TYPE_WRBACK)) {
1605 prev_match = MTRR_TYPE_WRTHROUGH;
1606 curr_match = MTRR_TYPE_WRTHROUGH;
1607 }
1608
1609 if (prev_match != curr_match)
1610 return MTRR_TYPE_UNCACHABLE;
1611 }
1612
1613 if (prev_match != 0xFF)
1614 return prev_match;
1615
1616 return mtrr_state->def_type;
1617}
1618
1619static u8 get_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1620{
1621 u8 mtrr;
1622
1623 mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1624 (gfn << PAGE_SHIFT) + PAGE_SIZE);
1625 if (mtrr == 0xfe || mtrr == 0xff)
1626 mtrr = MTRR_TYPE_WRBACK;
1627 return mtrr;
1628}
1629
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001630static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1631{
1632 unsigned index;
1633 struct hlist_head *bucket;
1634 struct kvm_mmu_page *s;
1635 struct hlist_node *node, *n;
1636
1637 index = kvm_page_table_hashfn(sp->gfn);
1638 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1639 /* don't unsync if pagetable is shadowed with multiple roles */
1640 hlist_for_each_entry_safe(s, node, n, bucket, hash_link) {
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001641 if (s->gfn != sp->gfn || s->role.direct)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001642 continue;
1643 if (s->role.word != sp->role.word)
1644 return 1;
1645 }
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001646 ++vcpu->kvm->stat.mmu_unsync;
1647 sp->unsync = 1;
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001648
1649 if (sp->global) {
1650 list_add(&sp->oos_link, &vcpu->kvm->arch.oos_global_pages);
1651 ++vcpu->kvm->stat.mmu_unsync_global;
1652 } else
1653 kvm_mmu_mark_parents_unsync(vcpu, sp);
1654
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001655 mmu_convert_notrap(sp);
1656 return 0;
1657}
1658
1659static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1660 bool can_unsync)
1661{
1662 struct kvm_mmu_page *shadow;
1663
1664 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
1665 if (shadow) {
1666 if (shadow->role.level != PT_PAGE_TABLE_LEVEL)
1667 return 1;
1668 if (shadow->unsync)
1669 return 0;
Marcelo Tosatti582801a2008-09-23 13:18:41 -03001670 if (can_unsync && oos_shadow)
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001671 return kvm_unsync_page(vcpu, shadow);
1672 return 1;
1673 }
1674 return 0;
1675}
1676
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001677static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1678 unsigned pte_access, int user_fault,
1679 int write_fault, int dirty, int largepage,
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001680 int global, gfn_t gfn, pfn_t pfn, bool speculative,
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001681 bool can_unsync)
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001682{
1683 u64 spte;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001684 int ret = 0;
Sheng Yang64d4d522008-10-09 16:01:57 +08001685 u64 mt_mask = shadow_mt_mask;
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001686 struct kvm_mmu_page *sp = page_header(__pa(shadow_pte));
1687
1688 if (!global && sp->global) {
1689 sp->global = 0;
1690 if (sp->unsync) {
1691 kvm_unlink_unsync_global(vcpu->kvm, sp);
1692 kvm_mmu_mark_parents_unsync(vcpu, sp);
1693 }
1694 }
Sheng Yang64d4d522008-10-09 16:01:57 +08001695
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001696 /*
1697 * We don't set the accessed bit, since we sometimes want to see
1698 * whether the guest actually used the pte (in order to detect
1699 * demand paging).
1700 */
Sheng Yang7b523452008-04-25 21:13:50 +08001701 spte = shadow_base_present_pte | shadow_dirty_mask;
Avi Kivity947da532008-03-18 11:05:52 +02001702 if (!speculative)
Avi Kivity3201b5d2008-08-27 20:01:04 +03001703 spte |= shadow_accessed_mask;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001704 if (!dirty)
1705 pte_access &= ~ACC_WRITE_MASK;
Sheng Yang7b523452008-04-25 21:13:50 +08001706 if (pte_access & ACC_EXEC_MASK)
1707 spte |= shadow_x_mask;
1708 else
1709 spte |= shadow_nx_mask;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001710 if (pte_access & ACC_USER_MASK)
Sheng Yang7b523452008-04-25 21:13:50 +08001711 spte |= shadow_user_mask;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001712 if (largepage)
1713 spte |= PT_PAGE_SIZE_MASK;
Sheng Yang64d4d522008-10-09 16:01:57 +08001714 if (mt_mask) {
Sheng Yang2aaf69d2009-01-21 16:52:16 +08001715 if (!kvm_is_mmio_pfn(pfn)) {
1716 mt_mask = get_memory_type(vcpu, gfn) <<
1717 kvm_x86_ops->get_mt_mask_shift();
1718 mt_mask |= VMX_EPT_IGMT_BIT;
1719 } else
1720 mt_mask = MTRR_TYPE_UNCACHABLE <<
1721 kvm_x86_ops->get_mt_mask_shift();
Sheng Yang64d4d522008-10-09 16:01:57 +08001722 spte |= mt_mask;
1723 }
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001724
Anthony Liguori35149e22008-04-02 14:46:56 -05001725 spte |= (u64)pfn << PAGE_SHIFT;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001726
1727 if ((pte_access & ACC_WRITE_MASK)
1728 || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001729
Marcelo Tosatti38187c82008-09-23 13:18:32 -03001730 if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
1731 ret = 1;
1732 spte = shadow_trap_nonpresent_pte;
1733 goto set_pte;
1734 }
1735
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001736 spte |= PT_WRITABLE_MASK;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001737
Marcelo Tosattiecc55892008-11-25 15:58:07 +01001738 /*
1739 * Optimization: for pte sync, if spte was writable the hash
1740 * lookup is unnecessary (and expensive). Write protection
1741 * is responsibility of mmu_get_page / kvm_sync_page.
1742 * Same reasoning can be applied to dirty page accounting.
1743 */
1744 if (!can_unsync && is_writeble_pte(*shadow_pte))
1745 goto set_pte;
1746
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -03001747 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001748 pgprintk("%s: found shadow page for %lx, marking ro\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08001749 __func__, gfn);
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001750 ret = 1;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001751 pte_access &= ~ACC_WRITE_MASK;
Marcelo Tosattia378b4e2008-09-23 13:18:31 -03001752 if (is_writeble_pte(spte))
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001753 spte &= ~PT_WRITABLE_MASK;
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001754 }
1755 }
1756
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001757 if (pte_access & ACC_WRITE_MASK)
1758 mark_page_dirty(vcpu->kvm, gfn);
1759
Marcelo Tosatti38187c82008-09-23 13:18:32 -03001760set_pte:
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001761 set_shadow_pte(shadow_pte, spte);
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001762 return ret;
1763}
1764
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001765static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1766 unsigned pt_access, unsigned pte_access,
1767 int user_fault, int write_fault, int dirty,
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001768 int *ptwrite, int largepage, int global,
1769 gfn_t gfn, pfn_t pfn, bool speculative)
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001770{
1771 int was_rmapped = 0;
1772 int was_writeble = is_writeble_pte(*shadow_pte);
1773
1774 pgprintk("%s: spte %llx access %x write_fault %d"
1775 " user_fault %d gfn %lx\n",
1776 __func__, *shadow_pte, pt_access,
1777 write_fault, user_fault, gfn);
1778
1779 if (is_rmap_pte(*shadow_pte)) {
1780 /*
1781 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
1782 * the parent of the now unreachable PTE.
1783 */
1784 if (largepage && !is_large_pte(*shadow_pte)) {
1785 struct kvm_mmu_page *child;
1786 u64 pte = *shadow_pte;
1787
1788 child = page_header(pte & PT64_BASE_ADDR_MASK);
1789 mmu_page_remove_parent_pte(child, shadow_pte);
1790 } else if (pfn != spte_to_pfn(*shadow_pte)) {
1791 pgprintk("hfn old %lx new %lx\n",
1792 spte_to_pfn(*shadow_pte), pfn);
1793 rmap_remove(vcpu->kvm, shadow_pte);
Joerg Roedel6bed6b92009-02-18 14:08:59 +01001794 } else
1795 was_rmapped = 1;
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001796 }
1797 if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02001798 dirty, largepage, global, gfn, pfn, speculative, true)) {
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001799 if (write_fault)
1800 *ptwrite = 1;
Marcelo Tosattia378b4e2008-09-23 13:18:31 -03001801 kvm_x86_ops->tlb_flush(vcpu);
1802 }
Marcelo Tosatti1e73f9d2008-09-23 13:18:30 -03001803
1804 pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
1805 pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
1806 is_large_pte(*shadow_pte)? "2MB" : "4kB",
1807 is_present_pte(*shadow_pte)?"RW":"R", gfn,
1808 *shadow_pte, shadow_pte);
1809 if (!was_rmapped && is_large_pte(*shadow_pte))
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001810 ++vcpu->kvm->stat.lpages;
1811
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001812 page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
1813 if (!was_rmapped) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001814 rmap_add(vcpu, shadow_pte, gfn, largepage);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001815 if (!is_rmap_pte(*shadow_pte))
Anthony Liguori35149e22008-04-02 14:46:56 -05001816 kvm_release_pfn_clean(pfn);
Izik Eidus75e68e62008-01-12 23:49:09 +02001817 } else {
1818 if (was_writeble)
Anthony Liguori35149e22008-04-02 14:46:56 -05001819 kvm_release_pfn_dirty(pfn);
Izik Eidus75e68e62008-01-12 23:49:09 +02001820 else
Anthony Liguori35149e22008-04-02 14:46:56 -05001821 kvm_release_pfn_clean(pfn);
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001822 }
Avi Kivity1b7fcd32008-05-15 13:51:35 +03001823 if (speculative) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001824 vcpu->arch.last_pte_updated = shadow_pte;
Avi Kivity1b7fcd32008-05-15 13:51:35 +03001825 vcpu->arch.last_pte_gfn = gfn;
1826 }
Avi Kivity1c4f1fd2007-12-09 17:40:31 +02001827}
1828
Avi Kivity6aa8b732006-12-10 02:21:36 -08001829static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
1830{
1831}
1832
Joerg Roedel4d9976b2008-02-07 13:47:42 +01001833static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
Avi Kivity6c41f422008-08-26 16:16:08 +03001834 int largepage, gfn_t gfn, pfn_t pfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001835{
Avi Kivity9f652d22008-12-25 14:54:25 +02001836 struct kvm_shadow_walk_iterator iterator;
1837 struct kvm_mmu_page *sp;
1838 int pt_write = 0;
1839 gfn_t pseudo_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001840
Avi Kivity9f652d22008-12-25 14:54:25 +02001841 for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
1842 if (iterator.level == PT_PAGE_TABLE_LEVEL
1843 || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) {
1844 mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
1845 0, write, 1, &pt_write,
1846 largepage, 0, gfn, pfn, false);
1847 ++vcpu->stat.pf_fixed;
1848 break;
1849 }
1850
1851 if (*iterator.sptep == shadow_trap_nonpresent_pte) {
1852 pseudo_gfn = (iterator.addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
1853 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
1854 iterator.level - 1,
1855 1, ACC_ALL, iterator.sptep);
1856 if (!sp) {
1857 pgprintk("nonpaging_map: ENOMEM\n");
1858 kvm_release_pfn_clean(pfn);
1859 return -ENOMEM;
1860 }
1861
1862 set_shadow_pte(iterator.sptep,
1863 __pa(sp->spt)
1864 | PT_PRESENT_MASK | PT_WRITABLE_MASK
1865 | shadow_user_mask | shadow_x_mask);
1866 }
1867 }
1868 return pt_write;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001869}
1870
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001871static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
1872{
1873 int r;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001874 int largepage = 0;
Anthony Liguori35149e22008-04-02 14:46:56 -05001875 pfn_t pfn;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001876 unsigned long mmu_seq;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001877
Marcelo Tosatti05da4552008-02-23 11:44:30 -03001878 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1879 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1880 largepage = 1;
1881 }
1882
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001883 mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03001884 smp_rmb();
Anthony Liguori35149e22008-04-02 14:46:56 -05001885 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001886
Avi Kivityd196e342008-01-24 11:44:11 +02001887 /* mmio */
Anthony Liguori35149e22008-04-02 14:46:56 -05001888 if (is_error_pfn(pfn)) {
1889 kvm_release_pfn_clean(pfn);
Avi Kivityd196e342008-01-24 11:44:11 +02001890 return 1;
1891 }
1892
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001893 spin_lock(&vcpu->kvm->mmu_lock);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001894 if (mmu_notifier_retry(vcpu, mmu_seq))
1895 goto out_unlock;
Avi Kivityeb787d12007-12-31 15:27:49 +02001896 kvm_mmu_free_some_pages(vcpu);
Avi Kivity6c41f422008-08-26 16:16:08 +03001897 r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001898 spin_unlock(&vcpu->kvm->mmu_lock);
1899
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001900
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001901 return r;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02001902
1903out_unlock:
1904 spin_unlock(&vcpu->kvm->mmu_lock);
1905 kvm_release_pfn_clean(pfn);
1906 return 0;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05001907}
1908
1909
Avi Kivity17ac10a2007-01-05 16:36:40 -08001910static void mmu_free_roots(struct kvm_vcpu *vcpu)
1911{
1912 int i;
Avi Kivity4db35312007-11-21 15:28:32 +02001913 struct kvm_mmu_page *sp;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001914
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001915 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
Avi Kivity7b53aa52007-06-05 12:17:03 +03001916 return;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001917 spin_lock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001918 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1919 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001920
Avi Kivity4db35312007-11-21 15:28:32 +02001921 sp = page_header(root);
1922 --sp->root_count;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001923 if (!sp->root_count && sp->role.invalid)
1924 kvm_mmu_zap_page(vcpu->kvm, sp);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001925 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001926 spin_unlock(&vcpu->kvm->mmu_lock);
Avi Kivity17ac10a2007-01-05 16:36:40 -08001927 return;
1928 }
Avi Kivity17ac10a2007-01-05 16:36:40 -08001929 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001930 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08001931
Avi Kivity417726a2007-04-12 17:35:58 +03001932 if (root) {
Avi Kivity417726a2007-04-12 17:35:58 +03001933 root &= PT64_BASE_ADDR_MASK;
Avi Kivity4db35312007-11-21 15:28:32 +02001934 sp = page_header(root);
1935 --sp->root_count;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -05001936 if (!sp->root_count && sp->role.invalid)
1937 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity417726a2007-04-12 17:35:58 +03001938 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001939 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001940 }
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05001941 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001942 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001943}
1944
1945static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
1946{
1947 int i;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001948 gfn_t root_gfn;
Avi Kivity4db35312007-11-21 15:28:32 +02001949 struct kvm_mmu_page *sp;
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001950 int direct = 0;
Avi Kivity3bb65a22007-01-05 16:36:51 -08001951
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001952 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001953
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001954 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1955 hpa_t root = vcpu->arch.mmu.root_hpa;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001956
1957 ASSERT(!VALID_PAGE(root));
Joerg Roedelfb72d162008-02-07 13:47:44 +01001958 if (tdp_enabled)
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001959 direct = 1;
Avi Kivity4db35312007-11-21 15:28:32 +02001960 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001961 PT64_ROOT_LEVEL, direct,
Joerg Roedelfb72d162008-02-07 13:47:44 +01001962 ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02001963 root = __pa(sp->spt);
1964 ++sp->root_count;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001965 vcpu->arch.mmu.root_hpa = root;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001966 return;
1967 }
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001968 direct = !is_paging(vcpu);
Joerg Roedelfb72d162008-02-07 13:47:44 +01001969 if (tdp_enabled)
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001970 direct = 1;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001971 for (i = 0; i < 4; ++i) {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001972 hpa_t root = vcpu->arch.mmu.pae_root[i];
Avi Kivity17ac10a2007-01-05 16:36:40 -08001973
1974 ASSERT(!VALID_PAGE(root));
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001975 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1976 if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1977 vcpu->arch.mmu.pae_root[i] = 0;
Avi Kivity417726a2007-04-12 17:35:58 +03001978 continue;
1979 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001980 root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1981 } else if (vcpu->arch.mmu.root_level == 0)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001982 root_gfn = 0;
Avi Kivity4db35312007-11-21 15:28:32 +02001983 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02001984 PT32_ROOT_LEVEL, direct,
Avi Kivityf7d9c7b2008-02-26 22:12:10 +02001985 ACC_ALL, NULL);
Avi Kivity4db35312007-11-21 15:28:32 +02001986 root = __pa(sp->spt);
1987 ++sp->root_count;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001988 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001989 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001990 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
Avi Kivity17ac10a2007-01-05 16:36:40 -08001991}
1992
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03001993static void mmu_sync_roots(struct kvm_vcpu *vcpu)
1994{
1995 int i;
1996 struct kvm_mmu_page *sp;
1997
1998 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1999 return;
2000 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2001 hpa_t root = vcpu->arch.mmu.root_hpa;
2002 sp = page_header(root);
2003 mmu_sync_children(vcpu, sp);
2004 return;
2005 }
2006 for (i = 0; i < 4; ++i) {
2007 hpa_t root = vcpu->arch.mmu.pae_root[i];
2008
2009 if (root) {
2010 root &= PT64_BASE_ADDR_MASK;
2011 sp = page_header(root);
2012 mmu_sync_children(vcpu, sp);
2013 }
2014 }
2015}
2016
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02002017static void mmu_sync_global(struct kvm_vcpu *vcpu)
2018{
2019 struct kvm *kvm = vcpu->kvm;
2020 struct kvm_mmu_page *sp, *n;
2021
2022 list_for_each_entry_safe(sp, n, &kvm->arch.oos_global_pages, oos_link)
2023 kvm_sync_page(vcpu, sp);
2024}
2025
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03002026void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
2027{
2028 spin_lock(&vcpu->kvm->mmu_lock);
2029 mmu_sync_roots(vcpu);
2030 spin_unlock(&vcpu->kvm->mmu_lock);
2031}
2032
Marcelo Tosatti6cffe8c2008-12-01 22:32:04 -02002033void kvm_mmu_sync_global(struct kvm_vcpu *vcpu)
2034{
2035 spin_lock(&vcpu->kvm->mmu_lock);
2036 mmu_sync_global(vcpu);
2037 spin_unlock(&vcpu->kvm->mmu_lock);
2038}
2039
Avi Kivity6aa8b732006-12-10 02:21:36 -08002040static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
2041{
2042 return vaddr;
2043}
2044
2045static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
Avi Kivity3f3e7122007-11-21 14:54:16 +02002046 u32 error_code)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002047{
Avi Kivitye8332402007-12-09 18:43:00 +02002048 gfn_t gfn;
Avi Kivitye2dec932007-01-05 16:36:54 -08002049 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002050
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002051 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
Avi Kivitye2dec932007-01-05 16:36:54 -08002052 r = mmu_topup_memory_caches(vcpu);
2053 if (r)
2054 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -08002055
Avi Kivity6aa8b732006-12-10 02:21:36 -08002056 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002057 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002058
Avi Kivitye8332402007-12-09 18:43:00 +02002059 gfn = gva >> PAGE_SHIFT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002060
Avi Kivitye8332402007-12-09 18:43:00 +02002061 return nonpaging_map(vcpu, gva & PAGE_MASK,
2062 error_code & PFERR_WRITE_MASK, gfn);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002063}
2064
Joerg Roedelfb72d162008-02-07 13:47:44 +01002065static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
2066 u32 error_code)
2067{
Anthony Liguori35149e22008-04-02 14:46:56 -05002068 pfn_t pfn;
Joerg Roedelfb72d162008-02-07 13:47:44 +01002069 int r;
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002070 int largepage = 0;
2071 gfn_t gfn = gpa >> PAGE_SHIFT;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002072 unsigned long mmu_seq;
Joerg Roedelfb72d162008-02-07 13:47:44 +01002073
2074 ASSERT(vcpu);
2075 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2076
2077 r = mmu_topup_memory_caches(vcpu);
2078 if (r)
2079 return r;
2080
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002081 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
2082 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
2083 largepage = 1;
2084 }
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002085 mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03002086 smp_rmb();
Anthony Liguori35149e22008-04-02 14:46:56 -05002087 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Anthony Liguori35149e22008-04-02 14:46:56 -05002088 if (is_error_pfn(pfn)) {
2089 kvm_release_pfn_clean(pfn);
Joerg Roedelfb72d162008-02-07 13:47:44 +01002090 return 1;
2091 }
2092 spin_lock(&vcpu->kvm->mmu_lock);
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002093 if (mmu_notifier_retry(vcpu, mmu_seq))
2094 goto out_unlock;
Joerg Roedelfb72d162008-02-07 13:47:44 +01002095 kvm_mmu_free_some_pages(vcpu);
2096 r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
Avi Kivity6c41f422008-08-26 16:16:08 +03002097 largepage, gfn, pfn);
Joerg Roedelfb72d162008-02-07 13:47:44 +01002098 spin_unlock(&vcpu->kvm->mmu_lock);
Joerg Roedelfb72d162008-02-07 13:47:44 +01002099
2100 return r;
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002101
2102out_unlock:
2103 spin_unlock(&vcpu->kvm->mmu_lock);
2104 kvm_release_pfn_clean(pfn);
2105 return 0;
Joerg Roedelfb72d162008-02-07 13:47:44 +01002106}
2107
Avi Kivity6aa8b732006-12-10 02:21:36 -08002108static void nonpaging_free(struct kvm_vcpu *vcpu)
2109{
Avi Kivity17ac10a2007-01-05 16:36:40 -08002110 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002111}
2112
2113static int nonpaging_init_context(struct kvm_vcpu *vcpu)
2114{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002115 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002116
2117 context->new_cr3 = nonpaging_new_cr3;
2118 context->page_fault = nonpaging_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002119 context->gva_to_gpa = nonpaging_gva_to_gpa;
2120 context->free = nonpaging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02002121 context->prefetch_page = nonpaging_prefetch_page;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03002122 context->sync_page = nonpaging_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03002123 context->invlpg = nonpaging_invlpg;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002124 context->root_level = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002125 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03002126 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002127 return 0;
2128}
2129
Avi Kivityd835dfe2007-11-21 02:57:59 +02002130void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002131{
Avi Kivity1165f5f2007-04-19 17:27:43 +03002132 ++vcpu->stat.tlb_flush;
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03002133 kvm_x86_ops->tlb_flush(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002134}
2135
2136static void paging_new_cr3(struct kvm_vcpu *vcpu)
2137{
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002138 pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
Avi Kivitycea0f0e2007-01-05 16:36:43 -08002139 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002140}
2141
Avi Kivity6aa8b732006-12-10 02:21:36 -08002142static void inject_page_fault(struct kvm_vcpu *vcpu,
2143 u64 addr,
2144 u32 err_code)
2145{
Avi Kivityc3c91fe2007-11-25 14:04:58 +02002146 kvm_inject_page_fault(vcpu, addr, err_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002147}
2148
Avi Kivity6aa8b732006-12-10 02:21:36 -08002149static void paging_free(struct kvm_vcpu *vcpu)
2150{
2151 nonpaging_free(vcpu);
2152}
2153
2154#define PTTYPE 64
2155#include "paging_tmpl.h"
2156#undef PTTYPE
2157
2158#define PTTYPE 32
2159#include "paging_tmpl.h"
2160#undef PTTYPE
2161
Avi Kivity17ac10a2007-01-05 16:36:40 -08002162static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002163{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002164 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002165
2166 ASSERT(is_pae(vcpu));
2167 context->new_cr3 = paging_new_cr3;
2168 context->page_fault = paging64_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002169 context->gva_to_gpa = paging64_gva_to_gpa;
Avi Kivityc7addb92007-09-16 18:58:32 +02002170 context->prefetch_page = paging64_prefetch_page;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03002171 context->sync_page = paging64_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03002172 context->invlpg = paging64_invlpg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002173 context->free = paging_free;
Avi Kivity17ac10a2007-01-05 16:36:40 -08002174 context->root_level = level;
2175 context->shadow_root_level = level;
Avi Kivity17c3ba92007-06-04 15:58:30 +03002176 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002177 return 0;
2178}
2179
Avi Kivity17ac10a2007-01-05 16:36:40 -08002180static int paging64_init_context(struct kvm_vcpu *vcpu)
2181{
2182 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
2183}
2184
Avi Kivity6aa8b732006-12-10 02:21:36 -08002185static int paging32_init_context(struct kvm_vcpu *vcpu)
2186{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002187 struct kvm_mmu *context = &vcpu->arch.mmu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002188
2189 context->new_cr3 = paging_new_cr3;
2190 context->page_fault = paging32_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002191 context->gva_to_gpa = paging32_gva_to_gpa;
2192 context->free = paging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02002193 context->prefetch_page = paging32_prefetch_page;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03002194 context->sync_page = paging32_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03002195 context->invlpg = paging32_invlpg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002196 context->root_level = PT32_ROOT_LEVEL;
2197 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03002198 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002199 return 0;
2200}
2201
2202static int paging32E_init_context(struct kvm_vcpu *vcpu)
2203{
Avi Kivity17ac10a2007-01-05 16:36:40 -08002204 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002205}
2206
Joerg Roedelfb72d162008-02-07 13:47:44 +01002207static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2208{
2209 struct kvm_mmu *context = &vcpu->arch.mmu;
2210
2211 context->new_cr3 = nonpaging_new_cr3;
2212 context->page_fault = tdp_page_fault;
2213 context->free = nonpaging_free;
2214 context->prefetch_page = nonpaging_prefetch_page;
Marcelo Tosattie8bc2172008-09-23 13:18:33 -03002215 context->sync_page = nonpaging_sync_page;
Marcelo Tosattia7052892008-09-23 13:18:35 -03002216 context->invlpg = nonpaging_invlpg;
Sheng Yang67253af2008-04-25 10:20:22 +08002217 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
Joerg Roedelfb72d162008-02-07 13:47:44 +01002218 context->root_hpa = INVALID_PAGE;
2219
2220 if (!is_paging(vcpu)) {
2221 context->gva_to_gpa = nonpaging_gva_to_gpa;
2222 context->root_level = 0;
2223 } else if (is_long_mode(vcpu)) {
2224 context->gva_to_gpa = paging64_gva_to_gpa;
2225 context->root_level = PT64_ROOT_LEVEL;
2226 } else if (is_pae(vcpu)) {
2227 context->gva_to_gpa = paging64_gva_to_gpa;
2228 context->root_level = PT32E_ROOT_LEVEL;
2229 } else {
2230 context->gva_to_gpa = paging32_gva_to_gpa;
2231 context->root_level = PT32_ROOT_LEVEL;
2232 }
2233
2234 return 0;
2235}
2236
2237static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002238{
Avi Kivitya770f6f2008-12-21 19:20:09 +02002239 int r;
2240
Avi Kivity6aa8b732006-12-10 02:21:36 -08002241 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002242 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002243
2244 if (!is_paging(vcpu))
Avi Kivitya770f6f2008-12-21 19:20:09 +02002245 r = nonpaging_init_context(vcpu);
Avi Kivitya9058ec2006-12-29 16:49:37 -08002246 else if (is_long_mode(vcpu))
Avi Kivitya770f6f2008-12-21 19:20:09 +02002247 r = paging64_init_context(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002248 else if (is_pae(vcpu))
Avi Kivitya770f6f2008-12-21 19:20:09 +02002249 r = paging32E_init_context(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002250 else
Avi Kivitya770f6f2008-12-21 19:20:09 +02002251 r = paging32_init_context(vcpu);
2252
2253 vcpu->arch.mmu.base_role.glevels = vcpu->arch.mmu.root_level;
2254
2255 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002256}
2257
Joerg Roedelfb72d162008-02-07 13:47:44 +01002258static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2259{
Anthony Liguori35149e22008-04-02 14:46:56 -05002260 vcpu->arch.update_pte.pfn = bad_pfn;
2261
Joerg Roedelfb72d162008-02-07 13:47:44 +01002262 if (tdp_enabled)
2263 return init_kvm_tdp_mmu(vcpu);
2264 else
2265 return init_kvm_softmmu(vcpu);
2266}
2267
Avi Kivity6aa8b732006-12-10 02:21:36 -08002268static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2269{
2270 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002271 if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
2272 vcpu->arch.mmu.free(vcpu);
2273 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002274 }
2275}
2276
2277int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2278{
Avi Kivity17c3ba92007-06-04 15:58:30 +03002279 destroy_kvm_mmu(vcpu);
2280 return init_kvm_mmu(vcpu);
2281}
Eddie Dong8668a3c2007-10-10 14:26:45 +08002282EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
Avi Kivity17c3ba92007-06-04 15:58:30 +03002283
2284int kvm_mmu_load(struct kvm_vcpu *vcpu)
2285{
Avi Kivity714b93d2007-01-05 16:36:53 -08002286 int r;
2287
Avi Kivitye2dec932007-01-05 16:36:54 -08002288 r = mmu_topup_memory_caches(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03002289 if (r)
2290 goto out;
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002291 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivityeb787d12007-12-31 15:27:49 +02002292 kvm_mmu_free_some_pages(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03002293 mmu_alloc_roots(vcpu);
Marcelo Tosatti0ba73cd2008-09-23 13:18:34 -03002294 mmu_sync_roots(vcpu);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002295 spin_unlock(&vcpu->kvm->mmu_lock);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002296 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
Avi Kivity17c3ba92007-06-04 15:58:30 +03002297 kvm_mmu_flush_tlb(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08002298out:
2299 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002300}
Avi Kivity17c3ba92007-06-04 15:58:30 +03002301EXPORT_SYMBOL_GPL(kvm_mmu_load);
2302
2303void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2304{
2305 mmu_free_roots(vcpu);
2306}
Avi Kivity6aa8b732006-12-10 02:21:36 -08002307
Avi Kivity09072da2007-05-01 14:16:52 +03002308static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +02002309 struct kvm_mmu_page *sp,
Avi Kivityac1b7142007-03-08 17:13:32 +02002310 u64 *spte)
2311{
2312 u64 pte;
2313 struct kvm_mmu_page *child;
2314
2315 pte = *spte;
Avi Kivityc7addb92007-09-16 18:58:32 +02002316 if (is_shadow_present_pte(pte)) {
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002317 if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
2318 is_large_pte(pte))
Izik Eidus290fc382007-09-27 14:11:22 +02002319 rmap_remove(vcpu->kvm, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02002320 else {
2321 child = page_header(pte & PT64_BASE_ADDR_MASK);
Avi Kivity90cb0522007-07-17 13:04:56 +03002322 mmu_page_remove_parent_pte(child, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02002323 }
2324 }
Avi Kivityc7addb92007-09-16 18:58:32 +02002325 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002326 if (is_large_pte(pte))
2327 --vcpu->kvm->stat.lpages;
Avi Kivityac1b7142007-03-08 17:13:32 +02002328}
2329
Avi Kivity00284252007-05-01 16:53:31 +03002330static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
Avi Kivity4db35312007-11-21 15:28:32 +02002331 struct kvm_mmu_page *sp,
Avi Kivity00284252007-05-01 16:53:31 +03002332 u64 *spte,
Dong, Eddie489f1d62008-01-07 11:14:20 +02002333 const void *new)
Avi Kivity00284252007-05-01 16:53:31 +03002334{
Marcelo Tosatti30945382008-06-11 20:32:40 -03002335 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
2336 if (!vcpu->arch.update_pte.largepage ||
2337 sp->role.glevels == PT32_ROOT_LEVEL) {
2338 ++vcpu->kvm->stat.mmu_pde_zapped;
2339 return;
2340 }
2341 }
Avi Kivity00284252007-05-01 16:53:31 +03002342
Avi Kivity4cee5762007-11-18 16:37:07 +02002343 ++vcpu->kvm->stat.mmu_pte_updated;
Avi Kivity4db35312007-11-21 15:28:32 +02002344 if (sp->role.glevels == PT32_ROOT_LEVEL)
Dong, Eddie489f1d62008-01-07 11:14:20 +02002345 paging32_update_pte(vcpu, sp, spte, new);
Avi Kivity00284252007-05-01 16:53:31 +03002346 else
Dong, Eddie489f1d62008-01-07 11:14:20 +02002347 paging64_update_pte(vcpu, sp, spte, new);
Avi Kivity00284252007-05-01 16:53:31 +03002348}
2349
Avi Kivity79539ce2007-11-21 02:06:21 +02002350static bool need_remote_flush(u64 old, u64 new)
2351{
2352 if (!is_shadow_present_pte(old))
2353 return false;
2354 if (!is_shadow_present_pte(new))
2355 return true;
2356 if ((old ^ new) & PT64_BASE_ADDR_MASK)
2357 return true;
2358 old ^= PT64_NX_MASK;
2359 new ^= PT64_NX_MASK;
2360 return (old & ~new & PT64_PERM_MASK) != 0;
2361}
2362
2363static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
2364{
2365 if (need_remote_flush(old, new))
2366 kvm_flush_remote_tlbs(vcpu->kvm);
2367 else
2368 kvm_mmu_flush_tlb(vcpu);
2369}
2370
Avi Kivity12b7d282007-09-23 14:10:49 +02002371static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
2372{
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002373 u64 *spte = vcpu->arch.last_pte_updated;
Avi Kivity12b7d282007-09-23 14:10:49 +02002374
Sheng Yang7b523452008-04-25 21:13:50 +08002375 return !!(spte && (*spte & shadow_accessed_mask));
Avi Kivity12b7d282007-09-23 14:10:49 +02002376}
2377
Avi Kivityd7824ff2007-12-30 12:29:05 +02002378static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2379 const u8 *new, int bytes)
2380{
2381 gfn_t gfn;
2382 int r;
2383 u64 gpte = 0;
Anthony Liguori35149e22008-04-02 14:46:56 -05002384 pfn_t pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02002385
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002386 vcpu->arch.update_pte.largepage = 0;
2387
Avi Kivityd7824ff2007-12-30 12:29:05 +02002388 if (bytes != 4 && bytes != 8)
2389 return;
2390
2391 /*
2392 * Assume that the pte write on a page table of the same type
2393 * as the current vcpu paging mode. This is nearly always true
2394 * (might be false while changing modes). Note it is verified later
2395 * by update_pte().
2396 */
2397 if (is_pae(vcpu)) {
2398 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
2399 if ((bytes == 4) && (gpa % 4 == 0)) {
2400 r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
2401 if (r)
2402 return;
2403 memcpy((void *)&gpte + (gpa % 8), new, 4);
2404 } else if ((bytes == 8) && (gpa % 8 == 0)) {
2405 memcpy((void *)&gpte, new, 8);
2406 }
2407 } else {
2408 if ((bytes == 4) && (gpa % 4 == 0))
2409 memcpy((void *)&gpte, new, 4);
2410 }
2411 if (!is_present_pte(gpte))
2412 return;
2413 gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
Izik Eidus72dc67a2008-02-10 18:04:15 +02002414
Marcelo Tosatti05da4552008-02-23 11:44:30 -03002415 if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
2416 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
2417 vcpu->arch.update_pte.largepage = 1;
2418 }
Andrea Arcangelie930bff2008-07-25 16:24:52 +02002419 vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03002420 smp_rmb();
Anthony Liguori35149e22008-04-02 14:46:56 -05002421 pfn = gfn_to_pfn(vcpu->kvm, gfn);
Izik Eidus72dc67a2008-02-10 18:04:15 +02002422
Anthony Liguori35149e22008-04-02 14:46:56 -05002423 if (is_error_pfn(pfn)) {
2424 kvm_release_pfn_clean(pfn);
Avi Kivityd196e342008-01-24 11:44:11 +02002425 return;
2426 }
Avi Kivityd7824ff2007-12-30 12:29:05 +02002427 vcpu->arch.update_pte.gfn = gfn;
Anthony Liguori35149e22008-04-02 14:46:56 -05002428 vcpu->arch.update_pte.pfn = pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02002429}
2430
Avi Kivity1b7fcd32008-05-15 13:51:35 +03002431static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
2432{
2433 u64 *spte = vcpu->arch.last_pte_updated;
2434
2435 if (spte
2436 && vcpu->arch.last_pte_gfn == gfn
2437 && shadow_accessed_mask
2438 && !(*spte & shadow_accessed_mask)
2439 && is_shadow_present_pte(*spte))
2440 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
2441}
2442
Avi Kivity09072da2007-05-01 14:16:52 +03002443void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
Marcelo Tosattiad218f82008-12-01 22:32:05 -02002444 const u8 *new, int bytes,
2445 bool guest_initiated)
Avi Kivityda4a00f2007-01-05 16:36:44 -08002446{
Avi Kivity9b7a0322007-01-05 16:36:45 -08002447 gfn_t gfn = gpa >> PAGE_SHIFT;
Avi Kivity4db35312007-11-21 15:28:32 +02002448 struct kvm_mmu_page *sp;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002449 struct hlist_node *node, *n;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002450 struct hlist_head *bucket;
2451 unsigned index;
Dong, Eddie489f1d62008-01-07 11:14:20 +02002452 u64 entry, gentry;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002453 u64 *spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002454 unsigned offset = offset_in_page(gpa);
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002455 unsigned pte_size;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002456 unsigned page_offset;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002457 unsigned misaligned;
Avi Kivityfce06572007-05-01 16:44:05 +03002458 unsigned quadrant;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002459 int level;
Avi Kivity86a5ba02007-01-05 16:36:50 -08002460 int flooded = 0;
Avi Kivityac1b7142007-03-08 17:13:32 +02002461 int npte;
Dong, Eddie489f1d62008-01-07 11:14:20 +02002462 int r;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002463
Harvey Harrisonb8688d52008-03-03 12:59:56 -08002464 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
Avi Kivityd7824ff2007-12-30 12:29:05 +02002465 mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002466 spin_lock(&vcpu->kvm->mmu_lock);
Avi Kivity1b7fcd32008-05-15 13:51:35 +03002467 kvm_mmu_access_page(vcpu, gfn);
Avi Kivityeb787d12007-12-31 15:27:49 +02002468 kvm_mmu_free_some_pages(vcpu);
Avi Kivity4cee5762007-11-18 16:37:07 +02002469 ++vcpu->kvm->stat.mmu_pte_write;
Avi Kivityc7addb92007-09-16 18:58:32 +02002470 kvm_mmu_audit(vcpu, "pre pte write");
Marcelo Tosattiad218f82008-12-01 22:32:05 -02002471 if (guest_initiated) {
2472 if (gfn == vcpu->arch.last_pt_write_gfn
2473 && !last_updated_pte_accessed(vcpu)) {
2474 ++vcpu->arch.last_pt_write_count;
2475 if (vcpu->arch.last_pt_write_count >= 3)
2476 flooded = 1;
2477 } else {
2478 vcpu->arch.last_pt_write_gfn = gfn;
2479 vcpu->arch.last_pt_write_count = 1;
2480 vcpu->arch.last_pte_updated = NULL;
2481 }
Avi Kivity86a5ba02007-01-05 16:36:50 -08002482 }
Dong, Eddie1ae0a132008-01-07 13:20:25 +02002483 index = kvm_page_table_hashfn(gfn);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002484 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
Avi Kivity4db35312007-11-21 15:28:32 +02002485 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02002486 if (sp->gfn != gfn || sp->role.direct || sp->role.invalid)
Avi Kivity9b7a0322007-01-05 16:36:45 -08002487 continue;
Avi Kivity4db35312007-11-21 15:28:32 +02002488 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002489 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
Avi Kivitye925c5b2007-04-30 14:47:02 +03002490 misaligned |= bytes < 4;
Avi Kivity86a5ba02007-01-05 16:36:50 -08002491 if (misaligned || flooded) {
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002492 /*
2493 * Misaligned accesses are too much trouble to fix
2494 * up; also, they usually indicate a page is not used
2495 * as a page table.
Avi Kivity86a5ba02007-01-05 16:36:50 -08002496 *
2497 * If we're seeing too many writes to a page,
2498 * it may no longer be a page table, or we may be
2499 * forking, in which case it is better to unmap the
2500 * page.
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002501 */
2502 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
Avi Kivity4db35312007-11-21 15:28:32 +02002503 gpa, bytes, sp->role.word);
Marcelo Tosatti07385412008-09-23 13:18:37 -03002504 if (kvm_mmu_zap_page(vcpu->kvm, sp))
2505 n = bucket->first;
Avi Kivity4cee5762007-11-18 16:37:07 +02002506 ++vcpu->kvm->stat.mmu_flooded;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08002507 continue;
2508 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08002509 page_offset = offset;
Avi Kivity4db35312007-11-21 15:28:32 +02002510 level = sp->role.level;
Avi Kivityac1b7142007-03-08 17:13:32 +02002511 npte = 1;
Avi Kivity4db35312007-11-21 15:28:32 +02002512 if (sp->role.glevels == PT32_ROOT_LEVEL) {
Avi Kivityac1b7142007-03-08 17:13:32 +02002513 page_offset <<= 1; /* 32->64 */
2514 /*
2515 * A 32-bit pde maps 4MB while the shadow pdes map
2516 * only 2MB. So we need to double the offset again
2517 * and zap two pdes instead of one.
2518 */
2519 if (level == PT32_ROOT_LEVEL) {
Avi Kivity6b8d0f92007-04-18 11:18:18 +03002520 page_offset &= ~7; /* kill rounding error */
Avi Kivityac1b7142007-03-08 17:13:32 +02002521 page_offset <<= 1;
2522 npte = 2;
2523 }
Avi Kivityfce06572007-05-01 16:44:05 +03002524 quadrant = page_offset >> PAGE_SHIFT;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002525 page_offset &= ~PAGE_MASK;
Avi Kivity4db35312007-11-21 15:28:32 +02002526 if (quadrant != sp->role.quadrant)
Avi Kivityfce06572007-05-01 16:44:05 +03002527 continue;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002528 }
Avi Kivity4db35312007-11-21 15:28:32 +02002529 spte = &sp->spt[page_offset / sizeof(*spte)];
Dong, Eddie489f1d62008-01-07 11:14:20 +02002530 if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
2531 gentry = 0;
2532 r = kvm_read_guest_atomic(vcpu->kvm,
2533 gpa & ~(u64)(pte_size - 1),
2534 &gentry, pte_size);
2535 new = (const void *)&gentry;
2536 if (r < 0)
2537 new = NULL;
2538 }
Avi Kivityac1b7142007-03-08 17:13:32 +02002539 while (npte--) {
Avi Kivity79539ce2007-11-21 02:06:21 +02002540 entry = *spte;
Avi Kivity4db35312007-11-21 15:28:32 +02002541 mmu_pte_write_zap_pte(vcpu, sp, spte);
Dong, Eddie489f1d62008-01-07 11:14:20 +02002542 if (new)
2543 mmu_pte_write_new_pte(vcpu, sp, spte, new);
Avi Kivity79539ce2007-11-21 02:06:21 +02002544 mmu_pte_write_flush_tlb(vcpu, entry, *spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02002545 ++spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08002546 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08002547 }
Avi Kivityc7addb92007-09-16 18:58:32 +02002548 kvm_mmu_audit(vcpu, "post pte write");
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002549 spin_unlock(&vcpu->kvm->mmu_lock);
Anthony Liguori35149e22008-04-02 14:46:56 -05002550 if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
2551 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
2552 vcpu->arch.update_pte.pfn = bad_pfn;
Avi Kivityd7824ff2007-12-30 12:29:05 +02002553 }
Avi Kivityda4a00f2007-01-05 16:36:44 -08002554}
2555
Avi Kivitya4360362007-01-05 16:36:45 -08002556int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2557{
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002558 gpa_t gpa;
2559 int r;
Avi Kivitya4360362007-01-05 16:36:45 -08002560
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002561 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002562
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002563 spin_lock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002564 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002565 spin_unlock(&vcpu->kvm->mmu_lock);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05002566 return r;
Avi Kivitya4360362007-01-05 16:36:45 -08002567}
Avi Kivity577bdc42008-07-19 08:57:05 +03002568EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
Avi Kivitya4360362007-01-05 16:36:45 -08002569
Avi Kivity22d95b12007-09-14 20:26:06 +03002570void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
Avi Kivityebeace82007-01-05 16:36:47 -08002571{
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002572 while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
Avi Kivity4db35312007-11-21 15:28:32 +02002573 struct kvm_mmu_page *sp;
Avi Kivityebeace82007-01-05 16:36:47 -08002574
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002575 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
Avi Kivity4db35312007-11-21 15:28:32 +02002576 struct kvm_mmu_page, link);
2577 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity4cee5762007-11-18 16:37:07 +02002578 ++vcpu->kvm->stat.mmu_recycled;
Avi Kivityebeace82007-01-05 16:36:47 -08002579 }
2580}
Avi Kivityebeace82007-01-05 16:36:47 -08002581
Avi Kivity30677142007-10-28 18:48:59 +02002582int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
2583{
2584 int r;
2585 enum emulation_result er;
2586
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002587 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
Avi Kivity30677142007-10-28 18:48:59 +02002588 if (r < 0)
2589 goto out;
2590
2591 if (!r) {
2592 r = 1;
2593 goto out;
2594 }
2595
Avi Kivityb733bfb2007-10-28 18:52:05 +02002596 r = mmu_topup_memory_caches(vcpu);
2597 if (r)
2598 goto out;
2599
Avi Kivity30677142007-10-28 18:48:59 +02002600 er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
Avi Kivity30677142007-10-28 18:48:59 +02002601
2602 switch (er) {
2603 case EMULATE_DONE:
2604 return 1;
2605 case EMULATE_DO_MMIO:
2606 ++vcpu->stat.mmio_exits;
2607 return 0;
2608 case EMULATE_FAIL:
2609 kvm_report_emulation_failure(vcpu, "pagetable");
2610 return 1;
2611 default:
2612 BUG();
2613 }
2614out:
Avi Kivity30677142007-10-28 18:48:59 +02002615 return r;
2616}
2617EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
2618
Marcelo Tosattia7052892008-09-23 13:18:35 -03002619void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
2620{
Marcelo Tosattia7052892008-09-23 13:18:35 -03002621 vcpu->arch.mmu.invlpg(vcpu, gva);
Marcelo Tosattia7052892008-09-23 13:18:35 -03002622 kvm_mmu_flush_tlb(vcpu);
2623 ++vcpu->stat.invlpg;
2624}
2625EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
2626
Joerg Roedel18552672008-02-07 13:47:41 +01002627void kvm_enable_tdp(void)
2628{
2629 tdp_enabled = true;
2630}
2631EXPORT_SYMBOL_GPL(kvm_enable_tdp);
2632
Joerg Roedel5f4cb662008-07-14 20:36:36 +02002633void kvm_disable_tdp(void)
2634{
2635 tdp_enabled = false;
2636}
2637EXPORT_SYMBOL_GPL(kvm_disable_tdp);
2638
Avi Kivity6aa8b732006-12-10 02:21:36 -08002639static void free_mmu_pages(struct kvm_vcpu *vcpu)
2640{
Avi Kivity4db35312007-11-21 15:28:32 +02002641 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002642
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002643 while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
2644 sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
Avi Kivity4db35312007-11-21 15:28:32 +02002645 struct kvm_mmu_page, link);
2646 kvm_mmu_zap_page(vcpu->kvm, sp);
Avi Kivity8d2d73b2008-06-04 18:42:24 +03002647 cond_resched();
Avi Kivityf51234c2007-01-05 16:36:52 -08002648 }
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002649 free_page((unsigned long)vcpu->arch.mmu.pae_root);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002650}
2651
2652static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
2653{
Avi Kivity17ac10a2007-01-05 16:36:40 -08002654 struct page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002655 int i;
2656
2657 ASSERT(vcpu);
2658
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002659 if (vcpu->kvm->arch.n_requested_mmu_pages)
2660 vcpu->kvm->arch.n_free_mmu_pages =
2661 vcpu->kvm->arch.n_requested_mmu_pages;
Izik Eidus82ce2c92007-10-02 18:52:55 +02002662 else
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002663 vcpu->kvm->arch.n_free_mmu_pages =
2664 vcpu->kvm->arch.n_alloc_mmu_pages;
Avi Kivity17ac10a2007-01-05 16:36:40 -08002665 /*
2666 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
2667 * Therefore we need to allocate shadow page tables in the first
2668 * 4GB of memory, which happens to fit the DMA32 zone.
2669 */
2670 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
2671 if (!page)
2672 goto error_1;
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002673 vcpu->arch.mmu.pae_root = page_address(page);
Avi Kivity17ac10a2007-01-05 16:36:40 -08002674 for (i = 0; i < 4; ++i)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002675 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
Avi Kivity17ac10a2007-01-05 16:36:40 -08002676
Avi Kivity6aa8b732006-12-10 02:21:36 -08002677 return 0;
2678
2679error_1:
2680 free_mmu_pages(vcpu);
2681 return -ENOMEM;
2682}
2683
Ingo Molnar8018c272006-12-29 16:50:01 -08002684int kvm_mmu_create(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002685{
Avi Kivity6aa8b732006-12-10 02:21:36 -08002686 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002687 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08002688
Ingo Molnar8018c272006-12-29 16:50:01 -08002689 return alloc_mmu_pages(vcpu);
2690}
Avi Kivity6aa8b732006-12-10 02:21:36 -08002691
Ingo Molnar8018c272006-12-29 16:50:01 -08002692int kvm_mmu_setup(struct kvm_vcpu *vcpu)
2693{
2694 ASSERT(vcpu);
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002695 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
Avi Kivity2c264952006-12-22 01:05:28 -08002696
Ingo Molnar8018c272006-12-29 16:50:01 -08002697 return init_kvm_mmu(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002698}
2699
2700void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
2701{
2702 ASSERT(vcpu);
2703
2704 destroy_kvm_mmu(vcpu);
2705 free_mmu_pages(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08002706 mmu_free_memory_caches(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002707}
2708
Avi Kivity90cb0522007-07-17 13:04:56 +03002709void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002710{
Avi Kivity4db35312007-11-21 15:28:32 +02002711 struct kvm_mmu_page *sp;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002712
Avi Kivity2245a282008-08-27 16:32:24 +03002713 spin_lock(&kvm->mmu_lock);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002714 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
Avi Kivity6aa8b732006-12-10 02:21:36 -08002715 int i;
2716 u64 *pt;
2717
Sheng Yang291f26b2008-10-16 17:30:57 +08002718 if (!test_bit(slot, sp->slot_bitmap))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002719 continue;
2720
Avi Kivity4db35312007-11-21 15:28:32 +02002721 pt = sp->spt;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002722 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2723 /* avoid RMW */
Izik Eidus9647c142007-10-16 14:43:46 +02002724 if (pt[i] & PT_WRITABLE_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002725 pt[i] &= ~PT_WRITABLE_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002726 }
Avi Kivity171d5952008-08-27 16:40:51 +03002727 kvm_flush_remote_tlbs(kvm);
Avi Kivity2245a282008-08-27 16:32:24 +03002728 spin_unlock(&kvm->mmu_lock);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002729}
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002730
Avi Kivity90cb0522007-07-17 13:04:56 +03002731void kvm_mmu_zap_all(struct kvm *kvm)
Dor Laore0fa8262007-03-30 13:06:33 +03002732{
Avi Kivity4db35312007-11-21 15:28:32 +02002733 struct kvm_mmu_page *sp, *node;
Dor Laore0fa8262007-03-30 13:06:33 +03002734
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002735 spin_lock(&kvm->mmu_lock);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08002736 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
Marcelo Tosatti07385412008-09-23 13:18:37 -03002737 if (kvm_mmu_zap_page(kvm, sp))
2738 node = container_of(kvm->arch.active_mmu_pages.next,
2739 struct kvm_mmu_page, link);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -05002740 spin_unlock(&kvm->mmu_lock);
Dor Laore0fa8262007-03-30 13:06:33 +03002741
Avi Kivity90cb0522007-07-17 13:04:56 +03002742 kvm_flush_remote_tlbs(kvm);
Dor Laore0fa8262007-03-30 13:06:33 +03002743}
2744
Harvey Harrison8b2cf732008-04-27 12:14:13 -07002745static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
Izik Eidus3ee16c82008-03-30 15:17:21 +03002746{
2747 struct kvm_mmu_page *page;
2748
2749 page = container_of(kvm->arch.active_mmu_pages.prev,
2750 struct kvm_mmu_page, link);
2751 kvm_mmu_zap_page(kvm, page);
2752}
2753
2754static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
2755{
2756 struct kvm *kvm;
2757 struct kvm *kvm_freed = NULL;
2758 int cache_count = 0;
2759
2760 spin_lock(&kvm_lock);
2761
2762 list_for_each_entry(kvm, &vm_list, vm_list) {
2763 int npages;
2764
Marcelo Tosatti5a4c9282008-07-03 18:33:02 -03002765 if (!down_read_trylock(&kvm->slots_lock))
2766 continue;
Izik Eidus3ee16c82008-03-30 15:17:21 +03002767 spin_lock(&kvm->mmu_lock);
2768 npages = kvm->arch.n_alloc_mmu_pages -
2769 kvm->arch.n_free_mmu_pages;
2770 cache_count += npages;
2771 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
2772 kvm_mmu_remove_one_alloc_mmu_page(kvm);
2773 cache_count--;
2774 kvm_freed = kvm;
2775 }
2776 nr_to_scan--;
2777
2778 spin_unlock(&kvm->mmu_lock);
Marcelo Tosatti5a4c9282008-07-03 18:33:02 -03002779 up_read(&kvm->slots_lock);
Izik Eidus3ee16c82008-03-30 15:17:21 +03002780 }
2781 if (kvm_freed)
2782 list_move_tail(&kvm_freed->vm_list, &vm_list);
2783
2784 spin_unlock(&kvm_lock);
2785
2786 return cache_count;
2787}
2788
2789static struct shrinker mmu_shrinker = {
2790 .shrink = mmu_shrink,
2791 .seeks = DEFAULT_SEEKS * 10,
2792};
2793
Ingo Molnar2ddfd202008-05-22 10:37:48 +02002794static void mmu_destroy_caches(void)
Avi Kivityb5a33a72007-04-15 16:31:09 +03002795{
2796 if (pte_chain_cache)
2797 kmem_cache_destroy(pte_chain_cache);
2798 if (rmap_desc_cache)
2799 kmem_cache_destroy(rmap_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +03002800 if (mmu_page_header_cache)
2801 kmem_cache_destroy(mmu_page_header_cache);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002802}
2803
Izik Eidus3ee16c82008-03-30 15:17:21 +03002804void kvm_mmu_module_exit(void)
2805{
2806 mmu_destroy_caches();
2807 unregister_shrinker(&mmu_shrinker);
2808}
2809
Avi Kivityb5a33a72007-04-15 16:31:09 +03002810int kvm_mmu_module_init(void)
2811{
2812 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
2813 sizeof(struct kvm_pte_chain),
Paul Mundt20c2df82007-07-20 10:11:58 +09002814 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002815 if (!pte_chain_cache)
2816 goto nomem;
2817 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
2818 sizeof(struct kvm_rmap_desc),
Paul Mundt20c2df82007-07-20 10:11:58 +09002819 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03002820 if (!rmap_desc_cache)
2821 goto nomem;
2822
Avi Kivityd3d25b02007-05-30 12:34:53 +03002823 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
2824 sizeof(struct kvm_mmu_page),
Paul Mundt20c2df82007-07-20 10:11:58 +09002825 0, 0, NULL);
Avi Kivityd3d25b02007-05-30 12:34:53 +03002826 if (!mmu_page_header_cache)
2827 goto nomem;
2828
Izik Eidus3ee16c82008-03-30 15:17:21 +03002829 register_shrinker(&mmu_shrinker);
2830
Avi Kivityb5a33a72007-04-15 16:31:09 +03002831 return 0;
2832
2833nomem:
Izik Eidus3ee16c82008-03-30 15:17:21 +03002834 mmu_destroy_caches();
Avi Kivityb5a33a72007-04-15 16:31:09 +03002835 return -ENOMEM;
2836}
2837
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08002838/*
2839 * Caculate mmu pages needed for kvm.
2840 */
2841unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
2842{
2843 int i;
2844 unsigned int nr_mmu_pages;
2845 unsigned int nr_pages = 0;
2846
2847 for (i = 0; i < kvm->nmemslots; i++)
2848 nr_pages += kvm->memslots[i].npages;
2849
2850 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
2851 nr_mmu_pages = max(nr_mmu_pages,
2852 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
2853
2854 return nr_mmu_pages;
2855}
2856
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002857static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2858 unsigned len)
2859{
2860 if (len > buffer->len)
2861 return NULL;
2862 return buffer->ptr;
2863}
2864
2865static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2866 unsigned len)
2867{
2868 void *ret;
2869
2870 ret = pv_mmu_peek_buffer(buffer, len);
2871 if (!ret)
2872 return ret;
2873 buffer->ptr += len;
2874 buffer->len -= len;
2875 buffer->processed += len;
2876 return ret;
2877}
2878
2879static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
2880 gpa_t addr, gpa_t value)
2881{
2882 int bytes = 8;
2883 int r;
2884
2885 if (!is_long_mode(vcpu) && !is_pae(vcpu))
2886 bytes = 4;
2887
2888 r = mmu_topup_memory_caches(vcpu);
2889 if (r)
2890 return r;
2891
Marcelo Tosatti3200f402008-03-29 20:17:59 -03002892 if (!emulator_write_phys(vcpu, addr, &value, bytes))
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002893 return -EFAULT;
2894
2895 return 1;
2896}
2897
2898static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2899{
2900 kvm_x86_ops->tlb_flush(vcpu);
Marcelo Tosatti6ad9f152008-10-15 07:45:08 -02002901 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002902 return 1;
2903}
2904
2905static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
2906{
2907 spin_lock(&vcpu->kvm->mmu_lock);
2908 mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
2909 spin_unlock(&vcpu->kvm->mmu_lock);
2910 return 1;
2911}
2912
2913static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
2914 struct kvm_pv_mmu_op_buffer *buffer)
2915{
2916 struct kvm_mmu_op_header *header;
2917
2918 header = pv_mmu_peek_buffer(buffer, sizeof *header);
2919 if (!header)
2920 return 0;
2921 switch (header->op) {
2922 case KVM_MMU_OP_WRITE_PTE: {
2923 struct kvm_mmu_op_write_pte *wpte;
2924
2925 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
2926 if (!wpte)
2927 return 0;
2928 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
2929 wpte->pte_val);
2930 }
2931 case KVM_MMU_OP_FLUSH_TLB: {
2932 struct kvm_mmu_op_flush_tlb *ftlb;
2933
2934 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
2935 if (!ftlb)
2936 return 0;
2937 return kvm_pv_mmu_flush_tlb(vcpu);
2938 }
2939 case KVM_MMU_OP_RELEASE_PT: {
2940 struct kvm_mmu_op_release_pt *rpt;
2941
2942 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
2943 if (!rpt)
2944 return 0;
2945 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
2946 }
2947 default: return 0;
2948 }
2949}
2950
2951int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
2952 gpa_t addr, unsigned long *ret)
2953{
2954 int r;
Dave Hansen6ad18fb2008-08-11 10:01:49 -07002955 struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002956
Dave Hansen6ad18fb2008-08-11 10:01:49 -07002957 buffer->ptr = buffer->buf;
2958 buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
2959 buffer->processed = 0;
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002960
Dave Hansen6ad18fb2008-08-11 10:01:49 -07002961 r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002962 if (r)
2963 goto out;
2964
Dave Hansen6ad18fb2008-08-11 10:01:49 -07002965 while (buffer->len) {
2966 r = kvm_pv_mmu_op_one(vcpu, buffer);
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002967 if (r < 0)
2968 goto out;
2969 if (r == 0)
2970 break;
2971 }
2972
2973 r = 1;
2974out:
Dave Hansen6ad18fb2008-08-11 10:01:49 -07002975 *ret = buffer->processed;
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05002976 return r;
2977}
2978
Avi Kivity37a7d8b2007-01-05 16:36:56 -08002979#ifdef AUDIT
2980
2981static const char *audit_msg;
2982
2983static gva_t canonicalize(gva_t gva)
2984{
2985#ifdef CONFIG_X86_64
2986 gva = (long long)(gva << 16) >> 16;
2987#endif
2988 return gva;
2989}
2990
2991static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
2992 gva_t va, int level)
2993{
2994 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
2995 int i;
2996 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
2997
2998 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
2999 u64 ent = pt[i];
3000
Avi Kivityc7addb92007-09-16 18:58:32 +02003001 if (ent == shadow_trap_nonpresent_pte)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003002 continue;
3003
3004 va = canonicalize(va);
Avi Kivityc7addb92007-09-16 18:58:32 +02003005 if (level > 1) {
3006 if (ent == shadow_notrap_nonpresent_pte)
3007 printk(KERN_ERR "audit: (%s) nontrapping pte"
3008 " in nonleaf level: levels %d gva %lx"
3009 " level %d pte %llx\n", audit_msg,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003010 vcpu->arch.mmu.root_level, va, level, ent);
Avi Kivityc7addb92007-09-16 18:58:32 +02003011
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003012 audit_mappings_page(vcpu, ent, va, level - 1);
Avi Kivityc7addb92007-09-16 18:58:32 +02003013 } else {
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003014 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
Anthony Liguori35149e22008-04-02 14:46:56 -05003015 hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003016
Avi Kivityc7addb92007-09-16 18:58:32 +02003017 if (is_shadow_present_pte(ent)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003018 && (ent & PT64_BASE_ADDR_MASK) != hpa)
Avi Kivityc7addb92007-09-16 18:58:32 +02003019 printk(KERN_ERR "xx audit error: (%s) levels %d"
3020 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003021 audit_msg, vcpu->arch.mmu.root_level,
Mike Dayd77c26f2007-10-08 09:02:08 -04003022 va, gpa, hpa, ent,
3023 is_shadow_present_pte(ent));
Avi Kivityc7addb92007-09-16 18:58:32 +02003024 else if (ent == shadow_notrap_nonpresent_pte
3025 && !is_error_hpa(hpa))
3026 printk(KERN_ERR "audit: (%s) notrap shadow,"
3027 " valid guest gva %lx\n", audit_msg, va);
Anthony Liguori35149e22008-04-02 14:46:56 -05003028 kvm_release_pfn_clean(pfn);
Avi Kivityc7addb92007-09-16 18:58:32 +02003029
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003030 }
3031 }
3032}
3033
3034static void audit_mappings(struct kvm_vcpu *vcpu)
3035{
Avi Kivity1ea252a2007-03-08 11:48:09 +02003036 unsigned i;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003037
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003038 if (vcpu->arch.mmu.root_level == 4)
3039 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003040 else
3041 for (i = 0; i < 4; ++i)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003042 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003043 audit_mappings_page(vcpu,
Zhang Xiantaoad312c72007-12-13 23:50:52 +08003044 vcpu->arch.mmu.pae_root[i],
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003045 i << 30,
3046 2);
3047}
3048
3049static int count_rmaps(struct kvm_vcpu *vcpu)
3050{
3051 int nmaps = 0;
3052 int i, j, k;
3053
3054 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
3055 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
3056 struct kvm_rmap_desc *d;
3057
3058 for (j = 0; j < m->npages; ++j) {
Izik Eidus290fc382007-09-27 14:11:22 +02003059 unsigned long *rmapp = &m->rmap[j];
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003060
Izik Eidus290fc382007-09-27 14:11:22 +02003061 if (!*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003062 continue;
Izik Eidus290fc382007-09-27 14:11:22 +02003063 if (!(*rmapp & 1)) {
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003064 ++nmaps;
3065 continue;
3066 }
Izik Eidus290fc382007-09-27 14:11:22 +02003067 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003068 while (d) {
3069 for (k = 0; k < RMAP_EXT; ++k)
3070 if (d->shadow_ptes[k])
3071 ++nmaps;
3072 else
3073 break;
3074 d = d->more;
3075 }
3076 }
3077 }
3078 return nmaps;
3079}
3080
3081static int count_writable_mappings(struct kvm_vcpu *vcpu)
3082{
3083 int nmaps = 0;
Avi Kivity4db35312007-11-21 15:28:32 +02003084 struct kvm_mmu_page *sp;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003085 int i;
3086
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08003087 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
Avi Kivity4db35312007-11-21 15:28:32 +02003088 u64 *pt = sp->spt;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003089
Avi Kivity4db35312007-11-21 15:28:32 +02003090 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003091 continue;
3092
3093 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
3094 u64 ent = pt[i];
3095
3096 if (!(ent & PT_PRESENT_MASK))
3097 continue;
3098 if (!(ent & PT_WRITABLE_MASK))
3099 continue;
3100 ++nmaps;
3101 }
3102 }
3103 return nmaps;
3104}
3105
3106static void audit_rmap(struct kvm_vcpu *vcpu)
3107{
3108 int n_rmap = count_rmaps(vcpu);
3109 int n_actual = count_writable_mappings(vcpu);
3110
3111 if (n_rmap != n_actual)
3112 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08003113 __func__, audit_msg, n_rmap, n_actual);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003114}
3115
3116static void audit_write_protection(struct kvm_vcpu *vcpu)
3117{
Avi Kivity4db35312007-11-21 15:28:32 +02003118 struct kvm_mmu_page *sp;
Izik Eidus290fc382007-09-27 14:11:22 +02003119 struct kvm_memory_slot *slot;
3120 unsigned long *rmapp;
3121 gfn_t gfn;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003122
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08003123 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
Avi Kivityf6e2c02b2009-01-11 13:02:10 +02003124 if (sp->role.direct)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003125 continue;
3126
Avi Kivity4db35312007-11-21 15:28:32 +02003127 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
Izik Eidus28430992008-10-03 17:40:32 +03003128 slot = gfn_to_memslot_unaliased(vcpu->kvm, sp->gfn);
Izik Eidus290fc382007-09-27 14:11:22 +02003129 rmapp = &slot->rmap[gfn - slot->base_gfn];
3130 if (*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003131 printk(KERN_ERR "%s: (%s) shadow page has writable"
3132 " mappings: gfn %lx role %x\n",
Harvey Harrisonb8688d52008-03-03 12:59:56 -08003133 __func__, audit_msg, sp->gfn,
Avi Kivity4db35312007-11-21 15:28:32 +02003134 sp->role.word);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08003135 }
3136}
3137
3138static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
3139{
3140 int olddbg = dbg;
3141
3142 dbg = 0;
3143 audit_msg = msg;
3144 audit_rmap(vcpu);
3145 audit_write_protection(vcpu);
3146 audit_mappings(vcpu);
3147 dbg = olddbg;
3148}
3149
3150#endif