blob: bbf5eb427dc6c88390c9170bdc544f3ffa7a3713 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
Avi Kivity6aa8b732006-12-10 02:21:36 -080019
20#include "vmx.h"
21#include "kvm.h"
22
Avi Kivitye4956062007-06-28 14:15:57 -040023#include <linux/types.h>
24#include <linux/string.h>
25#include <linux/mm.h>
26#include <linux/highmem.h>
27#include <linux/module.h>
28
29#include <asm/page.h>
30#include <asm/cmpxchg.h>
31
Avi Kivity37a7d8b2007-01-05 16:36:56 -080032#undef MMU_DEBUG
33
34#undef AUDIT
35
36#ifdef AUDIT
37static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
38#else
39static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
40#endif
41
42#ifdef MMU_DEBUG
43
44#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
45#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
46
47#else
48
49#define pgprintk(x...) do { } while (0)
50#define rmap_printk(x...) do { } while (0)
51
52#endif
53
54#if defined(MMU_DEBUG) || defined(AUDIT)
55static int dbg = 1;
56#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080057
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080058#ifndef MMU_DEBUG
59#define ASSERT(x) do { } while (0)
60#else
Avi Kivity6aa8b732006-12-10 02:21:36 -080061#define ASSERT(x) \
62 if (!(x)) { \
63 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
64 __FILE__, __LINE__, #x); \
65 }
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080066#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080067
Avi Kivitycea0f0e2007-01-05 16:36:43 -080068#define PT64_PT_BITS 9
69#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
70#define PT32_PT_BITS 10
71#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080072
73#define PT_WRITABLE_SHIFT 1
74
75#define PT_PRESENT_MASK (1ULL << 0)
76#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
77#define PT_USER_MASK (1ULL << 2)
78#define PT_PWT_MASK (1ULL << 3)
79#define PT_PCD_MASK (1ULL << 4)
80#define PT_ACCESSED_MASK (1ULL << 5)
81#define PT_DIRTY_MASK (1ULL << 6)
82#define PT_PAGE_SIZE_MASK (1ULL << 7)
83#define PT_PAT_MASK (1ULL << 7)
84#define PT_GLOBAL_MASK (1ULL << 8)
85#define PT64_NX_MASK (1ULL << 63)
86
87#define PT_PAT_SHIFT 7
88#define PT_DIR_PAT_SHIFT 12
89#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
90
91#define PT32_DIR_PSE36_SIZE 4
92#define PT32_DIR_PSE36_SHIFT 13
Mike Dayd77c26f2007-10-08 09:02:08 -040093#define PT32_DIR_PSE36_MASK \
94 (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
Avi Kivity6aa8b732006-12-10 02:21:36 -080095
96
Avi Kivity6aa8b732006-12-10 02:21:36 -080097#define PT_FIRST_AVAIL_BITS_SHIFT 9
98#define PT64_SECOND_AVAIL_BITS_SHIFT 52
99
Avi Kivity6aa8b732006-12-10 02:21:36 -0800100#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
101
Avi Kivity6aa8b732006-12-10 02:21:36 -0800102#define VALID_PAGE(x) ((x) != INVALID_PAGE)
103
104#define PT64_LEVEL_BITS 9
105
106#define PT64_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -0400107 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800108
109#define PT64_LEVEL_MASK(level) \
110 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
111
112#define PT64_INDEX(address, level)\
113 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
114
115
116#define PT32_LEVEL_BITS 10
117
118#define PT32_LEVEL_SHIFT(level) \
Mike Dayd77c26f2007-10-08 09:02:08 -0400119 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800120
121#define PT32_LEVEL_MASK(level) \
122 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
123
124#define PT32_INDEX(address, level)\
125 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
126
127
Avi Kivity27aba762007-03-09 13:04:31 +0200128#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800129#define PT64_DIR_BASE_ADDR_MASK \
130 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
131
132#define PT32_BASE_ADDR_MASK PAGE_MASK
133#define PT32_DIR_BASE_ADDR_MASK \
134 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
135
136
137#define PFERR_PRESENT_MASK (1U << 0)
138#define PFERR_WRITE_MASK (1U << 1)
139#define PFERR_USER_MASK (1U << 2)
Avi Kivity73b10872007-01-26 00:56:41 -0800140#define PFERR_FETCH_MASK (1U << 4)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800141
142#define PT64_ROOT_LEVEL 4
143#define PT32_ROOT_LEVEL 2
144#define PT32E_ROOT_LEVEL 3
145
146#define PT_DIRECTORY_LEVEL 2
147#define PT_PAGE_TABLE_LEVEL 1
148
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800149#define RMAP_EXT 4
150
151struct kvm_rmap_desc {
152 u64 *shadow_ptes[RMAP_EXT];
153 struct kvm_rmap_desc *more;
154};
155
Avi Kivityb5a33a72007-04-15 16:31:09 +0300156static struct kmem_cache *pte_chain_cache;
157static struct kmem_cache *rmap_desc_cache;
Avi Kivityd3d25b02007-05-30 12:34:53 +0300158static struct kmem_cache *mmu_page_header_cache;
Avi Kivityb5a33a72007-04-15 16:31:09 +0300159
Avi Kivityc7addb92007-09-16 18:58:32 +0200160static u64 __read_mostly shadow_trap_nonpresent_pte;
161static u64 __read_mostly shadow_notrap_nonpresent_pte;
162
163void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
164{
165 shadow_trap_nonpresent_pte = trap_pte;
166 shadow_notrap_nonpresent_pte = notrap_pte;
167}
168EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
169
Avi Kivity6aa8b732006-12-10 02:21:36 -0800170static int is_write_protection(struct kvm_vcpu *vcpu)
171{
Rusty Russell707d92fa2007-07-17 23:19:08 +1000172 return vcpu->cr0 & X86_CR0_WP;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800173}
174
175static int is_cpuid_PSE36(void)
176{
177 return 1;
178}
179
Avi Kivity73b10872007-01-26 00:56:41 -0800180static int is_nx(struct kvm_vcpu *vcpu)
181{
182 return vcpu->shadow_efer & EFER_NX;
183}
184
Avi Kivity6aa8b732006-12-10 02:21:36 -0800185static int is_present_pte(unsigned long pte)
186{
187 return pte & PT_PRESENT_MASK;
188}
189
Avi Kivityc7addb92007-09-16 18:58:32 +0200190static int is_shadow_present_pte(u64 pte)
191{
192 pte &= ~PT_SHADOW_IO_MARK;
193 return pte != shadow_trap_nonpresent_pte
194 && pte != shadow_notrap_nonpresent_pte;
195}
196
Avi Kivity6aa8b732006-12-10 02:21:36 -0800197static int is_writeble_pte(unsigned long pte)
198{
199 return pte & PT_WRITABLE_MASK;
200}
201
Avi Kivitye3c5e7ec2007-10-11 12:32:30 +0200202static int is_dirty_pte(unsigned long pte)
203{
204 return pte & PT_DIRTY_MASK;
205}
206
Avi Kivity6aa8b732006-12-10 02:21:36 -0800207static int is_io_pte(unsigned long pte)
208{
209 return pte & PT_SHADOW_IO_MARK;
210}
211
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800212static int is_rmap_pte(u64 pte)
213{
Izik Eidus9647c142007-10-16 14:43:46 +0200214 return pte != shadow_trap_nonpresent_pte
215 && pte != shadow_notrap_nonpresent_pte;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800216}
217
Avi Kivitye663ee62007-05-31 15:46:04 +0300218static void set_shadow_pte(u64 *sptep, u64 spte)
219{
220#ifdef CONFIG_X86_64
221 set_64bit((unsigned long *)sptep, spte);
222#else
223 set_64bit((unsigned long long *)sptep, spte);
224#endif
225}
226
Avi Kivitye2dec932007-01-05 16:36:54 -0800227static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300228 struct kmem_cache *base_cache, int min)
Avi Kivity714b93d2007-01-05 16:36:53 -0800229{
230 void *obj;
231
232 if (cache->nobjs >= min)
Avi Kivitye2dec932007-01-05 16:36:54 -0800233 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800234 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300235 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
Avi Kivity714b93d2007-01-05 16:36:53 -0800236 if (!obj)
Avi Kivitye2dec932007-01-05 16:36:54 -0800237 return -ENOMEM;
Avi Kivity714b93d2007-01-05 16:36:53 -0800238 cache->objects[cache->nobjs++] = obj;
239 }
Avi Kivitye2dec932007-01-05 16:36:54 -0800240 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800241}
242
243static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
244{
245 while (mc->nobjs)
246 kfree(mc->objects[--mc->nobjs]);
247}
248
Avi Kivityc1158e62007-07-20 08:18:27 +0300249static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
Avi Kivity2e3e5882007-09-10 11:28:17 +0300250 int min)
Avi Kivityc1158e62007-07-20 08:18:27 +0300251{
252 struct page *page;
253
254 if (cache->nobjs >= min)
255 return 0;
256 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity2e3e5882007-09-10 11:28:17 +0300257 page = alloc_page(GFP_KERNEL);
Avi Kivityc1158e62007-07-20 08:18:27 +0300258 if (!page)
259 return -ENOMEM;
260 set_page_private(page, 0);
261 cache->objects[cache->nobjs++] = page_address(page);
262 }
263 return 0;
264}
265
266static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
267{
268 while (mc->nobjs)
Avi Kivityc4d198d2007-07-21 09:06:46 +0300269 free_page((unsigned long)mc->objects[--mc->nobjs]);
Avi Kivityc1158e62007-07-20 08:18:27 +0300270}
271
Avi Kivity8c438502007-04-16 11:53:17 +0300272static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
273{
274 int r;
275
Avi Kivity22d95b12007-09-14 20:26:06 +0300276 kvm_mmu_free_some_pages(vcpu);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300277 r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
278 pte_chain_cache, 4);
279 if (r)
280 goto out;
281 r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
282 rmap_desc_cache, 1);
283 if (r)
284 goto out;
Izik Eidus290fc382007-09-27 14:11:22 +0200285 r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8);
Avi Kivity2e3e5882007-09-10 11:28:17 +0300286 if (r)
287 goto out;
288 r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
289 mmu_page_header_cache, 4);
290out:
Avi Kivity8c438502007-04-16 11:53:17 +0300291 return r;
292}
293
Avi Kivity714b93d2007-01-05 16:36:53 -0800294static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
295{
296 mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
297 mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
Avi Kivityc1158e62007-07-20 08:18:27 +0300298 mmu_free_memory_cache_page(&vcpu->mmu_page_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +0300299 mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800300}
301
302static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
303 size_t size)
304{
305 void *p;
306
307 BUG_ON(!mc->nobjs);
308 p = mc->objects[--mc->nobjs];
309 memset(p, 0, size);
310 return p;
311}
312
Avi Kivity714b93d2007-01-05 16:36:53 -0800313static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
314{
315 return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
316 sizeof(struct kvm_pte_chain));
317}
318
Avi Kivity90cb0522007-07-17 13:04:56 +0300319static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
Avi Kivity714b93d2007-01-05 16:36:53 -0800320{
Avi Kivity90cb0522007-07-17 13:04:56 +0300321 kfree(pc);
Avi Kivity714b93d2007-01-05 16:36:53 -0800322}
323
324static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
325{
326 return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
327 sizeof(struct kvm_rmap_desc));
328}
329
Avi Kivity90cb0522007-07-17 13:04:56 +0300330static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
Avi Kivity714b93d2007-01-05 16:36:53 -0800331{
Avi Kivity90cb0522007-07-17 13:04:56 +0300332 kfree(rd);
Avi Kivity714b93d2007-01-05 16:36:53 -0800333}
334
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800335/*
Izik Eidus290fc382007-09-27 14:11:22 +0200336 * Take gfn and return the reverse mapping to it.
337 * Note: gfn must be unaliased before this function get called
338 */
339
340static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn)
341{
342 struct kvm_memory_slot *slot;
343
344 slot = gfn_to_memslot(kvm, gfn);
345 return &slot->rmap[gfn - slot->base_gfn];
346}
347
348/*
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800349 * Reverse mapping data structures:
350 *
Izik Eidus290fc382007-09-27 14:11:22 +0200351 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
352 * that points to page_address(page).
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800353 *
Izik Eidus290fc382007-09-27 14:11:22 +0200354 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
355 * containing more mappings.
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800356 */
Izik Eidus290fc382007-09-27 14:11:22 +0200357static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800358{
Izik Eidus290fc382007-09-27 14:11:22 +0200359 struct kvm_mmu_page *page;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800360 struct kvm_rmap_desc *desc;
Izik Eidus290fc382007-09-27 14:11:22 +0200361 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800362 int i;
363
364 if (!is_rmap_pte(*spte))
365 return;
Izik Eidus290fc382007-09-27 14:11:22 +0200366 gfn = unalias_gfn(vcpu->kvm, gfn);
367 page = page_header(__pa(spte));
368 page->gfns[spte - page->spt] = gfn;
369 rmapp = gfn_to_rmap(vcpu->kvm, gfn);
370 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800371 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200372 *rmapp = (unsigned long)spte;
373 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800374 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
Avi Kivity714b93d2007-01-05 16:36:53 -0800375 desc = mmu_alloc_rmap_desc(vcpu);
Izik Eidus290fc382007-09-27 14:11:22 +0200376 desc->shadow_ptes[0] = (u64 *)*rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800377 desc->shadow_ptes[1] = spte;
Izik Eidus290fc382007-09-27 14:11:22 +0200378 *rmapp = (unsigned long)desc | 1;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800379 } else {
380 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200381 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800382 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
383 desc = desc->more;
384 if (desc->shadow_ptes[RMAP_EXT-1]) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800385 desc->more = mmu_alloc_rmap_desc(vcpu);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800386 desc = desc->more;
387 }
388 for (i = 0; desc->shadow_ptes[i]; ++i)
389 ;
390 desc->shadow_ptes[i] = spte;
391 }
392}
393
Izik Eidus290fc382007-09-27 14:11:22 +0200394static void rmap_desc_remove_entry(unsigned long *rmapp,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800395 struct kvm_rmap_desc *desc,
396 int i,
397 struct kvm_rmap_desc *prev_desc)
398{
399 int j;
400
401 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
402 ;
403 desc->shadow_ptes[i] = desc->shadow_ptes[j];
Al Viro11718b4d2007-02-09 16:39:20 +0000404 desc->shadow_ptes[j] = NULL;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800405 if (j != 0)
406 return;
407 if (!prev_desc && !desc->more)
Izik Eidus290fc382007-09-27 14:11:22 +0200408 *rmapp = (unsigned long)desc->shadow_ptes[0];
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800409 else
410 if (prev_desc)
411 prev_desc->more = desc->more;
412 else
Izik Eidus290fc382007-09-27 14:11:22 +0200413 *rmapp = (unsigned long)desc->more | 1;
Avi Kivity90cb0522007-07-17 13:04:56 +0300414 mmu_free_rmap_desc(desc);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800415}
416
Izik Eidus290fc382007-09-27 14:11:22 +0200417static void rmap_remove(struct kvm *kvm, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800418{
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800419 struct kvm_rmap_desc *desc;
420 struct kvm_rmap_desc *prev_desc;
Izik Eidus290fc382007-09-27 14:11:22 +0200421 struct kvm_mmu_page *page;
422 unsigned long *rmapp;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800423 int i;
424
425 if (!is_rmap_pte(*spte))
426 return;
Izik Eidus290fc382007-09-27 14:11:22 +0200427 page = page_header(__pa(spte));
428 rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]);
429 if (!*rmapp) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800430 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
431 BUG();
Izik Eidus290fc382007-09-27 14:11:22 +0200432 } else if (!(*rmapp & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800433 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200434 if ((u64 *)*rmapp != spte) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800435 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
436 spte, *spte);
437 BUG();
438 }
Izik Eidus290fc382007-09-27 14:11:22 +0200439 *rmapp = 0;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800440 } else {
441 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
Izik Eidus290fc382007-09-27 14:11:22 +0200442 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800443 prev_desc = NULL;
444 while (desc) {
445 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
446 if (desc->shadow_ptes[i] == spte) {
Izik Eidus290fc382007-09-27 14:11:22 +0200447 rmap_desc_remove_entry(rmapp,
Avi Kivity714b93d2007-01-05 16:36:53 -0800448 desc, i,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800449 prev_desc);
450 return;
451 }
452 prev_desc = desc;
453 desc = desc->more;
454 }
455 BUG();
456 }
457}
458
Izik Eidus98348e92007-10-16 14:42:30 +0200459static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
Avi Kivity374cbac2007-01-05 16:36:43 -0800460{
Avi Kivity374cbac2007-01-05 16:36:43 -0800461 struct kvm_rmap_desc *desc;
Izik Eidus98348e92007-10-16 14:42:30 +0200462 struct kvm_rmap_desc *prev_desc;
463 u64 *prev_spte;
464 int i;
465
466 if (!*rmapp)
467 return NULL;
468 else if (!(*rmapp & 1)) {
469 if (!spte)
470 return (u64 *)*rmapp;
471 return NULL;
472 }
473 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
474 prev_desc = NULL;
475 prev_spte = NULL;
476 while (desc) {
477 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
478 if (prev_spte == spte)
479 return desc->shadow_ptes[i];
480 prev_spte = desc->shadow_ptes[i];
481 }
482 desc = desc->more;
483 }
484 return NULL;
485}
486
487static void rmap_write_protect(struct kvm *kvm, u64 gfn)
488{
Izik Eidus290fc382007-09-27 14:11:22 +0200489 unsigned long *rmapp;
Avi Kivity374cbac2007-01-05 16:36:43 -0800490 u64 *spte;
491
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500492 gfn = unalias_gfn(kvm, gfn);
493 rmapp = gfn_to_rmap(kvm, gfn);
Avi Kivity374cbac2007-01-05 16:36:43 -0800494
Izik Eidus98348e92007-10-16 14:42:30 +0200495 spte = rmap_next(kvm, rmapp, NULL);
496 while (spte) {
Avi Kivity374cbac2007-01-05 16:36:43 -0800497 BUG_ON(!spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800498 BUG_ON(!(*spte & PT_PRESENT_MASK));
Avi Kivity374cbac2007-01-05 16:36:43 -0800499 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
Izik Eidus9647c142007-10-16 14:43:46 +0200500 if (is_writeble_pte(*spte))
501 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500502 kvm_flush_remote_tlbs(kvm);
Izik Eidus9647c142007-10-16 14:43:46 +0200503 spte = rmap_next(kvm, rmapp, spte);
Avi Kivity374cbac2007-01-05 16:36:43 -0800504 }
505}
506
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800507#ifdef MMU_DEBUG
Avi Kivity47ad8e62007-05-06 15:50:58 +0300508static int is_empty_shadow_page(u64 *spt)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800509{
Avi Kivity139bdb22007-01-05 16:36:50 -0800510 u64 *pos;
511 u64 *end;
512
Avi Kivity47ad8e62007-05-06 15:50:58 +0300513 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
Avi Kivityc7addb92007-09-16 18:58:32 +0200514 if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) {
Avi Kivity139bdb22007-01-05 16:36:50 -0800515 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
516 pos, *pos);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800517 return 0;
Avi Kivity139bdb22007-01-05 16:36:50 -0800518 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800519 return 1;
520}
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800521#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800522
Avi Kivity90cb0522007-07-17 13:04:56 +0300523static void kvm_mmu_free_page(struct kvm *kvm,
Avi Kivity4b02d6d2007-05-06 15:36:30 +0300524 struct kvm_mmu_page *page_head)
Avi Kivity260746c2007-01-05 16:36:49 -0800525{
Avi Kivity47ad8e62007-05-06 15:50:58 +0300526 ASSERT(is_empty_shadow_page(page_head->spt));
Avi Kivityd3d25b02007-05-30 12:34:53 +0300527 list_del(&page_head->link);
Avi Kivityc1158e62007-07-20 08:18:27 +0300528 __free_page(virt_to_page(page_head->spt));
Izik Eidus290fc382007-09-27 14:11:22 +0200529 __free_page(virt_to_page(page_head->gfns));
Avi Kivity90cb0522007-07-17 13:04:56 +0300530 kfree(page_head);
531 ++kvm->n_free_mmu_pages;
Avi Kivity260746c2007-01-05 16:36:49 -0800532}
533
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800534static unsigned kvm_page_table_hashfn(gfn_t gfn)
535{
536 return gfn;
537}
538
Avi Kivity25c0de22007-01-05 16:36:42 -0800539static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
540 u64 *parent_pte)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800541{
542 struct kvm_mmu_page *page;
543
Avi Kivityd3d25b02007-05-30 12:34:53 +0300544 if (!vcpu->kvm->n_free_mmu_pages)
Avi Kivity25c0de22007-01-05 16:36:42 -0800545 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800546
Avi Kivityd3d25b02007-05-30 12:34:53 +0300547 page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
548 sizeof *page);
549 page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
Izik Eidus290fc382007-09-27 14:11:22 +0200550 page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
Avi Kivityd3d25b02007-05-30 12:34:53 +0300551 set_page_private(virt_to_page(page->spt), (unsigned long)page);
552 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
Avi Kivity47ad8e62007-05-06 15:50:58 +0300553 ASSERT(is_empty_shadow_page(page->spt));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800554 page->slot_bitmap = 0;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800555 page->multimapped = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800556 page->parent_pte = parent_pte;
Avi Kivityebeace82007-01-05 16:36:47 -0800557 --vcpu->kvm->n_free_mmu_pages;
Avi Kivity25c0de22007-01-05 16:36:42 -0800558 return page;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800559}
560
Avi Kivity714b93d2007-01-05 16:36:53 -0800561static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
562 struct kvm_mmu_page *page, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800563{
564 struct kvm_pte_chain *pte_chain;
565 struct hlist_node *node;
566 int i;
567
568 if (!parent_pte)
569 return;
570 if (!page->multimapped) {
571 u64 *old = page->parent_pte;
572
573 if (!old) {
574 page->parent_pte = parent_pte;
575 return;
576 }
577 page->multimapped = 1;
Avi Kivity714b93d2007-01-05 16:36:53 -0800578 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800579 INIT_HLIST_HEAD(&page->parent_ptes);
580 hlist_add_head(&pte_chain->link, &page->parent_ptes);
581 pte_chain->parent_ptes[0] = old;
582 }
583 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
584 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
585 continue;
586 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
587 if (!pte_chain->parent_ptes[i]) {
588 pte_chain->parent_ptes[i] = parent_pte;
589 return;
590 }
591 }
Avi Kivity714b93d2007-01-05 16:36:53 -0800592 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800593 BUG_ON(!pte_chain);
594 hlist_add_head(&pte_chain->link, &page->parent_ptes);
595 pte_chain->parent_ptes[0] = parent_pte;
596}
597
Avi Kivity90cb0522007-07-17 13:04:56 +0300598static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800599 u64 *parent_pte)
600{
601 struct kvm_pte_chain *pte_chain;
602 struct hlist_node *node;
603 int i;
604
605 if (!page->multimapped) {
606 BUG_ON(page->parent_pte != parent_pte);
607 page->parent_pte = NULL;
608 return;
609 }
610 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
611 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
612 if (!pte_chain->parent_ptes[i])
613 break;
614 if (pte_chain->parent_ptes[i] != parent_pte)
615 continue;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800616 while (i + 1 < NR_PTE_CHAIN_ENTRIES
617 && pte_chain->parent_ptes[i + 1]) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800618 pte_chain->parent_ptes[i]
619 = pte_chain->parent_ptes[i + 1];
620 ++i;
621 }
622 pte_chain->parent_ptes[i] = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800623 if (i == 0) {
624 hlist_del(&pte_chain->link);
Avi Kivity90cb0522007-07-17 13:04:56 +0300625 mmu_free_pte_chain(pte_chain);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800626 if (hlist_empty(&page->parent_ptes)) {
627 page->multimapped = 0;
628 page->parent_pte = NULL;
629 }
630 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800631 return;
632 }
633 BUG();
634}
635
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500636static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800637 gfn_t gfn)
638{
639 unsigned index;
640 struct hlist_head *bucket;
641 struct kvm_mmu_page *page;
642 struct hlist_node *node;
643
644 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
645 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500646 bucket = &kvm->mmu_page_hash[index];
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800647 hlist_for_each_entry(page, node, bucket, hash_link)
648 if (page->gfn == gfn && !page->role.metaphysical) {
649 pgprintk("%s: found role %x\n",
650 __FUNCTION__, page->role.word);
651 return page;
652 }
653 return NULL;
654}
655
656static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
657 gfn_t gfn,
658 gva_t gaddr,
659 unsigned level,
660 int metaphysical,
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200661 unsigned hugepage_access,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800662 u64 *parent_pte)
663{
664 union kvm_mmu_page_role role;
665 unsigned index;
666 unsigned quadrant;
667 struct hlist_head *bucket;
668 struct kvm_mmu_page *page;
669 struct hlist_node *node;
670
671 role.word = 0;
672 role.glevels = vcpu->mmu.root_level;
673 role.level = level;
674 role.metaphysical = metaphysical;
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200675 role.hugepage_access = hugepage_access;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800676 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
677 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
678 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
679 role.quadrant = quadrant;
680 }
681 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
682 gfn, role.word);
683 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
684 bucket = &vcpu->kvm->mmu_page_hash[index];
685 hlist_for_each_entry(page, node, bucket, hash_link)
686 if (page->gfn == gfn && page->role.word == role.word) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800687 mmu_page_add_parent_pte(vcpu, page, parent_pte);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800688 pgprintk("%s: found\n", __FUNCTION__);
689 return page;
690 }
691 page = kvm_mmu_alloc_page(vcpu, parent_pte);
692 if (!page)
693 return page;
694 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
695 page->gfn = gfn;
696 page->role = role;
697 hlist_add_head(&page->hash_link, bucket);
Avi Kivityc7addb92007-09-16 18:58:32 +0200698 vcpu->mmu.prefetch_page(vcpu, page);
Avi Kivity374cbac2007-01-05 16:36:43 -0800699 if (!metaphysical)
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500700 rmap_write_protect(vcpu->kvm, gfn);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800701 return page;
702}
703
Avi Kivity90cb0522007-07-17 13:04:56 +0300704static void kvm_mmu_page_unlink_children(struct kvm *kvm,
Avi Kivitya4360362007-01-05 16:36:45 -0800705 struct kvm_mmu_page *page)
706{
Avi Kivity697fe2e2007-01-05 16:36:46 -0800707 unsigned i;
708 u64 *pt;
709 u64 ent;
710
Avi Kivity47ad8e62007-05-06 15:50:58 +0300711 pt = page->spt;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800712
713 if (page->role.level == PT_PAGE_TABLE_LEVEL) {
714 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
Avi Kivityc7addb92007-09-16 18:58:32 +0200715 if (is_shadow_present_pte(pt[i]))
Izik Eidus290fc382007-09-27 14:11:22 +0200716 rmap_remove(kvm, &pt[i]);
Avi Kivityc7addb92007-09-16 18:58:32 +0200717 pt[i] = shadow_trap_nonpresent_pte;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800718 }
Avi Kivity90cb0522007-07-17 13:04:56 +0300719 kvm_flush_remote_tlbs(kvm);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800720 return;
721 }
722
723 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
724 ent = pt[i];
725
Avi Kivityc7addb92007-09-16 18:58:32 +0200726 pt[i] = shadow_trap_nonpresent_pte;
727 if (!is_shadow_present_pte(ent))
Avi Kivity697fe2e2007-01-05 16:36:46 -0800728 continue;
729 ent &= PT64_BASE_ADDR_MASK;
Avi Kivity90cb0522007-07-17 13:04:56 +0300730 mmu_page_remove_parent_pte(page_header(ent), &pt[i]);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800731 }
Avi Kivity90cb0522007-07-17 13:04:56 +0300732 kvm_flush_remote_tlbs(kvm);
Avi Kivitya4360362007-01-05 16:36:45 -0800733}
734
Avi Kivity90cb0522007-07-17 13:04:56 +0300735static void kvm_mmu_put_page(struct kvm_mmu_page *page,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800736 u64 *parent_pte)
737{
Avi Kivity90cb0522007-07-17 13:04:56 +0300738 mmu_page_remove_parent_pte(page, parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800739}
740
Avi Kivity12b7d282007-09-23 14:10:49 +0200741static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
742{
743 int i;
744
745 for (i = 0; i < KVM_MAX_VCPUS; ++i)
746 if (kvm->vcpus[i])
747 kvm->vcpus[i]->last_pte_updated = NULL;
748}
749
Avi Kivity90cb0522007-07-17 13:04:56 +0300750static void kvm_mmu_zap_page(struct kvm *kvm,
Avi Kivitya4360362007-01-05 16:36:45 -0800751 struct kvm_mmu_page *page)
752{
753 u64 *parent_pte;
754
755 while (page->multimapped || page->parent_pte) {
756 if (!page->multimapped)
757 parent_pte = page->parent_pte;
758 else {
759 struct kvm_pte_chain *chain;
760
761 chain = container_of(page->parent_ptes.first,
762 struct kvm_pte_chain, link);
763 parent_pte = chain->parent_ptes[0];
764 }
Avi Kivity697fe2e2007-01-05 16:36:46 -0800765 BUG_ON(!parent_pte);
Avi Kivity90cb0522007-07-17 13:04:56 +0300766 kvm_mmu_put_page(page, parent_pte);
Avi Kivityc7addb92007-09-16 18:58:32 +0200767 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800768 }
Avi Kivity90cb0522007-07-17 13:04:56 +0300769 kvm_mmu_page_unlink_children(kvm, page);
Avi Kivity3bb65a22007-01-05 16:36:51 -0800770 if (!page->root_count) {
771 hlist_del(&page->hash_link);
Avi Kivity90cb0522007-07-17 13:04:56 +0300772 kvm_mmu_free_page(kvm, page);
Avi Kivity36868f72007-03-26 19:31:52 +0200773 } else
Avi Kivity90cb0522007-07-17 13:04:56 +0300774 list_move(&page->link, &kvm->active_mmu_pages);
Avi Kivity12b7d282007-09-23 14:10:49 +0200775 kvm_mmu_reset_last_pte_updated(kvm);
Avi Kivitya4360362007-01-05 16:36:45 -0800776}
777
Izik Eidus82ce2c92007-10-02 18:52:55 +0200778/*
779 * Changing the number of mmu pages allocated to the vm
780 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
781 */
782void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
783{
784 /*
785 * If we set the number of mmu pages to be smaller be than the
786 * number of actived pages , we must to free some mmu pages before we
787 * change the value
788 */
789
790 if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) >
791 kvm_nr_mmu_pages) {
792 int n_used_mmu_pages = kvm->n_alloc_mmu_pages
793 - kvm->n_free_mmu_pages;
794
795 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
796 struct kvm_mmu_page *page;
797
798 page = container_of(kvm->active_mmu_pages.prev,
799 struct kvm_mmu_page, link);
800 kvm_mmu_zap_page(kvm, page);
801 n_used_mmu_pages--;
802 }
803 kvm->n_free_mmu_pages = 0;
804 }
805 else
806 kvm->n_free_mmu_pages += kvm_nr_mmu_pages
807 - kvm->n_alloc_mmu_pages;
808
809 kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages;
810}
811
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500812static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
Avi Kivitya4360362007-01-05 16:36:45 -0800813{
814 unsigned index;
815 struct hlist_head *bucket;
816 struct kvm_mmu_page *page;
817 struct hlist_node *node, *n;
818 int r;
819
820 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
821 r = 0;
822 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500823 bucket = &kvm->mmu_page_hash[index];
Avi Kivitya4360362007-01-05 16:36:45 -0800824 hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
825 if (page->gfn == gfn && !page->role.metaphysical) {
Avi Kivity697fe2e2007-01-05 16:36:46 -0800826 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
827 page->role.word);
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500828 kvm_mmu_zap_page(kvm, page);
Avi Kivitya4360362007-01-05 16:36:45 -0800829 r = 1;
830 }
831 return r;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800832}
833
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500834static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
Avi Kivity97a0a012007-05-31 15:08:29 +0300835{
836 struct kvm_mmu_page *page;
837
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500838 while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
Avi Kivity97a0a012007-05-31 15:08:29 +0300839 pgprintk("%s: zap %lx %x\n",
840 __FUNCTION__, gfn, page->role.word);
Anthony Liguorif67a46f2007-10-10 19:25:50 -0500841 kvm_mmu_zap_page(kvm, page);
Avi Kivity97a0a012007-05-31 15:08:29 +0300842 }
843}
844
Avi Kivity6aa8b732006-12-10 02:21:36 -0800845static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
846{
847 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
848 struct kvm_mmu_page *page_head = page_header(__pa(pte));
849
850 __set_bit(slot, &page_head->slot_bitmap);
851}
852
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500853hpa_t safe_gpa_to_hpa(struct kvm *kvm, gpa_t gpa)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800854{
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500855 hpa_t hpa = gpa_to_hpa(kvm, gpa);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800856
857 return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
858}
859
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500860hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800861{
Avi Kivity6aa8b732006-12-10 02:21:36 -0800862 struct page *page;
863
864 ASSERT((gpa & HPA_ERR_MASK) == 0);
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500865 page = gfn_to_page(kvm, gpa >> PAGE_SHIFT);
Avi Kivity954bbbc2007-03-30 14:02:32 +0300866 if (!page)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800867 return gpa | HPA_ERR_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800868 return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
869 | (gpa & (PAGE_SIZE-1));
870}
871
872hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
873{
874 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
875
876 if (gpa == UNMAPPED_GVA)
877 return UNMAPPED_GVA;
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500878 return gpa_to_hpa(vcpu->kvm, gpa);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800879}
880
Avi Kivity039576c2007-03-20 12:46:50 +0200881struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
882{
883 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
884
885 if (gpa == UNMAPPED_GVA)
886 return NULL;
Anthony Liguori4a4c9922007-10-10 20:08:41 -0500887 return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT);
Avi Kivity039576c2007-03-20 12:46:50 +0200888}
889
Avi Kivity6aa8b732006-12-10 02:21:36 -0800890static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
891{
892}
893
894static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
895{
896 int level = PT32E_ROOT_LEVEL;
897 hpa_t table_addr = vcpu->mmu.root_hpa;
898
899 for (; ; level--) {
900 u32 index = PT64_INDEX(v, level);
901 u64 *table;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800902 u64 pte;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800903
904 ASSERT(VALID_PAGE(table_addr));
905 table = __va(table_addr);
906
907 if (level == 1) {
Izik Eidus9647c142007-10-16 14:43:46 +0200908 int was_rmapped;
909
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800910 pte = table[index];
Izik Eidus9647c142007-10-16 14:43:46 +0200911 was_rmapped = is_rmap_pte(pte);
Avi Kivityc7addb92007-09-16 18:58:32 +0200912 if (is_shadow_present_pte(pte) && is_writeble_pte(pte))
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800913 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800914 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
915 page_header_update_slot(vcpu->kvm, table, v);
916 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
917 PT_USER_MASK;
Izik Eidus9647c142007-10-16 14:43:46 +0200918 if (!was_rmapped)
919 rmap_add(vcpu, &table[index], v >> PAGE_SHIFT);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800920 return 0;
921 }
922
Avi Kivityc7addb92007-09-16 18:58:32 +0200923 if (table[index] == shadow_trap_nonpresent_pte) {
Avi Kivity25c0de22007-01-05 16:36:42 -0800924 struct kvm_mmu_page *new_table;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800925 gfn_t pseudo_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800926
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800927 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
928 >> PAGE_SHIFT;
929 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
930 v, level - 1,
Avi Kivity6bfccdc2007-10-11 15:13:49 +0200931 1, 3, &table[index]);
Avi Kivity25c0de22007-01-05 16:36:42 -0800932 if (!new_table) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800933 pgprintk("nonpaging_map: ENOMEM\n");
934 return -ENOMEM;
935 }
936
Avi Kivity47ad8e62007-05-06 15:50:58 +0300937 table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
Avi Kivity25c0de22007-01-05 16:36:42 -0800938 | PT_WRITABLE_MASK | PT_USER_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800939 }
940 table_addr = table[index] & PT64_BASE_ADDR_MASK;
941 }
942}
943
Avi Kivityc7addb92007-09-16 18:58:32 +0200944static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
945 struct kvm_mmu_page *sp)
946{
947 int i;
948
949 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
950 sp->spt[i] = shadow_trap_nonpresent_pte;
951}
952
Avi Kivity17ac10a2007-01-05 16:36:40 -0800953static void mmu_free_roots(struct kvm_vcpu *vcpu)
954{
955 int i;
Avi Kivity3bb65a22007-01-05 16:36:51 -0800956 struct kvm_mmu_page *page;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800957
Avi Kivity7b53aa52007-06-05 12:17:03 +0300958 if (!VALID_PAGE(vcpu->mmu.root_hpa))
959 return;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800960#ifdef CONFIG_X86_64
961 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
962 hpa_t root = vcpu->mmu.root_hpa;
963
Avi Kivity3bb65a22007-01-05 16:36:51 -0800964 page = page_header(root);
965 --page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800966 vcpu->mmu.root_hpa = INVALID_PAGE;
967 return;
968 }
969#endif
970 for (i = 0; i < 4; ++i) {
971 hpa_t root = vcpu->mmu.pae_root[i];
972
Avi Kivity417726a2007-04-12 17:35:58 +0300973 if (root) {
Avi Kivity417726a2007-04-12 17:35:58 +0300974 root &= PT64_BASE_ADDR_MASK;
975 page = page_header(root);
976 --page->root_count;
977 }
Avi Kivity17ac10a2007-01-05 16:36:40 -0800978 vcpu->mmu.pae_root[i] = INVALID_PAGE;
979 }
980 vcpu->mmu.root_hpa = INVALID_PAGE;
981}
982
983static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
984{
985 int i;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800986 gfn_t root_gfn;
Avi Kivity3bb65a22007-01-05 16:36:51 -0800987 struct kvm_mmu_page *page;
988
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800989 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800990
991#ifdef CONFIG_X86_64
992 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
993 hpa_t root = vcpu->mmu.root_hpa;
994
995 ASSERT(!VALID_PAGE(root));
Ingo Molnar68a99f62007-01-05 16:36:59 -0800996 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200997 PT64_ROOT_LEVEL, 0, 0, NULL);
Avi Kivity47ad8e62007-05-06 15:50:58 +0300998 root = __pa(page->spt);
Avi Kivity3bb65a22007-01-05 16:36:51 -0800999 ++page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001000 vcpu->mmu.root_hpa = root;
1001 return;
1002 }
1003#endif
1004 for (i = 0; i < 4; ++i) {
1005 hpa_t root = vcpu->mmu.pae_root[i];
1006
1007 ASSERT(!VALID_PAGE(root));
Avi Kivity417726a2007-04-12 17:35:58 +03001008 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
1009 if (!is_present_pte(vcpu->pdptrs[i])) {
1010 vcpu->mmu.pae_root[i] = 0;
1011 continue;
1012 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001013 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
Avi Kivity417726a2007-04-12 17:35:58 +03001014 } else if (vcpu->mmu.root_level == 0)
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001015 root_gfn = 0;
Ingo Molnar68a99f62007-01-05 16:36:59 -08001016 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001017 PT32_ROOT_LEVEL, !is_paging(vcpu),
Avi Kivityd28c6cfb2007-03-23 09:55:25 +02001018 0, NULL);
Avi Kivity47ad8e62007-05-06 15:50:58 +03001019 root = __pa(page->spt);
Avi Kivity3bb65a22007-01-05 16:36:51 -08001020 ++page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001021 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
1022 }
1023 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
1024}
1025
Avi Kivity6aa8b732006-12-10 02:21:36 -08001026static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1027{
1028 return vaddr;
1029}
1030
1031static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1032 u32 error_code)
1033{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001034 gpa_t addr = gva;
Avi Kivityebeace82007-01-05 16:36:47 -08001035 hpa_t paddr;
Avi Kivitye2dec932007-01-05 16:36:54 -08001036 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001037
Avi Kivitye2dec932007-01-05 16:36:54 -08001038 r = mmu_topup_memory_caches(vcpu);
1039 if (r)
1040 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -08001041
Avi Kivity6aa8b732006-12-10 02:21:36 -08001042 ASSERT(vcpu);
1043 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
1044
Avi Kivity6aa8b732006-12-10 02:21:36 -08001045
Anthony Liguori4a4c9922007-10-10 20:08:41 -05001046 paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001047
Avi Kivityebeace82007-01-05 16:36:47 -08001048 if (is_error_hpa(paddr))
1049 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001050
Avi Kivityebeace82007-01-05 16:36:47 -08001051 return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001052}
1053
Avi Kivity6aa8b732006-12-10 02:21:36 -08001054static void nonpaging_free(struct kvm_vcpu *vcpu)
1055{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001056 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001057}
1058
1059static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1060{
1061 struct kvm_mmu *context = &vcpu->mmu;
1062
1063 context->new_cr3 = nonpaging_new_cr3;
1064 context->page_fault = nonpaging_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001065 context->gva_to_gpa = nonpaging_gva_to_gpa;
1066 context->free = nonpaging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02001067 context->prefetch_page = nonpaging_prefetch_page;
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001068 context->root_level = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001069 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001070 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001071 return 0;
1072}
1073
Avi Kivity6aa8b732006-12-10 02:21:36 -08001074static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1075{
Avi Kivity1165f5f2007-04-19 17:27:43 +03001076 ++vcpu->stat.tlb_flush;
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03001077 kvm_x86_ops->tlb_flush(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001078}
1079
1080static void paging_new_cr3(struct kvm_vcpu *vcpu)
1081{
Avi Kivity374cbac2007-01-05 16:36:43 -08001082 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
Avi Kivitycea0f0e2007-01-05 16:36:43 -08001083 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001084}
1085
Avi Kivity6aa8b732006-12-10 02:21:36 -08001086static void inject_page_fault(struct kvm_vcpu *vcpu,
1087 u64 addr,
1088 u32 err_code)
1089{
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03001090 kvm_x86_ops->inject_page_fault(vcpu, addr, err_code);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001091}
1092
Avi Kivity6aa8b732006-12-10 02:21:36 -08001093static void paging_free(struct kvm_vcpu *vcpu)
1094{
1095 nonpaging_free(vcpu);
1096}
1097
1098#define PTTYPE 64
1099#include "paging_tmpl.h"
1100#undef PTTYPE
1101
1102#define PTTYPE 32
1103#include "paging_tmpl.h"
1104#undef PTTYPE
1105
Avi Kivity17ac10a2007-01-05 16:36:40 -08001106static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001107{
1108 struct kvm_mmu *context = &vcpu->mmu;
1109
1110 ASSERT(is_pae(vcpu));
1111 context->new_cr3 = paging_new_cr3;
1112 context->page_fault = paging64_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001113 context->gva_to_gpa = paging64_gva_to_gpa;
Avi Kivityc7addb92007-09-16 18:58:32 +02001114 context->prefetch_page = paging64_prefetch_page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001115 context->free = paging_free;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001116 context->root_level = level;
1117 context->shadow_root_level = level;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001118 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001119 return 0;
1120}
1121
Avi Kivity17ac10a2007-01-05 16:36:40 -08001122static int paging64_init_context(struct kvm_vcpu *vcpu)
1123{
1124 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1125}
1126
Avi Kivity6aa8b732006-12-10 02:21:36 -08001127static int paging32_init_context(struct kvm_vcpu *vcpu)
1128{
1129 struct kvm_mmu *context = &vcpu->mmu;
1130
1131 context->new_cr3 = paging_new_cr3;
1132 context->page_fault = paging32_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001133 context->gva_to_gpa = paging32_gva_to_gpa;
1134 context->free = paging_free;
Avi Kivityc7addb92007-09-16 18:58:32 +02001135 context->prefetch_page = paging32_prefetch_page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001136 context->root_level = PT32_ROOT_LEVEL;
1137 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17c3ba92007-06-04 15:58:30 +03001138 context->root_hpa = INVALID_PAGE;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001139 return 0;
1140}
1141
1142static int paging32E_init_context(struct kvm_vcpu *vcpu)
1143{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001144 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001145}
1146
1147static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1148{
1149 ASSERT(vcpu);
1150 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1151
1152 if (!is_paging(vcpu))
1153 return nonpaging_init_context(vcpu);
Avi Kivitya9058ec2006-12-29 16:49:37 -08001154 else if (is_long_mode(vcpu))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001155 return paging64_init_context(vcpu);
1156 else if (is_pae(vcpu))
1157 return paging32E_init_context(vcpu);
1158 else
1159 return paging32_init_context(vcpu);
1160}
1161
1162static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1163{
1164 ASSERT(vcpu);
1165 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
1166 vcpu->mmu.free(vcpu);
1167 vcpu->mmu.root_hpa = INVALID_PAGE;
1168 }
1169}
1170
1171int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1172{
Avi Kivity17c3ba92007-06-04 15:58:30 +03001173 destroy_kvm_mmu(vcpu);
1174 return init_kvm_mmu(vcpu);
1175}
Eddie Dong8668a3c2007-10-10 14:26:45 +08001176EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001177
1178int kvm_mmu_load(struct kvm_vcpu *vcpu)
1179{
Avi Kivity714b93d2007-01-05 16:36:53 -08001180 int r;
1181
Shaohua Li11ec2802007-07-23 14:51:37 +08001182 mutex_lock(&vcpu->kvm->lock);
Avi Kivitye2dec932007-01-05 16:36:54 -08001183 r = mmu_topup_memory_caches(vcpu);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001184 if (r)
1185 goto out;
1186 mmu_alloc_roots(vcpu);
Christian Ehrhardtcbdd1be2007-09-09 15:41:59 +03001187 kvm_x86_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
Avi Kivity17c3ba92007-06-04 15:58:30 +03001188 kvm_mmu_flush_tlb(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001189out:
Shaohua Li11ec2802007-07-23 14:51:37 +08001190 mutex_unlock(&vcpu->kvm->lock);
Avi Kivity714b93d2007-01-05 16:36:53 -08001191 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001192}
Avi Kivity17c3ba92007-06-04 15:58:30 +03001193EXPORT_SYMBOL_GPL(kvm_mmu_load);
1194
1195void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1196{
1197 mmu_free_roots(vcpu);
1198}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001199
Avi Kivity09072da2007-05-01 14:16:52 +03001200static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
Avi Kivityac1b7142007-03-08 17:13:32 +02001201 struct kvm_mmu_page *page,
1202 u64 *spte)
1203{
1204 u64 pte;
1205 struct kvm_mmu_page *child;
1206
1207 pte = *spte;
Avi Kivityc7addb92007-09-16 18:58:32 +02001208 if (is_shadow_present_pte(pte)) {
Avi Kivityac1b7142007-03-08 17:13:32 +02001209 if (page->role.level == PT_PAGE_TABLE_LEVEL)
Izik Eidus290fc382007-09-27 14:11:22 +02001210 rmap_remove(vcpu->kvm, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02001211 else {
1212 child = page_header(pte & PT64_BASE_ADDR_MASK);
Avi Kivity90cb0522007-07-17 13:04:56 +03001213 mmu_page_remove_parent_pte(child, spte);
Avi Kivityac1b7142007-03-08 17:13:32 +02001214 }
1215 }
Avi Kivityc7addb92007-09-16 18:58:32 +02001216 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
Avi Kivityd9e368d2007-06-07 19:18:30 +03001217 kvm_flush_remote_tlbs(vcpu->kvm);
Avi Kivityac1b7142007-03-08 17:13:32 +02001218}
1219
Avi Kivity00284252007-05-01 16:53:31 +03001220static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1221 struct kvm_mmu_page *page,
1222 u64 *spte,
Avi Kivityc7addb92007-09-16 18:58:32 +02001223 const void *new, int bytes,
1224 int offset_in_pte)
Avi Kivity00284252007-05-01 16:53:31 +03001225{
1226 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1227 return;
1228
1229 if (page->role.glevels == PT32_ROOT_LEVEL)
Avi Kivityc7addb92007-09-16 18:58:32 +02001230 paging32_update_pte(vcpu, page, spte, new, bytes,
1231 offset_in_pte);
Avi Kivity00284252007-05-01 16:53:31 +03001232 else
Avi Kivityc7addb92007-09-16 18:58:32 +02001233 paging64_update_pte(vcpu, page, spte, new, bytes,
1234 offset_in_pte);
Avi Kivity00284252007-05-01 16:53:31 +03001235}
1236
Avi Kivity12b7d282007-09-23 14:10:49 +02001237static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
1238{
1239 u64 *spte = vcpu->last_pte_updated;
1240
1241 return !!(spte && (*spte & PT_ACCESSED_MASK));
1242}
1243
Avi Kivity09072da2007-05-01 14:16:52 +03001244void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
Shaohua Life551882007-07-23 14:51:39 +08001245 const u8 *new, int bytes)
Avi Kivityda4a00f2007-01-05 16:36:44 -08001246{
Avi Kivity9b7a0322007-01-05 16:36:45 -08001247 gfn_t gfn = gpa >> PAGE_SHIFT;
1248 struct kvm_mmu_page *page;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001249 struct hlist_node *node, *n;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001250 struct hlist_head *bucket;
1251 unsigned index;
1252 u64 *spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001253 unsigned offset = offset_in_page(gpa);
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001254 unsigned pte_size;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001255 unsigned page_offset;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001256 unsigned misaligned;
Avi Kivityfce06572007-05-01 16:44:05 +03001257 unsigned quadrant;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001258 int level;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001259 int flooded = 0;
Avi Kivityac1b7142007-03-08 17:13:32 +02001260 int npte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001261
Avi Kivityda4a00f2007-01-05 16:36:44 -08001262 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
Avi Kivityc7addb92007-09-16 18:58:32 +02001263 kvm_mmu_audit(vcpu, "pre pte write");
Avi Kivity12b7d282007-09-23 14:10:49 +02001264 if (gfn == vcpu->last_pt_write_gfn
1265 && !last_updated_pte_accessed(vcpu)) {
Avi Kivity86a5ba02007-01-05 16:36:50 -08001266 ++vcpu->last_pt_write_count;
1267 if (vcpu->last_pt_write_count >= 3)
1268 flooded = 1;
1269 } else {
1270 vcpu->last_pt_write_gfn = gfn;
1271 vcpu->last_pt_write_count = 1;
Avi Kivity12b7d282007-09-23 14:10:49 +02001272 vcpu->last_pte_updated = NULL;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001273 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001274 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1275 bucket = &vcpu->kvm->mmu_page_hash[index];
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001276 hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
Avi Kivity9b7a0322007-01-05 16:36:45 -08001277 if (page->gfn != gfn || page->role.metaphysical)
1278 continue;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001279 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1280 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
Avi Kivitye925c5b2007-04-30 14:47:02 +03001281 misaligned |= bytes < 4;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001282 if (misaligned || flooded) {
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001283 /*
1284 * Misaligned accesses are too much trouble to fix
1285 * up; also, they usually indicate a page is not used
1286 * as a page table.
Avi Kivity86a5ba02007-01-05 16:36:50 -08001287 *
1288 * If we're seeing too many writes to a page,
1289 * it may no longer be a page table, or we may be
1290 * forking, in which case it is better to unmap the
1291 * page.
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001292 */
1293 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1294 gpa, bytes, page->role.word);
Avi Kivity90cb0522007-07-17 13:04:56 +03001295 kvm_mmu_zap_page(vcpu->kvm, page);
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001296 continue;
1297 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001298 page_offset = offset;
1299 level = page->role.level;
Avi Kivityac1b7142007-03-08 17:13:32 +02001300 npte = 1;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001301 if (page->role.glevels == PT32_ROOT_LEVEL) {
Avi Kivityac1b7142007-03-08 17:13:32 +02001302 page_offset <<= 1; /* 32->64 */
1303 /*
1304 * A 32-bit pde maps 4MB while the shadow pdes map
1305 * only 2MB. So we need to double the offset again
1306 * and zap two pdes instead of one.
1307 */
1308 if (level == PT32_ROOT_LEVEL) {
Avi Kivity6b8d0f92007-04-18 11:18:18 +03001309 page_offset &= ~7; /* kill rounding error */
Avi Kivityac1b7142007-03-08 17:13:32 +02001310 page_offset <<= 1;
1311 npte = 2;
1312 }
Avi Kivityfce06572007-05-01 16:44:05 +03001313 quadrant = page_offset >> PAGE_SHIFT;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001314 page_offset &= ~PAGE_MASK;
Avi Kivityfce06572007-05-01 16:44:05 +03001315 if (quadrant != page->role.quadrant)
1316 continue;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001317 }
Avi Kivity47ad8e62007-05-06 15:50:58 +03001318 spte = &page->spt[page_offset / sizeof(*spte)];
Avi Kivityac1b7142007-03-08 17:13:32 +02001319 while (npte--) {
Avi Kivity09072da2007-05-01 14:16:52 +03001320 mmu_pte_write_zap_pte(vcpu, page, spte);
Avi Kivityc7addb92007-09-16 18:58:32 +02001321 mmu_pte_write_new_pte(vcpu, page, spte, new, bytes,
1322 page_offset & (pte_size - 1));
Avi Kivityac1b7142007-03-08 17:13:32 +02001323 ++spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001324 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001325 }
Avi Kivityc7addb92007-09-16 18:58:32 +02001326 kvm_mmu_audit(vcpu, "post pte write");
Avi Kivityda4a00f2007-01-05 16:36:44 -08001327}
1328
Avi Kivitya4360362007-01-05 16:36:45 -08001329int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1330{
1331 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
1332
Anthony Liguorif67a46f2007-10-10 19:25:50 -05001333 return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
Avi Kivitya4360362007-01-05 16:36:45 -08001334}
1335
Avi Kivity22d95b12007-09-14 20:26:06 +03001336void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
Avi Kivityebeace82007-01-05 16:36:47 -08001337{
1338 while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
1339 struct kvm_mmu_page *page;
1340
1341 page = container_of(vcpu->kvm->active_mmu_pages.prev,
1342 struct kvm_mmu_page, link);
Avi Kivity90cb0522007-07-17 13:04:56 +03001343 kvm_mmu_zap_page(vcpu->kvm, page);
Avi Kivityebeace82007-01-05 16:36:47 -08001344 }
1345}
Avi Kivityebeace82007-01-05 16:36:47 -08001346
Avi Kivity6aa8b732006-12-10 02:21:36 -08001347static void free_mmu_pages(struct kvm_vcpu *vcpu)
1348{
Avi Kivityf51234c2007-01-05 16:36:52 -08001349 struct kvm_mmu_page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001350
Avi Kivityf51234c2007-01-05 16:36:52 -08001351 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1352 page = container_of(vcpu->kvm->active_mmu_pages.next,
1353 struct kvm_mmu_page, link);
Avi Kivity90cb0522007-07-17 13:04:56 +03001354 kvm_mmu_zap_page(vcpu->kvm, page);
Avi Kivityf51234c2007-01-05 16:36:52 -08001355 }
Avi Kivity17ac10a2007-01-05 16:36:40 -08001356 free_page((unsigned long)vcpu->mmu.pae_root);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001357}
1358
1359static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1360{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001361 struct page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001362 int i;
1363
1364 ASSERT(vcpu);
1365
Izik Eidus82ce2c92007-10-02 18:52:55 +02001366 if (vcpu->kvm->n_requested_mmu_pages)
1367 vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages;
1368 else
1369 vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001370 /*
1371 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1372 * Therefore we need to allocate shadow page tables in the first
1373 * 4GB of memory, which happens to fit the DMA32 zone.
1374 */
1375 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1376 if (!page)
1377 goto error_1;
1378 vcpu->mmu.pae_root = page_address(page);
1379 for (i = 0; i < 4; ++i)
1380 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1381
Avi Kivity6aa8b732006-12-10 02:21:36 -08001382 return 0;
1383
1384error_1:
1385 free_mmu_pages(vcpu);
1386 return -ENOMEM;
1387}
1388
Ingo Molnar8018c272006-12-29 16:50:01 -08001389int kvm_mmu_create(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001390{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001391 ASSERT(vcpu);
1392 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001393
Ingo Molnar8018c272006-12-29 16:50:01 -08001394 return alloc_mmu_pages(vcpu);
1395}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001396
Ingo Molnar8018c272006-12-29 16:50:01 -08001397int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1398{
1399 ASSERT(vcpu);
1400 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
Avi Kivity2c264952006-12-22 01:05:28 -08001401
Ingo Molnar8018c272006-12-29 16:50:01 -08001402 return init_kvm_mmu(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001403}
1404
1405void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1406{
1407 ASSERT(vcpu);
1408
1409 destroy_kvm_mmu(vcpu);
1410 free_mmu_pages(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001411 mmu_free_memory_caches(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001412}
1413
Avi Kivity90cb0522007-07-17 13:04:56 +03001414void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001415{
1416 struct kvm_mmu_page *page;
1417
1418 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1419 int i;
1420 u64 *pt;
1421
1422 if (!test_bit(slot, &page->slot_bitmap))
1423 continue;
1424
Avi Kivity47ad8e62007-05-06 15:50:58 +03001425 pt = page->spt;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001426 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1427 /* avoid RMW */
Izik Eidus9647c142007-10-16 14:43:46 +02001428 if (pt[i] & PT_WRITABLE_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001429 pt[i] &= ~PT_WRITABLE_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001430 }
1431}
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001432
Avi Kivity90cb0522007-07-17 13:04:56 +03001433void kvm_mmu_zap_all(struct kvm *kvm)
Dor Laore0fa8262007-03-30 13:06:33 +03001434{
Avi Kivity90cb0522007-07-17 13:04:56 +03001435 struct kvm_mmu_page *page, *node;
Dor Laore0fa8262007-03-30 13:06:33 +03001436
Avi Kivity90cb0522007-07-17 13:04:56 +03001437 list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link)
1438 kvm_mmu_zap_page(kvm, page);
Dor Laore0fa8262007-03-30 13:06:33 +03001439
Avi Kivity90cb0522007-07-17 13:04:56 +03001440 kvm_flush_remote_tlbs(kvm);
Dor Laore0fa8262007-03-30 13:06:33 +03001441}
1442
Avi Kivityb5a33a72007-04-15 16:31:09 +03001443void kvm_mmu_module_exit(void)
1444{
1445 if (pte_chain_cache)
1446 kmem_cache_destroy(pte_chain_cache);
1447 if (rmap_desc_cache)
1448 kmem_cache_destroy(rmap_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +03001449 if (mmu_page_header_cache)
1450 kmem_cache_destroy(mmu_page_header_cache);
Avi Kivityb5a33a72007-04-15 16:31:09 +03001451}
1452
1453int kvm_mmu_module_init(void)
1454{
1455 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
1456 sizeof(struct kvm_pte_chain),
Paul Mundt20c2df82007-07-20 10:11:58 +09001457 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03001458 if (!pte_chain_cache)
1459 goto nomem;
1460 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
1461 sizeof(struct kvm_rmap_desc),
Paul Mundt20c2df82007-07-20 10:11:58 +09001462 0, 0, NULL);
Avi Kivityb5a33a72007-04-15 16:31:09 +03001463 if (!rmap_desc_cache)
1464 goto nomem;
1465
Avi Kivityd3d25b02007-05-30 12:34:53 +03001466 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
1467 sizeof(struct kvm_mmu_page),
Paul Mundt20c2df82007-07-20 10:11:58 +09001468 0, 0, NULL);
Avi Kivityd3d25b02007-05-30 12:34:53 +03001469 if (!mmu_page_header_cache)
1470 goto nomem;
1471
Avi Kivityb5a33a72007-04-15 16:31:09 +03001472 return 0;
1473
1474nomem:
1475 kvm_mmu_module_exit();
1476 return -ENOMEM;
1477}
1478
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001479#ifdef AUDIT
1480
1481static const char *audit_msg;
1482
1483static gva_t canonicalize(gva_t gva)
1484{
1485#ifdef CONFIG_X86_64
1486 gva = (long long)(gva << 16) >> 16;
1487#endif
1488 return gva;
1489}
1490
1491static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1492 gva_t va, int level)
1493{
1494 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1495 int i;
1496 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1497
1498 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1499 u64 ent = pt[i];
1500
Avi Kivityc7addb92007-09-16 18:58:32 +02001501 if (ent == shadow_trap_nonpresent_pte)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001502 continue;
1503
1504 va = canonicalize(va);
Avi Kivityc7addb92007-09-16 18:58:32 +02001505 if (level > 1) {
1506 if (ent == shadow_notrap_nonpresent_pte)
1507 printk(KERN_ERR "audit: (%s) nontrapping pte"
1508 " in nonleaf level: levels %d gva %lx"
1509 " level %d pte %llx\n", audit_msg,
1510 vcpu->mmu.root_level, va, level, ent);
1511
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001512 audit_mappings_page(vcpu, ent, va, level - 1);
Avi Kivityc7addb92007-09-16 18:58:32 +02001513 } else {
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001514 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
1515 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
1516
Avi Kivityc7addb92007-09-16 18:58:32 +02001517 if (is_shadow_present_pte(ent)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001518 && (ent & PT64_BASE_ADDR_MASK) != hpa)
Avi Kivityc7addb92007-09-16 18:58:32 +02001519 printk(KERN_ERR "xx audit error: (%s) levels %d"
1520 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001521 audit_msg, vcpu->mmu.root_level,
Mike Dayd77c26f2007-10-08 09:02:08 -04001522 va, gpa, hpa, ent,
1523 is_shadow_present_pte(ent));
Avi Kivityc7addb92007-09-16 18:58:32 +02001524 else if (ent == shadow_notrap_nonpresent_pte
1525 && !is_error_hpa(hpa))
1526 printk(KERN_ERR "audit: (%s) notrap shadow,"
1527 " valid guest gva %lx\n", audit_msg, va);
1528
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001529 }
1530 }
1531}
1532
1533static void audit_mappings(struct kvm_vcpu *vcpu)
1534{
Avi Kivity1ea252a2007-03-08 11:48:09 +02001535 unsigned i;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001536
1537 if (vcpu->mmu.root_level == 4)
1538 audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
1539 else
1540 for (i = 0; i < 4; ++i)
1541 if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
1542 audit_mappings_page(vcpu,
1543 vcpu->mmu.pae_root[i],
1544 i << 30,
1545 2);
1546}
1547
1548static int count_rmaps(struct kvm_vcpu *vcpu)
1549{
1550 int nmaps = 0;
1551 int i, j, k;
1552
1553 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1554 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1555 struct kvm_rmap_desc *d;
1556
1557 for (j = 0; j < m->npages; ++j) {
Izik Eidus290fc382007-09-27 14:11:22 +02001558 unsigned long *rmapp = &m->rmap[j];
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001559
Izik Eidus290fc382007-09-27 14:11:22 +02001560 if (!*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001561 continue;
Izik Eidus290fc382007-09-27 14:11:22 +02001562 if (!(*rmapp & 1)) {
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001563 ++nmaps;
1564 continue;
1565 }
Izik Eidus290fc382007-09-27 14:11:22 +02001566 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001567 while (d) {
1568 for (k = 0; k < RMAP_EXT; ++k)
1569 if (d->shadow_ptes[k])
1570 ++nmaps;
1571 else
1572 break;
1573 d = d->more;
1574 }
1575 }
1576 }
1577 return nmaps;
1578}
1579
1580static int count_writable_mappings(struct kvm_vcpu *vcpu)
1581{
1582 int nmaps = 0;
1583 struct kvm_mmu_page *page;
1584 int i;
1585
1586 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
Avi Kivity47ad8e62007-05-06 15:50:58 +03001587 u64 *pt = page->spt;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001588
1589 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1590 continue;
1591
1592 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1593 u64 ent = pt[i];
1594
1595 if (!(ent & PT_PRESENT_MASK))
1596 continue;
1597 if (!(ent & PT_WRITABLE_MASK))
1598 continue;
1599 ++nmaps;
1600 }
1601 }
1602 return nmaps;
1603}
1604
1605static void audit_rmap(struct kvm_vcpu *vcpu)
1606{
1607 int n_rmap = count_rmaps(vcpu);
1608 int n_actual = count_writable_mappings(vcpu);
1609
1610 if (n_rmap != n_actual)
1611 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1612 __FUNCTION__, audit_msg, n_rmap, n_actual);
1613}
1614
1615static void audit_write_protection(struct kvm_vcpu *vcpu)
1616{
1617 struct kvm_mmu_page *page;
Izik Eidus290fc382007-09-27 14:11:22 +02001618 struct kvm_memory_slot *slot;
1619 unsigned long *rmapp;
1620 gfn_t gfn;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001621
1622 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001623 if (page->role.metaphysical)
1624 continue;
1625
Izik Eidus290fc382007-09-27 14:11:22 +02001626 slot = gfn_to_memslot(vcpu->kvm, page->gfn);
1627 gfn = unalias_gfn(vcpu->kvm, page->gfn);
1628 rmapp = &slot->rmap[gfn - slot->base_gfn];
1629 if (*rmapp)
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001630 printk(KERN_ERR "%s: (%s) shadow page has writable"
1631 " mappings: gfn %lx role %x\n",
1632 __FUNCTION__, audit_msg, page->gfn,
1633 page->role.word);
1634 }
1635}
1636
1637static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1638{
1639 int olddbg = dbg;
1640
1641 dbg = 0;
1642 audit_msg = msg;
1643 audit_rmap(vcpu);
1644 audit_write_protection(vcpu);
1645 audit_mappings(vcpu);
1646 dbg = olddbg;
1647}
1648
1649#endif