blob: 3cdbf687df25d8e57aff64ebab1fbeb07c91fbd3 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19#include <linux/types.h>
20#include <linux/string.h>
21#include <asm/page.h>
22#include <linux/mm.h>
23#include <linux/highmem.h>
24#include <linux/module.h>
25
26#include "vmx.h"
27#include "kvm.h"
28
Avi Kivity37a7d8b2007-01-05 16:36:56 -080029#undef MMU_DEBUG
30
31#undef AUDIT
32
33#ifdef AUDIT
34static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
35#else
36static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
37#endif
38
39#ifdef MMU_DEBUG
40
41#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
42#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
43
44#else
45
46#define pgprintk(x...) do { } while (0)
47#define rmap_printk(x...) do { } while (0)
48
49#endif
50
51#if defined(MMU_DEBUG) || defined(AUDIT)
52static int dbg = 1;
53#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080054
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080055#ifndef MMU_DEBUG
56#define ASSERT(x) do { } while (0)
57#else
Avi Kivity6aa8b732006-12-10 02:21:36 -080058#define ASSERT(x) \
59 if (!(x)) { \
60 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
61 __FILE__, __LINE__, #x); \
62 }
Yaozu Dongd6c69ee2007-04-25 14:17:25 +080063#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -080064
Avi Kivitycea0f0e2007-01-05 16:36:43 -080065#define PT64_PT_BITS 9
66#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
67#define PT32_PT_BITS 10
68#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
Avi Kivity6aa8b732006-12-10 02:21:36 -080069
70#define PT_WRITABLE_SHIFT 1
71
72#define PT_PRESENT_MASK (1ULL << 0)
73#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
74#define PT_USER_MASK (1ULL << 2)
75#define PT_PWT_MASK (1ULL << 3)
76#define PT_PCD_MASK (1ULL << 4)
77#define PT_ACCESSED_MASK (1ULL << 5)
78#define PT_DIRTY_MASK (1ULL << 6)
79#define PT_PAGE_SIZE_MASK (1ULL << 7)
80#define PT_PAT_MASK (1ULL << 7)
81#define PT_GLOBAL_MASK (1ULL << 8)
82#define PT64_NX_MASK (1ULL << 63)
83
84#define PT_PAT_SHIFT 7
85#define PT_DIR_PAT_SHIFT 12
86#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
87
88#define PT32_DIR_PSE36_SIZE 4
89#define PT32_DIR_PSE36_SHIFT 13
90#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
91
92
93#define PT32_PTE_COPY_MASK \
Avi Kivity8c7bb722006-12-13 00:34:02 -080094 (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -080095
Avi Kivity8c7bb722006-12-13 00:34:02 -080096#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
Avi Kivity6aa8b732006-12-10 02:21:36 -080097
98#define PT_FIRST_AVAIL_BITS_SHIFT 9
99#define PT64_SECOND_AVAIL_BITS_SHIFT 52
100
101#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
102#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
103
104#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
105#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
106
107#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
108#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
109
110#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
111
112#define VALID_PAGE(x) ((x) != INVALID_PAGE)
113
114#define PT64_LEVEL_BITS 9
115
116#define PT64_LEVEL_SHIFT(level) \
117 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
118
119#define PT64_LEVEL_MASK(level) \
120 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
121
122#define PT64_INDEX(address, level)\
123 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
124
125
126#define PT32_LEVEL_BITS 10
127
128#define PT32_LEVEL_SHIFT(level) \
129 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
130
131#define PT32_LEVEL_MASK(level) \
132 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
133
134#define PT32_INDEX(address, level)\
135 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
136
137
Avi Kivity27aba762007-03-09 13:04:31 +0200138#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800139#define PT64_DIR_BASE_ADDR_MASK \
140 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
141
142#define PT32_BASE_ADDR_MASK PAGE_MASK
143#define PT32_DIR_BASE_ADDR_MASK \
144 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
145
146
147#define PFERR_PRESENT_MASK (1U << 0)
148#define PFERR_WRITE_MASK (1U << 1)
149#define PFERR_USER_MASK (1U << 2)
Avi Kivity73b10872007-01-26 00:56:41 -0800150#define PFERR_FETCH_MASK (1U << 4)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800151
152#define PT64_ROOT_LEVEL 4
153#define PT32_ROOT_LEVEL 2
154#define PT32E_ROOT_LEVEL 3
155
156#define PT_DIRECTORY_LEVEL 2
157#define PT_PAGE_TABLE_LEVEL 1
158
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800159#define RMAP_EXT 4
160
161struct kvm_rmap_desc {
162 u64 *shadow_ptes[RMAP_EXT];
163 struct kvm_rmap_desc *more;
164};
165
Avi Kivityb5a33a72007-04-15 16:31:09 +0300166static struct kmem_cache *pte_chain_cache;
167static struct kmem_cache *rmap_desc_cache;
Avi Kivityd3d25b02007-05-30 12:34:53 +0300168static struct kmem_cache *mmu_page_cache;
169static struct kmem_cache *mmu_page_header_cache;
Avi Kivityb5a33a72007-04-15 16:31:09 +0300170
Avi Kivity6aa8b732006-12-10 02:21:36 -0800171static int is_write_protection(struct kvm_vcpu *vcpu)
172{
173 return vcpu->cr0 & CR0_WP_MASK;
174}
175
176static int is_cpuid_PSE36(void)
177{
178 return 1;
179}
180
Avi Kivity73b10872007-01-26 00:56:41 -0800181static int is_nx(struct kvm_vcpu *vcpu)
182{
183 return vcpu->shadow_efer & EFER_NX;
184}
185
Avi Kivity6aa8b732006-12-10 02:21:36 -0800186static int is_present_pte(unsigned long pte)
187{
188 return pte & PT_PRESENT_MASK;
189}
190
191static int is_writeble_pte(unsigned long pte)
192{
193 return pte & PT_WRITABLE_MASK;
194}
195
196static int is_io_pte(unsigned long pte)
197{
198 return pte & PT_SHADOW_IO_MARK;
199}
200
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800201static int is_rmap_pte(u64 pte)
202{
203 return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
204 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
205}
206
Avi Kivitye2dec932007-01-05 16:36:54 -0800207static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
Avi Kivity8c438502007-04-16 11:53:17 +0300208 struct kmem_cache *base_cache, int min,
209 gfp_t gfp_flags)
Avi Kivity714b93d2007-01-05 16:36:53 -0800210{
211 void *obj;
212
213 if (cache->nobjs >= min)
Avi Kivitye2dec932007-01-05 16:36:54 -0800214 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800215 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
Avi Kivity8c438502007-04-16 11:53:17 +0300216 obj = kmem_cache_zalloc(base_cache, gfp_flags);
Avi Kivity714b93d2007-01-05 16:36:53 -0800217 if (!obj)
Avi Kivitye2dec932007-01-05 16:36:54 -0800218 return -ENOMEM;
Avi Kivity714b93d2007-01-05 16:36:53 -0800219 cache->objects[cache->nobjs++] = obj;
220 }
Avi Kivitye2dec932007-01-05 16:36:54 -0800221 return 0;
Avi Kivity714b93d2007-01-05 16:36:53 -0800222}
223
224static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
225{
226 while (mc->nobjs)
227 kfree(mc->objects[--mc->nobjs]);
228}
229
Avi Kivity8c438502007-04-16 11:53:17 +0300230static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
Avi Kivity714b93d2007-01-05 16:36:53 -0800231{
Avi Kivitye2dec932007-01-05 16:36:54 -0800232 int r;
233
234 r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
Avi Kivity8c438502007-04-16 11:53:17 +0300235 pte_chain_cache, 4, gfp_flags);
Avi Kivitye2dec932007-01-05 16:36:54 -0800236 if (r)
237 goto out;
238 r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
Avi Kivity8c438502007-04-16 11:53:17 +0300239 rmap_desc_cache, 1, gfp_flags);
Avi Kivityd3d25b02007-05-30 12:34:53 +0300240 if (r)
241 goto out;
242 r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
243 mmu_page_cache, 4, gfp_flags);
244 if (r)
245 goto out;
246 r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
247 mmu_page_header_cache, 4, gfp_flags);
Avi Kivitye2dec932007-01-05 16:36:54 -0800248out:
249 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -0800250}
251
Avi Kivity8c438502007-04-16 11:53:17 +0300252static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
253{
254 int r;
255
256 r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
257 if (r < 0) {
258 spin_unlock(&vcpu->kvm->lock);
259 kvm_arch_ops->vcpu_put(vcpu);
260 r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
261 kvm_arch_ops->vcpu_load(vcpu);
262 spin_lock(&vcpu->kvm->lock);
263 }
264 return r;
265}
266
Avi Kivity714b93d2007-01-05 16:36:53 -0800267static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
268{
269 mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
270 mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +0300271 mmu_free_memory_cache(&vcpu->mmu_page_cache);
272 mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
Avi Kivity714b93d2007-01-05 16:36:53 -0800273}
274
275static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
276 size_t size)
277{
278 void *p;
279
280 BUG_ON(!mc->nobjs);
281 p = mc->objects[--mc->nobjs];
282 memset(p, 0, size);
283 return p;
284}
285
286static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj)
287{
288 if (mc->nobjs < KVM_NR_MEM_OBJS)
289 mc->objects[mc->nobjs++] = obj;
290 else
291 kfree(obj);
292}
293
294static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
295{
296 return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
297 sizeof(struct kvm_pte_chain));
298}
299
300static void mmu_free_pte_chain(struct kvm_vcpu *vcpu,
301 struct kvm_pte_chain *pc)
302{
303 mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc);
304}
305
306static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
307{
308 return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
309 sizeof(struct kvm_rmap_desc));
310}
311
312static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu,
313 struct kvm_rmap_desc *rd)
314{
315 mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd);
316}
317
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800318/*
319 * Reverse mapping data structures:
320 *
321 * If page->private bit zero is zero, then page->private points to the
322 * shadow page table entry that points to page_address(page).
323 *
324 * If page->private bit zero is one, (then page->private & ~1) points
325 * to a struct kvm_rmap_desc containing more mappings.
326 */
Avi Kivity714b93d2007-01-05 16:36:53 -0800327static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800328{
329 struct page *page;
330 struct kvm_rmap_desc *desc;
331 int i;
332
333 if (!is_rmap_pte(*spte))
334 return;
335 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
Markus Rechberger5972e952007-02-19 14:37:47 +0200336 if (!page_private(page)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800337 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
Markus Rechberger5972e952007-02-19 14:37:47 +0200338 set_page_private(page,(unsigned long)spte);
339 } else if (!(page_private(page) & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800340 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
Avi Kivity714b93d2007-01-05 16:36:53 -0800341 desc = mmu_alloc_rmap_desc(vcpu);
Markus Rechberger5972e952007-02-19 14:37:47 +0200342 desc->shadow_ptes[0] = (u64 *)page_private(page);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800343 desc->shadow_ptes[1] = spte;
Markus Rechberger5972e952007-02-19 14:37:47 +0200344 set_page_private(page,(unsigned long)desc | 1);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800345 } else {
346 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
Markus Rechberger5972e952007-02-19 14:37:47 +0200347 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800348 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
349 desc = desc->more;
350 if (desc->shadow_ptes[RMAP_EXT-1]) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800351 desc->more = mmu_alloc_rmap_desc(vcpu);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800352 desc = desc->more;
353 }
354 for (i = 0; desc->shadow_ptes[i]; ++i)
355 ;
356 desc->shadow_ptes[i] = spte;
357 }
358}
359
Avi Kivity714b93d2007-01-05 16:36:53 -0800360static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu,
361 struct page *page,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800362 struct kvm_rmap_desc *desc,
363 int i,
364 struct kvm_rmap_desc *prev_desc)
365{
366 int j;
367
368 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
369 ;
370 desc->shadow_ptes[i] = desc->shadow_ptes[j];
Al Viro11718b4d2007-02-09 16:39:20 +0000371 desc->shadow_ptes[j] = NULL;
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800372 if (j != 0)
373 return;
374 if (!prev_desc && !desc->more)
Markus Rechberger5972e952007-02-19 14:37:47 +0200375 set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800376 else
377 if (prev_desc)
378 prev_desc->more = desc->more;
379 else
Markus Rechberger5972e952007-02-19 14:37:47 +0200380 set_page_private(page,(unsigned long)desc->more | 1);
Avi Kivity714b93d2007-01-05 16:36:53 -0800381 mmu_free_rmap_desc(vcpu, desc);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800382}
383
Avi Kivity714b93d2007-01-05 16:36:53 -0800384static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte)
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800385{
386 struct page *page;
387 struct kvm_rmap_desc *desc;
388 struct kvm_rmap_desc *prev_desc;
389 int i;
390
391 if (!is_rmap_pte(*spte))
392 return;
393 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
Markus Rechberger5972e952007-02-19 14:37:47 +0200394 if (!page_private(page)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800395 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
396 BUG();
Markus Rechberger5972e952007-02-19 14:37:47 +0200397 } else if (!(page_private(page) & 1)) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800398 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
Markus Rechberger5972e952007-02-19 14:37:47 +0200399 if ((u64 *)page_private(page) != spte) {
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800400 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
401 spte, *spte);
402 BUG();
403 }
Markus Rechberger5972e952007-02-19 14:37:47 +0200404 set_page_private(page,0);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800405 } else {
406 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
Markus Rechberger5972e952007-02-19 14:37:47 +0200407 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800408 prev_desc = NULL;
409 while (desc) {
410 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
411 if (desc->shadow_ptes[i] == spte) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800412 rmap_desc_remove_entry(vcpu, page,
413 desc, i,
Avi Kivitycd4a4e52007-01-05 16:36:38 -0800414 prev_desc);
415 return;
416 }
417 prev_desc = desc;
418 desc = desc->more;
419 }
420 BUG();
421 }
422}
423
Avi Kivity714b93d2007-01-05 16:36:53 -0800424static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
Avi Kivity374cbac2007-01-05 16:36:43 -0800425{
Avi Kivity714b93d2007-01-05 16:36:53 -0800426 struct kvm *kvm = vcpu->kvm;
Avi Kivity374cbac2007-01-05 16:36:43 -0800427 struct page *page;
Avi Kivity374cbac2007-01-05 16:36:43 -0800428 struct kvm_rmap_desc *desc;
429 u64 *spte;
430
Avi Kivity954bbbc2007-03-30 14:02:32 +0300431 page = gfn_to_page(kvm, gfn);
432 BUG_ON(!page);
Avi Kivity374cbac2007-01-05 16:36:43 -0800433
Markus Rechberger5972e952007-02-19 14:37:47 +0200434 while (page_private(page)) {
435 if (!(page_private(page) & 1))
436 spte = (u64 *)page_private(page);
Avi Kivity374cbac2007-01-05 16:36:43 -0800437 else {
Markus Rechberger5972e952007-02-19 14:37:47 +0200438 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
Avi Kivity374cbac2007-01-05 16:36:43 -0800439 spte = desc->shadow_ptes[0];
440 }
441 BUG_ON(!spte);
Avi Kivity27aba762007-03-09 13:04:31 +0200442 BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
443 != page_to_pfn(page));
Avi Kivity374cbac2007-01-05 16:36:43 -0800444 BUG_ON(!(*spte & PT_PRESENT_MASK));
445 BUG_ON(!(*spte & PT_WRITABLE_MASK));
446 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
Avi Kivity714b93d2007-01-05 16:36:53 -0800447 rmap_remove(vcpu, spte);
Avi Kivity40907d52007-01-05 16:36:55 -0800448 kvm_arch_ops->tlb_flush(vcpu);
Avi Kivity374cbac2007-01-05 16:36:43 -0800449 *spte &= ~(u64)PT_WRITABLE_MASK;
450 }
451}
452
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800453#ifdef MMU_DEBUG
Avi Kivity47ad8e62007-05-06 15:50:58 +0300454static int is_empty_shadow_page(u64 *spt)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800455{
Avi Kivity139bdb22007-01-05 16:36:50 -0800456 u64 *pos;
457 u64 *end;
458
Avi Kivity47ad8e62007-05-06 15:50:58 +0300459 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
Avi Kivity139bdb22007-01-05 16:36:50 -0800460 if (*pos != 0) {
461 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
462 pos, *pos);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800463 return 0;
Avi Kivity139bdb22007-01-05 16:36:50 -0800464 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800465 return 1;
466}
Yaozu Dongd6c69ee2007-04-25 14:17:25 +0800467#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800468
Avi Kivity4b02d6d2007-05-06 15:36:30 +0300469static void kvm_mmu_free_page(struct kvm_vcpu *vcpu,
470 struct kvm_mmu_page *page_head)
Avi Kivity260746c2007-01-05 16:36:49 -0800471{
Avi Kivity47ad8e62007-05-06 15:50:58 +0300472 ASSERT(is_empty_shadow_page(page_head->spt));
Avi Kivityd3d25b02007-05-30 12:34:53 +0300473 list_del(&page_head->link);
474 mmu_memory_cache_free(&vcpu->mmu_page_cache, page_head->spt);
475 mmu_memory_cache_free(&vcpu->mmu_page_header_cache, page_head);
Avi Kivity260746c2007-01-05 16:36:49 -0800476 ++vcpu->kvm->n_free_mmu_pages;
477}
478
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800479static unsigned kvm_page_table_hashfn(gfn_t gfn)
480{
481 return gfn;
482}
483
Avi Kivity25c0de22007-01-05 16:36:42 -0800484static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
485 u64 *parent_pte)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800486{
487 struct kvm_mmu_page *page;
488
Avi Kivityd3d25b02007-05-30 12:34:53 +0300489 if (!vcpu->kvm->n_free_mmu_pages)
Avi Kivity25c0de22007-01-05 16:36:42 -0800490 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800491
Avi Kivityd3d25b02007-05-30 12:34:53 +0300492 page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
493 sizeof *page);
494 page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
495 set_page_private(virt_to_page(page->spt), (unsigned long)page);
496 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
Avi Kivity47ad8e62007-05-06 15:50:58 +0300497 ASSERT(is_empty_shadow_page(page->spt));
Avi Kivity6aa8b732006-12-10 02:21:36 -0800498 page->slot_bitmap = 0;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800499 page->multimapped = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800500 page->parent_pte = parent_pte;
Avi Kivityebeace82007-01-05 16:36:47 -0800501 --vcpu->kvm->n_free_mmu_pages;
Avi Kivity25c0de22007-01-05 16:36:42 -0800502 return page;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800503}
504
Avi Kivity714b93d2007-01-05 16:36:53 -0800505static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
506 struct kvm_mmu_page *page, u64 *parent_pte)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800507{
508 struct kvm_pte_chain *pte_chain;
509 struct hlist_node *node;
510 int i;
511
512 if (!parent_pte)
513 return;
514 if (!page->multimapped) {
515 u64 *old = page->parent_pte;
516
517 if (!old) {
518 page->parent_pte = parent_pte;
519 return;
520 }
521 page->multimapped = 1;
Avi Kivity714b93d2007-01-05 16:36:53 -0800522 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800523 INIT_HLIST_HEAD(&page->parent_ptes);
524 hlist_add_head(&pte_chain->link, &page->parent_ptes);
525 pte_chain->parent_ptes[0] = old;
526 }
527 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
528 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
529 continue;
530 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
531 if (!pte_chain->parent_ptes[i]) {
532 pte_chain->parent_ptes[i] = parent_pte;
533 return;
534 }
535 }
Avi Kivity714b93d2007-01-05 16:36:53 -0800536 pte_chain = mmu_alloc_pte_chain(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800537 BUG_ON(!pte_chain);
538 hlist_add_head(&pte_chain->link, &page->parent_ptes);
539 pte_chain->parent_ptes[0] = parent_pte;
540}
541
Avi Kivity714b93d2007-01-05 16:36:53 -0800542static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu,
543 struct kvm_mmu_page *page,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800544 u64 *parent_pte)
545{
546 struct kvm_pte_chain *pte_chain;
547 struct hlist_node *node;
548 int i;
549
550 if (!page->multimapped) {
551 BUG_ON(page->parent_pte != parent_pte);
552 page->parent_pte = NULL;
553 return;
554 }
555 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
556 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
557 if (!pte_chain->parent_ptes[i])
558 break;
559 if (pte_chain->parent_ptes[i] != parent_pte)
560 continue;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800561 while (i + 1 < NR_PTE_CHAIN_ENTRIES
562 && pte_chain->parent_ptes[i + 1]) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800563 pte_chain->parent_ptes[i]
564 = pte_chain->parent_ptes[i + 1];
565 ++i;
566 }
567 pte_chain->parent_ptes[i] = NULL;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800568 if (i == 0) {
569 hlist_del(&pte_chain->link);
Avi Kivity714b93d2007-01-05 16:36:53 -0800570 mmu_free_pte_chain(vcpu, pte_chain);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800571 if (hlist_empty(&page->parent_ptes)) {
572 page->multimapped = 0;
573 page->parent_pte = NULL;
574 }
575 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800576 return;
577 }
578 BUG();
579}
580
581static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
582 gfn_t gfn)
583{
584 unsigned index;
585 struct hlist_head *bucket;
586 struct kvm_mmu_page *page;
587 struct hlist_node *node;
588
589 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
590 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
591 bucket = &vcpu->kvm->mmu_page_hash[index];
592 hlist_for_each_entry(page, node, bucket, hash_link)
593 if (page->gfn == gfn && !page->role.metaphysical) {
594 pgprintk("%s: found role %x\n",
595 __FUNCTION__, page->role.word);
596 return page;
597 }
598 return NULL;
599}
600
601static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
602 gfn_t gfn,
603 gva_t gaddr,
604 unsigned level,
605 int metaphysical,
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200606 unsigned hugepage_access,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800607 u64 *parent_pte)
608{
609 union kvm_mmu_page_role role;
610 unsigned index;
611 unsigned quadrant;
612 struct hlist_head *bucket;
613 struct kvm_mmu_page *page;
614 struct hlist_node *node;
615
616 role.word = 0;
617 role.glevels = vcpu->mmu.root_level;
618 role.level = level;
619 role.metaphysical = metaphysical;
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200620 role.hugepage_access = hugepage_access;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800621 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
622 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
623 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
624 role.quadrant = quadrant;
625 }
626 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
627 gfn, role.word);
628 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
629 bucket = &vcpu->kvm->mmu_page_hash[index];
630 hlist_for_each_entry(page, node, bucket, hash_link)
631 if (page->gfn == gfn && page->role.word == role.word) {
Avi Kivity714b93d2007-01-05 16:36:53 -0800632 mmu_page_add_parent_pte(vcpu, page, parent_pte);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800633 pgprintk("%s: found\n", __FUNCTION__);
634 return page;
635 }
636 page = kvm_mmu_alloc_page(vcpu, parent_pte);
637 if (!page)
638 return page;
639 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
640 page->gfn = gfn;
641 page->role = role;
642 hlist_add_head(&page->hash_link, bucket);
Avi Kivity374cbac2007-01-05 16:36:43 -0800643 if (!metaphysical)
Avi Kivity714b93d2007-01-05 16:36:53 -0800644 rmap_write_protect(vcpu, gfn);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800645 return page;
646}
647
Avi Kivitya4360362007-01-05 16:36:45 -0800648static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
649 struct kvm_mmu_page *page)
650{
Avi Kivity697fe2e2007-01-05 16:36:46 -0800651 unsigned i;
652 u64 *pt;
653 u64 ent;
654
Avi Kivity47ad8e62007-05-06 15:50:58 +0300655 pt = page->spt;
Avi Kivity697fe2e2007-01-05 16:36:46 -0800656
657 if (page->role.level == PT_PAGE_TABLE_LEVEL) {
658 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
659 if (pt[i] & PT_PRESENT_MASK)
Avi Kivity714b93d2007-01-05 16:36:53 -0800660 rmap_remove(vcpu, &pt[i]);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800661 pt[i] = 0;
662 }
Avi Kivity40907d52007-01-05 16:36:55 -0800663 kvm_arch_ops->tlb_flush(vcpu);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800664 return;
665 }
666
667 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
668 ent = pt[i];
669
670 pt[i] = 0;
671 if (!(ent & PT_PRESENT_MASK))
672 continue;
673 ent &= PT64_BASE_ADDR_MASK;
Avi Kivity714b93d2007-01-05 16:36:53 -0800674 mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
Avi Kivity697fe2e2007-01-05 16:36:46 -0800675 }
Avi Kivitya4360362007-01-05 16:36:45 -0800676}
677
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800678static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
679 struct kvm_mmu_page *page,
680 u64 *parent_pte)
681{
Avi Kivity714b93d2007-01-05 16:36:53 -0800682 mmu_page_remove_parent_pte(vcpu, page, parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800683}
684
685static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
686 struct kvm_mmu_page *page)
687{
688 u64 *parent_pte;
689
690 while (page->multimapped || page->parent_pte) {
691 if (!page->multimapped)
692 parent_pte = page->parent_pte;
693 else {
694 struct kvm_pte_chain *chain;
695
696 chain = container_of(page->parent_ptes.first,
697 struct kvm_pte_chain, link);
698 parent_pte = chain->parent_ptes[0];
699 }
Avi Kivity697fe2e2007-01-05 16:36:46 -0800700 BUG_ON(!parent_pte);
Avi Kivitya4360362007-01-05 16:36:45 -0800701 kvm_mmu_put_page(vcpu, page, parent_pte);
702 *parent_pte = 0;
703 }
Avi Kivitycc4529e2007-01-05 16:36:47 -0800704 kvm_mmu_page_unlink_children(vcpu, page);
Avi Kivity3bb65a22007-01-05 16:36:51 -0800705 if (!page->root_count) {
706 hlist_del(&page->hash_link);
Avi Kivity4b02d6d2007-05-06 15:36:30 +0300707 kvm_mmu_free_page(vcpu, page);
Avi Kivity36868f72007-03-26 19:31:52 +0200708 } else
709 list_move(&page->link, &vcpu->kvm->active_mmu_pages);
Avi Kivitya4360362007-01-05 16:36:45 -0800710}
711
712static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
713{
714 unsigned index;
715 struct hlist_head *bucket;
716 struct kvm_mmu_page *page;
717 struct hlist_node *node, *n;
718 int r;
719
720 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
721 r = 0;
722 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
723 bucket = &vcpu->kvm->mmu_page_hash[index];
724 hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
725 if (page->gfn == gfn && !page->role.metaphysical) {
Avi Kivity697fe2e2007-01-05 16:36:46 -0800726 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
727 page->role.word);
Avi Kivitya4360362007-01-05 16:36:45 -0800728 kvm_mmu_zap_page(vcpu, page);
729 r = 1;
730 }
731 return r;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800732}
733
Avi Kivity97a0a012007-05-31 15:08:29 +0300734static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
735{
736 struct kvm_mmu_page *page;
737
738 while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
739 pgprintk("%s: zap %lx %x\n",
740 __FUNCTION__, gfn, page->role.word);
741 kvm_mmu_zap_page(vcpu, page);
742 }
743}
744
Avi Kivity6aa8b732006-12-10 02:21:36 -0800745static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
746{
747 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
748 struct kvm_mmu_page *page_head = page_header(__pa(pte));
749
750 __set_bit(slot, &page_head->slot_bitmap);
751}
752
753hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
754{
755 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
756
757 return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
758}
759
760hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
761{
Avi Kivity6aa8b732006-12-10 02:21:36 -0800762 struct page *page;
763
764 ASSERT((gpa & HPA_ERR_MASK) == 0);
Avi Kivity954bbbc2007-03-30 14:02:32 +0300765 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
766 if (!page)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800767 return gpa | HPA_ERR_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800768 return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
769 | (gpa & (PAGE_SIZE-1));
770}
771
772hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
773{
774 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
775
776 if (gpa == UNMAPPED_GVA)
777 return UNMAPPED_GVA;
778 return gpa_to_hpa(vcpu, gpa);
779}
780
Avi Kivity039576c2007-03-20 12:46:50 +0200781struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
782{
783 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
784
785 if (gpa == UNMAPPED_GVA)
786 return NULL;
787 return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
788}
789
Avi Kivity6aa8b732006-12-10 02:21:36 -0800790static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
791{
792}
793
794static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
795{
796 int level = PT32E_ROOT_LEVEL;
797 hpa_t table_addr = vcpu->mmu.root_hpa;
798
799 for (; ; level--) {
800 u32 index = PT64_INDEX(v, level);
801 u64 *table;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800802 u64 pte;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800803
804 ASSERT(VALID_PAGE(table_addr));
805 table = __va(table_addr);
806
807 if (level == 1) {
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800808 pte = table[index];
809 if (is_present_pte(pte) && is_writeble_pte(pte))
810 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800811 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
812 page_header_update_slot(vcpu->kvm, table, v);
813 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
814 PT_USER_MASK;
Avi Kivity714b93d2007-01-05 16:36:53 -0800815 rmap_add(vcpu, &table[index]);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800816 return 0;
817 }
818
819 if (table[index] == 0) {
Avi Kivity25c0de22007-01-05 16:36:42 -0800820 struct kvm_mmu_page *new_table;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800821 gfn_t pseudo_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800822
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800823 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
824 >> PAGE_SHIFT;
825 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
826 v, level - 1,
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200827 1, 0, &table[index]);
Avi Kivity25c0de22007-01-05 16:36:42 -0800828 if (!new_table) {
Avi Kivity6aa8b732006-12-10 02:21:36 -0800829 pgprintk("nonpaging_map: ENOMEM\n");
830 return -ENOMEM;
831 }
832
Avi Kivity47ad8e62007-05-06 15:50:58 +0300833 table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
Avi Kivity25c0de22007-01-05 16:36:42 -0800834 | PT_WRITABLE_MASK | PT_USER_MASK;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800835 }
836 table_addr = table[index] & PT64_BASE_ADDR_MASK;
837 }
838}
839
Avi Kivity17ac10a2007-01-05 16:36:40 -0800840static void mmu_free_roots(struct kvm_vcpu *vcpu)
841{
842 int i;
Avi Kivity3bb65a22007-01-05 16:36:51 -0800843 struct kvm_mmu_page *page;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800844
845#ifdef CONFIG_X86_64
846 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
847 hpa_t root = vcpu->mmu.root_hpa;
848
849 ASSERT(VALID_PAGE(root));
Avi Kivity3bb65a22007-01-05 16:36:51 -0800850 page = page_header(root);
851 --page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800852 vcpu->mmu.root_hpa = INVALID_PAGE;
853 return;
854 }
855#endif
856 for (i = 0; i < 4; ++i) {
857 hpa_t root = vcpu->mmu.pae_root[i];
858
Avi Kivity417726a2007-04-12 17:35:58 +0300859 if (root) {
860 ASSERT(VALID_PAGE(root));
861 root &= PT64_BASE_ADDR_MASK;
862 page = page_header(root);
863 --page->root_count;
864 }
Avi Kivity17ac10a2007-01-05 16:36:40 -0800865 vcpu->mmu.pae_root[i] = INVALID_PAGE;
866 }
867 vcpu->mmu.root_hpa = INVALID_PAGE;
868}
869
870static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
871{
872 int i;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800873 gfn_t root_gfn;
Avi Kivity3bb65a22007-01-05 16:36:51 -0800874 struct kvm_mmu_page *page;
875
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800876 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800877
878#ifdef CONFIG_X86_64
879 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
880 hpa_t root = vcpu->mmu.root_hpa;
881
882 ASSERT(!VALID_PAGE(root));
Ingo Molnar68a99f62007-01-05 16:36:59 -0800883 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200884 PT64_ROOT_LEVEL, 0, 0, NULL);
Avi Kivity47ad8e62007-05-06 15:50:58 +0300885 root = __pa(page->spt);
Avi Kivity3bb65a22007-01-05 16:36:51 -0800886 ++page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800887 vcpu->mmu.root_hpa = root;
888 return;
889 }
890#endif
891 for (i = 0; i < 4; ++i) {
892 hpa_t root = vcpu->mmu.pae_root[i];
893
894 ASSERT(!VALID_PAGE(root));
Avi Kivity417726a2007-04-12 17:35:58 +0300895 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
896 if (!is_present_pte(vcpu->pdptrs[i])) {
897 vcpu->mmu.pae_root[i] = 0;
898 continue;
899 }
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800900 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
Avi Kivity417726a2007-04-12 17:35:58 +0300901 } else if (vcpu->mmu.root_level == 0)
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800902 root_gfn = 0;
Ingo Molnar68a99f62007-01-05 16:36:59 -0800903 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800904 PT32_ROOT_LEVEL, !is_paging(vcpu),
Avi Kivityd28c6cfb2007-03-23 09:55:25 +0200905 0, NULL);
Avi Kivity47ad8e62007-05-06 15:50:58 +0300906 root = __pa(page->spt);
Avi Kivity3bb65a22007-01-05 16:36:51 -0800907 ++page->root_count;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800908 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
909 }
910 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
911}
912
Avi Kivity6aa8b732006-12-10 02:21:36 -0800913static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
914{
915 return vaddr;
916}
917
918static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
919 u32 error_code)
920{
Avi Kivity6aa8b732006-12-10 02:21:36 -0800921 gpa_t addr = gva;
Avi Kivityebeace82007-01-05 16:36:47 -0800922 hpa_t paddr;
Avi Kivitye2dec932007-01-05 16:36:54 -0800923 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800924
Avi Kivitye2dec932007-01-05 16:36:54 -0800925 r = mmu_topup_memory_caches(vcpu);
926 if (r)
927 return r;
Avi Kivity714b93d2007-01-05 16:36:53 -0800928
Avi Kivity6aa8b732006-12-10 02:21:36 -0800929 ASSERT(vcpu);
930 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
931
Avi Kivity6aa8b732006-12-10 02:21:36 -0800932
Avi Kivityebeace82007-01-05 16:36:47 -0800933 paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800934
Avi Kivityebeace82007-01-05 16:36:47 -0800935 if (is_error_hpa(paddr))
936 return 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800937
Avi Kivityebeace82007-01-05 16:36:47 -0800938 return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800939}
940
Avi Kivity6aa8b732006-12-10 02:21:36 -0800941static void nonpaging_free(struct kvm_vcpu *vcpu)
942{
Avi Kivity17ac10a2007-01-05 16:36:40 -0800943 mmu_free_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800944}
945
946static int nonpaging_init_context(struct kvm_vcpu *vcpu)
947{
948 struct kvm_mmu *context = &vcpu->mmu;
949
950 context->new_cr3 = nonpaging_new_cr3;
951 context->page_fault = nonpaging_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800952 context->gva_to_gpa = nonpaging_gva_to_gpa;
953 context->free = nonpaging_free;
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800954 context->root_level = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800955 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17ac10a2007-01-05 16:36:40 -0800956 mmu_alloc_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800957 ASSERT(VALID_PAGE(context->root_hpa));
958 kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
959 return 0;
960}
961
Avi Kivity6aa8b732006-12-10 02:21:36 -0800962static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
963{
Avi Kivity1165f5f2007-04-19 17:27:43 +0300964 ++vcpu->stat.tlb_flush;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800965 kvm_arch_ops->tlb_flush(vcpu);
966}
967
968static void paging_new_cr3(struct kvm_vcpu *vcpu)
969{
Avi Kivity374cbac2007-01-05 16:36:43 -0800970 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800971 mmu_free_roots(vcpu);
Ingo Molnar7f7417d2007-01-05 16:36:57 -0800972 if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
973 kvm_mmu_free_some_pages(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800974 mmu_alloc_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800975 kvm_mmu_flush_tlb(vcpu);
Avi Kivitycea0f0e2007-01-05 16:36:43 -0800976 kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800977}
978
Avi Kivity6aa8b732006-12-10 02:21:36 -0800979static void inject_page_fault(struct kvm_vcpu *vcpu,
980 u64 addr,
981 u32 err_code)
982{
983 kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
984}
985
Avi Kivity6aa8b732006-12-10 02:21:36 -0800986static void paging_free(struct kvm_vcpu *vcpu)
987{
988 nonpaging_free(vcpu);
989}
990
991#define PTTYPE 64
992#include "paging_tmpl.h"
993#undef PTTYPE
994
995#define PTTYPE 32
996#include "paging_tmpl.h"
997#undef PTTYPE
998
Avi Kivity17ac10a2007-01-05 16:36:40 -0800999static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001000{
1001 struct kvm_mmu *context = &vcpu->mmu;
1002
1003 ASSERT(is_pae(vcpu));
1004 context->new_cr3 = paging_new_cr3;
1005 context->page_fault = paging64_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001006 context->gva_to_gpa = paging64_gva_to_gpa;
1007 context->free = paging_free;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001008 context->root_level = level;
1009 context->shadow_root_level = level;
1010 mmu_alloc_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001011 ASSERT(VALID_PAGE(context->root_hpa));
1012 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
1013 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
1014 return 0;
1015}
1016
Avi Kivity17ac10a2007-01-05 16:36:40 -08001017static int paging64_init_context(struct kvm_vcpu *vcpu)
1018{
1019 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1020}
1021
Avi Kivity6aa8b732006-12-10 02:21:36 -08001022static int paging32_init_context(struct kvm_vcpu *vcpu)
1023{
1024 struct kvm_mmu *context = &vcpu->mmu;
1025
1026 context->new_cr3 = paging_new_cr3;
1027 context->page_fault = paging32_page_fault;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001028 context->gva_to_gpa = paging32_gva_to_gpa;
1029 context->free = paging_free;
1030 context->root_level = PT32_ROOT_LEVEL;
1031 context->shadow_root_level = PT32E_ROOT_LEVEL;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001032 mmu_alloc_roots(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001033 ASSERT(VALID_PAGE(context->root_hpa));
1034 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
1035 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
1036 return 0;
1037}
1038
1039static int paging32E_init_context(struct kvm_vcpu *vcpu)
1040{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001041 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001042}
1043
1044static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1045{
1046 ASSERT(vcpu);
1047 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1048
Avi Kivityd3d25b02007-05-30 12:34:53 +03001049 mmu_topup_memory_caches(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001050 if (!is_paging(vcpu))
1051 return nonpaging_init_context(vcpu);
Avi Kivitya9058ec2006-12-29 16:49:37 -08001052 else if (is_long_mode(vcpu))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001053 return paging64_init_context(vcpu);
1054 else if (is_pae(vcpu))
1055 return paging32E_init_context(vcpu);
1056 else
1057 return paging32_init_context(vcpu);
1058}
1059
1060static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1061{
1062 ASSERT(vcpu);
1063 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
1064 vcpu->mmu.free(vcpu);
1065 vcpu->mmu.root_hpa = INVALID_PAGE;
1066 }
1067}
1068
1069int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1070{
Avi Kivity714b93d2007-01-05 16:36:53 -08001071 int r;
1072
Avi Kivity6aa8b732006-12-10 02:21:36 -08001073 destroy_kvm_mmu(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001074 r = init_kvm_mmu(vcpu);
1075 if (r < 0)
1076 goto out;
Avi Kivitye2dec932007-01-05 16:36:54 -08001077 r = mmu_topup_memory_caches(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001078out:
1079 return r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001080}
1081
Avi Kivity09072da2007-05-01 14:16:52 +03001082static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
Avi Kivityac1b7142007-03-08 17:13:32 +02001083 struct kvm_mmu_page *page,
1084 u64 *spte)
1085{
1086 u64 pte;
1087 struct kvm_mmu_page *child;
1088
1089 pte = *spte;
1090 if (is_present_pte(pte)) {
1091 if (page->role.level == PT_PAGE_TABLE_LEVEL)
1092 rmap_remove(vcpu, spte);
1093 else {
1094 child = page_header(pte & PT64_BASE_ADDR_MASK);
1095 mmu_page_remove_parent_pte(vcpu, child, spte);
1096 }
1097 }
1098 *spte = 0;
1099}
1100
Avi Kivity00284252007-05-01 16:53:31 +03001101static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1102 struct kvm_mmu_page *page,
1103 u64 *spte,
1104 const void *new, int bytes)
1105{
1106 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1107 return;
1108
1109 if (page->role.glevels == PT32_ROOT_LEVEL)
1110 paging32_update_pte(vcpu, page, spte, new, bytes);
1111 else
1112 paging64_update_pte(vcpu, page, spte, new, bytes);
1113}
1114
Avi Kivity09072da2007-05-01 14:16:52 +03001115void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1116 const u8 *old, const u8 *new, int bytes)
Avi Kivityda4a00f2007-01-05 16:36:44 -08001117{
Avi Kivity9b7a0322007-01-05 16:36:45 -08001118 gfn_t gfn = gpa >> PAGE_SHIFT;
1119 struct kvm_mmu_page *page;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001120 struct hlist_node *node, *n;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001121 struct hlist_head *bucket;
1122 unsigned index;
1123 u64 *spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001124 unsigned offset = offset_in_page(gpa);
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001125 unsigned pte_size;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001126 unsigned page_offset;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001127 unsigned misaligned;
Avi Kivityfce06572007-05-01 16:44:05 +03001128 unsigned quadrant;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001129 int level;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001130 int flooded = 0;
Avi Kivityac1b7142007-03-08 17:13:32 +02001131 int npte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001132
Avi Kivityda4a00f2007-01-05 16:36:44 -08001133 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
Avi Kivity86a5ba02007-01-05 16:36:50 -08001134 if (gfn == vcpu->last_pt_write_gfn) {
1135 ++vcpu->last_pt_write_count;
1136 if (vcpu->last_pt_write_count >= 3)
1137 flooded = 1;
1138 } else {
1139 vcpu->last_pt_write_gfn = gfn;
1140 vcpu->last_pt_write_count = 1;
1141 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001142 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1143 bucket = &vcpu->kvm->mmu_page_hash[index];
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001144 hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
Avi Kivity9b7a0322007-01-05 16:36:45 -08001145 if (page->gfn != gfn || page->role.metaphysical)
1146 continue;
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001147 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1148 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
Avi Kivitye925c5b2007-04-30 14:47:02 +03001149 misaligned |= bytes < 4;
Avi Kivity86a5ba02007-01-05 16:36:50 -08001150 if (misaligned || flooded) {
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001151 /*
1152 * Misaligned accesses are too much trouble to fix
1153 * up; also, they usually indicate a page is not used
1154 * as a page table.
Avi Kivity86a5ba02007-01-05 16:36:50 -08001155 *
1156 * If we're seeing too many writes to a page,
1157 * it may no longer be a page table, or we may be
1158 * forking, in which case it is better to unmap the
1159 * page.
Avi Kivity0e7bc4b2007-01-05 16:36:48 -08001160 */
1161 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1162 gpa, bytes, page->role.word);
1163 kvm_mmu_zap_page(vcpu, page);
1164 continue;
1165 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001166 page_offset = offset;
1167 level = page->role.level;
Avi Kivityac1b7142007-03-08 17:13:32 +02001168 npte = 1;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001169 if (page->role.glevels == PT32_ROOT_LEVEL) {
Avi Kivityac1b7142007-03-08 17:13:32 +02001170 page_offset <<= 1; /* 32->64 */
1171 /*
1172 * A 32-bit pde maps 4MB while the shadow pdes map
1173 * only 2MB. So we need to double the offset again
1174 * and zap two pdes instead of one.
1175 */
1176 if (level == PT32_ROOT_LEVEL) {
Avi Kivity6b8d0f92007-04-18 11:18:18 +03001177 page_offset &= ~7; /* kill rounding error */
Avi Kivityac1b7142007-03-08 17:13:32 +02001178 page_offset <<= 1;
1179 npte = 2;
1180 }
Avi Kivityfce06572007-05-01 16:44:05 +03001181 quadrant = page_offset >> PAGE_SHIFT;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001182 page_offset &= ~PAGE_MASK;
Avi Kivityfce06572007-05-01 16:44:05 +03001183 if (quadrant != page->role.quadrant)
1184 continue;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001185 }
Avi Kivity47ad8e62007-05-06 15:50:58 +03001186 spte = &page->spt[page_offset / sizeof(*spte)];
Avi Kivityac1b7142007-03-08 17:13:32 +02001187 while (npte--) {
Avi Kivity09072da2007-05-01 14:16:52 +03001188 mmu_pte_write_zap_pte(vcpu, page, spte);
Avi Kivity00284252007-05-01 16:53:31 +03001189 mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
Avi Kivityac1b7142007-03-08 17:13:32 +02001190 ++spte;
Avi Kivity9b7a0322007-01-05 16:36:45 -08001191 }
Avi Kivity9b7a0322007-01-05 16:36:45 -08001192 }
Avi Kivityda4a00f2007-01-05 16:36:44 -08001193}
1194
Avi Kivitya4360362007-01-05 16:36:45 -08001195int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1196{
1197 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
1198
1199 return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
1200}
1201
Avi Kivityebeace82007-01-05 16:36:47 -08001202void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1203{
1204 while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
1205 struct kvm_mmu_page *page;
1206
1207 page = container_of(vcpu->kvm->active_mmu_pages.prev,
1208 struct kvm_mmu_page, link);
1209 kvm_mmu_zap_page(vcpu, page);
1210 }
1211}
1212EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);
1213
Avi Kivity6aa8b732006-12-10 02:21:36 -08001214static void free_mmu_pages(struct kvm_vcpu *vcpu)
1215{
Avi Kivityf51234c2007-01-05 16:36:52 -08001216 struct kvm_mmu_page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001217
Avi Kivityf51234c2007-01-05 16:36:52 -08001218 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1219 page = container_of(vcpu->kvm->active_mmu_pages.next,
1220 struct kvm_mmu_page, link);
1221 kvm_mmu_zap_page(vcpu, page);
1222 }
Avi Kivity17ac10a2007-01-05 16:36:40 -08001223 free_page((unsigned long)vcpu->mmu.pae_root);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001224}
1225
1226static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1227{
Avi Kivity17ac10a2007-01-05 16:36:40 -08001228 struct page *page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001229 int i;
1230
1231 ASSERT(vcpu);
1232
Avi Kivityd3d25b02007-05-30 12:34:53 +03001233 vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
Avi Kivity17ac10a2007-01-05 16:36:40 -08001234
1235 /*
1236 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1237 * Therefore we need to allocate shadow page tables in the first
1238 * 4GB of memory, which happens to fit the DMA32 zone.
1239 */
1240 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1241 if (!page)
1242 goto error_1;
1243 vcpu->mmu.pae_root = page_address(page);
1244 for (i = 0; i < 4; ++i)
1245 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1246
Avi Kivity6aa8b732006-12-10 02:21:36 -08001247 return 0;
1248
1249error_1:
1250 free_mmu_pages(vcpu);
1251 return -ENOMEM;
1252}
1253
Ingo Molnar8018c272006-12-29 16:50:01 -08001254int kvm_mmu_create(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001255{
Avi Kivity6aa8b732006-12-10 02:21:36 -08001256 ASSERT(vcpu);
1257 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
Avi Kivity6aa8b732006-12-10 02:21:36 -08001258
Ingo Molnar8018c272006-12-29 16:50:01 -08001259 return alloc_mmu_pages(vcpu);
1260}
Avi Kivity6aa8b732006-12-10 02:21:36 -08001261
Ingo Molnar8018c272006-12-29 16:50:01 -08001262int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1263{
1264 ASSERT(vcpu);
1265 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
Avi Kivity2c264952006-12-22 01:05:28 -08001266
Ingo Molnar8018c272006-12-29 16:50:01 -08001267 return init_kvm_mmu(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001268}
1269
1270void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1271{
1272 ASSERT(vcpu);
1273
1274 destroy_kvm_mmu(vcpu);
1275 free_mmu_pages(vcpu);
Avi Kivity714b93d2007-01-05 16:36:53 -08001276 mmu_free_memory_caches(vcpu);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001277}
1278
Avi Kivity714b93d2007-01-05 16:36:53 -08001279void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001280{
Avi Kivity714b93d2007-01-05 16:36:53 -08001281 struct kvm *kvm = vcpu->kvm;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001282 struct kvm_mmu_page *page;
1283
1284 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1285 int i;
1286 u64 *pt;
1287
1288 if (!test_bit(slot, &page->slot_bitmap))
1289 continue;
1290
Avi Kivity47ad8e62007-05-06 15:50:58 +03001291 pt = page->spt;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001292 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1293 /* avoid RMW */
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001294 if (pt[i] & PT_WRITABLE_MASK) {
Avi Kivity714b93d2007-01-05 16:36:53 -08001295 rmap_remove(vcpu, &pt[i]);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001296 pt[i] &= ~PT_WRITABLE_MASK;
Avi Kivitycd4a4e52007-01-05 16:36:38 -08001297 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001298 }
1299}
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001300
Dor Laore0fa8262007-03-30 13:06:33 +03001301void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
1302{
1303 destroy_kvm_mmu(vcpu);
1304
1305 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1306 struct kvm_mmu_page *page;
1307
1308 page = container_of(vcpu->kvm->active_mmu_pages.next,
1309 struct kvm_mmu_page, link);
1310 kvm_mmu_zap_page(vcpu, page);
1311 }
1312
1313 mmu_free_memory_caches(vcpu);
1314 kvm_arch_ops->tlb_flush(vcpu);
1315 init_kvm_mmu(vcpu);
1316}
1317
Avi Kivityb5a33a72007-04-15 16:31:09 +03001318void kvm_mmu_module_exit(void)
1319{
1320 if (pte_chain_cache)
1321 kmem_cache_destroy(pte_chain_cache);
1322 if (rmap_desc_cache)
1323 kmem_cache_destroy(rmap_desc_cache);
Avi Kivityd3d25b02007-05-30 12:34:53 +03001324 if (mmu_page_cache)
1325 kmem_cache_destroy(mmu_page_cache);
1326 if (mmu_page_header_cache)
1327 kmem_cache_destroy(mmu_page_header_cache);
Avi Kivityb5a33a72007-04-15 16:31:09 +03001328}
1329
1330int kvm_mmu_module_init(void)
1331{
1332 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
1333 sizeof(struct kvm_pte_chain),
1334 0, 0, NULL, NULL);
1335 if (!pte_chain_cache)
1336 goto nomem;
1337 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
1338 sizeof(struct kvm_rmap_desc),
1339 0, 0, NULL, NULL);
1340 if (!rmap_desc_cache)
1341 goto nomem;
1342
Avi Kivityd3d25b02007-05-30 12:34:53 +03001343 mmu_page_cache = kmem_cache_create("kvm_mmu_page",
1344 PAGE_SIZE,
1345 PAGE_SIZE, 0, NULL, NULL);
1346 if (!mmu_page_cache)
1347 goto nomem;
1348
1349 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
1350 sizeof(struct kvm_mmu_page),
1351 0, 0, NULL, NULL);
1352 if (!mmu_page_header_cache)
1353 goto nomem;
1354
Avi Kivityb5a33a72007-04-15 16:31:09 +03001355 return 0;
1356
1357nomem:
1358 kvm_mmu_module_exit();
1359 return -ENOMEM;
1360}
1361
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001362#ifdef AUDIT
1363
1364static const char *audit_msg;
1365
1366static gva_t canonicalize(gva_t gva)
1367{
1368#ifdef CONFIG_X86_64
1369 gva = (long long)(gva << 16) >> 16;
1370#endif
1371 return gva;
1372}
1373
1374static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1375 gva_t va, int level)
1376{
1377 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1378 int i;
1379 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1380
1381 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1382 u64 ent = pt[i];
1383
Adrian Bunk28076962007-04-28 21:20:48 +02001384 if (!(ent & PT_PRESENT_MASK))
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001385 continue;
1386
1387 va = canonicalize(va);
1388 if (level > 1)
1389 audit_mappings_page(vcpu, ent, va, level - 1);
1390 else {
1391 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
1392 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
1393
1394 if ((ent & PT_PRESENT_MASK)
1395 && (ent & PT64_BASE_ADDR_MASK) != hpa)
1396 printk(KERN_ERR "audit error: (%s) levels %d"
1397 " gva %lx gpa %llx hpa %llx ent %llx\n",
1398 audit_msg, vcpu->mmu.root_level,
1399 va, gpa, hpa, ent);
1400 }
1401 }
1402}
1403
1404static void audit_mappings(struct kvm_vcpu *vcpu)
1405{
Avi Kivity1ea252a2007-03-08 11:48:09 +02001406 unsigned i;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001407
1408 if (vcpu->mmu.root_level == 4)
1409 audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
1410 else
1411 for (i = 0; i < 4; ++i)
1412 if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
1413 audit_mappings_page(vcpu,
1414 vcpu->mmu.pae_root[i],
1415 i << 30,
1416 2);
1417}
1418
1419static int count_rmaps(struct kvm_vcpu *vcpu)
1420{
1421 int nmaps = 0;
1422 int i, j, k;
1423
1424 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1425 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1426 struct kvm_rmap_desc *d;
1427
1428 for (j = 0; j < m->npages; ++j) {
1429 struct page *page = m->phys_mem[j];
1430
1431 if (!page->private)
1432 continue;
1433 if (!(page->private & 1)) {
1434 ++nmaps;
1435 continue;
1436 }
1437 d = (struct kvm_rmap_desc *)(page->private & ~1ul);
1438 while (d) {
1439 for (k = 0; k < RMAP_EXT; ++k)
1440 if (d->shadow_ptes[k])
1441 ++nmaps;
1442 else
1443 break;
1444 d = d->more;
1445 }
1446 }
1447 }
1448 return nmaps;
1449}
1450
1451static int count_writable_mappings(struct kvm_vcpu *vcpu)
1452{
1453 int nmaps = 0;
1454 struct kvm_mmu_page *page;
1455 int i;
1456
1457 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
Avi Kivity47ad8e62007-05-06 15:50:58 +03001458 u64 *pt = page->spt;
Avi Kivity37a7d8b2007-01-05 16:36:56 -08001459
1460 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1461 continue;
1462
1463 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1464 u64 ent = pt[i];
1465
1466 if (!(ent & PT_PRESENT_MASK))
1467 continue;
1468 if (!(ent & PT_WRITABLE_MASK))
1469 continue;
1470 ++nmaps;
1471 }
1472 }
1473 return nmaps;
1474}
1475
1476static void audit_rmap(struct kvm_vcpu *vcpu)
1477{
1478 int n_rmap = count_rmaps(vcpu);
1479 int n_actual = count_writable_mappings(vcpu);
1480
1481 if (n_rmap != n_actual)
1482 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1483 __FUNCTION__, audit_msg, n_rmap, n_actual);
1484}
1485
1486static void audit_write_protection(struct kvm_vcpu *vcpu)
1487{
1488 struct kvm_mmu_page *page;
1489
1490 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1491 hfn_t hfn;
1492 struct page *pg;
1493
1494 if (page->role.metaphysical)
1495 continue;
1496
1497 hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
1498 >> PAGE_SHIFT;
1499 pg = pfn_to_page(hfn);
1500 if (pg->private)
1501 printk(KERN_ERR "%s: (%s) shadow page has writable"
1502 " mappings: gfn %lx role %x\n",
1503 __FUNCTION__, audit_msg, page->gfn,
1504 page->role.word);
1505 }
1506}
1507
1508static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1509{
1510 int olddbg = dbg;
1511
1512 dbg = 0;
1513 audit_msg = msg;
1514 audit_rmap(vcpu);
1515 audit_write_protection(vcpu);
1516 audit_mappings(vcpu);
1517 dbg = olddbg;
1518}
1519
1520#endif