blob: 04b22128a47421cac65a6dd73480ee56c87ded1d [file] [log] [blame]
Rusty Russell2e04ef72009-07-30 16:03:45 -06001/*P:700
2 * The pagetable code, on the other hand, still shows the scars of
Rusty Russellf938d2c2007-07-26 10:41:02 -07003 * previous encounters. It's functional, and as neat as it can be in the
4 * circumstances, but be wary, for these things are subtle and break easily.
5 * The Guest provides a virtual to physical mapping, but we can neither trust
Rusty Russella6bd8e12008-03-28 11:05:53 -05006 * it nor use it: we verify and convert it here then point the CPU to the
Rusty Russell2e04ef72009-07-30 16:03:45 -06007 * converted Guest pages when running the Guest.
8:*/
Rusty Russellf938d2c2007-07-26 10:41:02 -07009
10/* Copyright (C) Rusty Russell IBM Corporation 2006.
Rusty Russelld7e28ff2007-07-19 01:49:23 -070011 * GPL v2 and any later version */
12#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/gfp.h>
Rusty Russelld7e28ff2007-07-19 01:49:23 -070014#include <linux/types.h>
15#include <linux/spinlock.h>
16#include <linux/random.h>
17#include <linux/percpu.h>
18#include <asm/tlbflush.h>
Rusty Russell47436aa2007-10-22 11:03:36 +100019#include <asm/uaccess.h>
Matias Zabaljauregui58a24562008-09-29 01:40:07 -030020#include <asm/bootparam.h>
Rusty Russelld7e28ff2007-07-19 01:49:23 -070021#include "lg.h"
22
Rusty Russell2e04ef72009-07-30 16:03:45 -060023/*M:008
24 * We hold reference to pages, which prevents them from being swapped.
Rusty Russellf56a3842007-07-26 10:41:05 -070025 * It'd be nice to have a callback in the "struct mm_struct" when Linux wants
26 * to swap out. If we had this, and a shrinker callback to trim PTE pages, we
Rusty Russell2e04ef72009-07-30 16:03:45 -060027 * could probably consider launching Guests as non-root.
28:*/
Rusty Russellf56a3842007-07-26 10:41:05 -070029
Rusty Russellbff672e2007-07-26 10:41:04 -070030/*H:300
31 * The Page Table Code
32 *
Rusty Russella91d74a2009-07-30 16:03:45 -060033 * We use two-level page tables for the Guest, or three-level with PAE. If
34 * you're not entirely comfortable with virtual addresses, physical addresses
35 * and page tables then I recommend you review arch/x86/lguest/boot.c's "Page
36 * Table Handling" (with diagrams!).
Rusty Russellbff672e2007-07-26 10:41:04 -070037 *
38 * The Guest keeps page tables, but we maintain the actual ones here: these are
39 * called "shadow" page tables. Which is a very Guest-centric name: these are
40 * the real page tables the CPU uses, although we keep them up to date to
41 * reflect the Guest's. (See what I mean about weird naming? Since when do
42 * shadows reflect anything?)
43 *
44 * Anyway, this is the most complicated part of the Host code. There are seven
45 * parts to this:
Rusty Russelle1e72962007-10-25 15:02:50 +100046 * (i) Looking up a page table entry when the Guest faults,
47 * (ii) Making sure the Guest stack is mapped,
48 * (iii) Setting up a page table entry when the Guest tells us one has changed,
Rusty Russellbff672e2007-07-26 10:41:04 -070049 * (iv) Switching page tables,
Rusty Russelle1e72962007-10-25 15:02:50 +100050 * (v) Flushing (throwing away) page tables,
Rusty Russellbff672e2007-07-26 10:41:04 -070051 * (vi) Mapping the Switcher when the Guest is about to run,
52 * (vii) Setting up the page tables initially.
Rusty Russell2e04ef72009-07-30 16:03:45 -060053:*/
Rusty Russellbff672e2007-07-26 10:41:04 -070054
Rusty Russell2e04ef72009-07-30 16:03:45 -060055/*
Rusty Russella91d74a2009-07-30 16:03:45 -060056 * The Switcher uses the complete top PTE page. That's 1024 PTE entries (4MB)
57 * or 512 PTE entries with PAE (2MB).
Rusty Russell2e04ef72009-07-30 16:03:45 -060058 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100059#define SWITCHER_PGD_INDEX (PTRS_PER_PGD - 1)
Rusty Russelld7e28ff2007-07-19 01:49:23 -070060
Rusty Russell2e04ef72009-07-30 16:03:45 -060061/*
62 * For PAE we need the PMD index as well. We use the last 2MB, so we
63 * will need the last pmd entry of the last pmd page.
64 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060065#ifdef CONFIG_X86_PAE
66#define SWITCHER_PMD_INDEX (PTRS_PER_PMD - 1)
67#define RESERVE_MEM 2U
68#define CHECK_GPGD_MASK _PAGE_PRESENT
69#else
70#define RESERVE_MEM 4U
71#define CHECK_GPGD_MASK _PAGE_TABLE
72#endif
73
Rusty Russell2e04ef72009-07-30 16:03:45 -060074/*
75 * We actually need a separate PTE page for each CPU. Remember that after the
Rusty Russellbff672e2007-07-26 10:41:04 -070076 * Switcher code itself comes two pages for each CPU, and we don't want this
Rusty Russell2e04ef72009-07-30 16:03:45 -060077 * CPU's guest to see the pages of any other CPU.
78 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100079static DEFINE_PER_CPU(pte_t *, switcher_pte_pages);
Rusty Russelld7e28ff2007-07-19 01:49:23 -070080#define switcher_pte_page(cpu) per_cpu(switcher_pte_pages, cpu)
81
Rusty Russell2e04ef72009-07-30 16:03:45 -060082/*H:320
83 * The page table code is curly enough to need helper functions to keep it
Rusty Russella91d74a2009-07-30 16:03:45 -060084 * clear and clean. The kernel itself provides many of them; one advantage
85 * of insisting that the Guest and Host use the same CONFIG_PAE setting.
Rusty Russellbff672e2007-07-26 10:41:04 -070086 *
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100087 * There are two functions which return pointers to the shadow (aka "real")
Rusty Russellbff672e2007-07-26 10:41:04 -070088 * page tables.
89 *
90 * spgd_addr() takes the virtual address and returns a pointer to the top-level
Rusty Russelle1e72962007-10-25 15:02:50 +100091 * page directory entry (PGD) for that address. Since we keep track of several
92 * page tables, the "i" argument tells us which one we're interested in (it's
Rusty Russell2e04ef72009-07-30 16:03:45 -060093 * usually the current one).
94 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -020095static pgd_t *spgd_addr(struct lg_cpu *cpu, u32 i, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -070096{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100097 unsigned int index = pgd_index(vaddr);
Rusty Russelld7e28ff2007-07-19 01:49:23 -070098
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060099#ifndef CONFIG_X86_PAE
Rusty Russellbff672e2007-07-26 10:41:04 -0700100 /* We kill any Guest trying to touch the Switcher addresses. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700101 if (index >= SWITCHER_PGD_INDEX) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200102 kill_guest(cpu, "attempt to access switcher pages");
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700103 index = 0;
104 }
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600105#endif
Rusty Russellbff672e2007-07-26 10:41:04 -0700106 /* Return a pointer index'th pgd entry for the i'th page table. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200107 return &cpu->lg->pgdirs[i].pgdir[index];
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700108}
109
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600110#ifdef CONFIG_X86_PAE
Rusty Russell2e04ef72009-07-30 16:03:45 -0600111/*
112 * This routine then takes the PGD entry given above, which contains the
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600113 * address of the PMD page. It then returns a pointer to the PMD entry for the
Rusty Russell2e04ef72009-07-30 16:03:45 -0600114 * given address.
115 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600116static pmd_t *spmd_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
117{
118 unsigned int index = pmd_index(vaddr);
119 pmd_t *page;
120
121 /* We kill any Guest trying to touch the Switcher addresses. */
122 if (pgd_index(vaddr) == SWITCHER_PGD_INDEX &&
123 index >= SWITCHER_PMD_INDEX) {
124 kill_guest(cpu, "attempt to access switcher pages");
125 index = 0;
126 }
127
128 /* You should never call this if the PGD entry wasn't valid */
129 BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
130 page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
131
132 return &page[index];
133}
134#endif
135
Rusty Russell2e04ef72009-07-30 16:03:45 -0600136/*
137 * This routine then takes the page directory entry returned above, which
Rusty Russelle1e72962007-10-25 15:02:50 +1000138 * contains the address of the page table entry (PTE) page. It then returns a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600139 * pointer to the PTE entry for the given address.
140 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600141static pte_t *spte_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700142{
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600143#ifdef CONFIG_X86_PAE
144 pmd_t *pmd = spmd_addr(cpu, spgd, vaddr);
145 pte_t *page = __va(pmd_pfn(*pmd) << PAGE_SHIFT);
146
147 /* You should never call this if the PMD entry wasn't valid */
148 BUG_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT));
149#else
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000150 pte_t *page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
Rusty Russellbff672e2007-07-26 10:41:04 -0700151 /* You should never call this if the PGD entry wasn't valid */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000152 BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600153#endif
154
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600155 return &page[pte_index(vaddr)];
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700156}
157
Rusty Russell2e04ef72009-07-30 16:03:45 -0600158/*
Rusty Russella91d74a2009-07-30 16:03:45 -0600159 * These functions are just like the above two, except they access the Guest
Rusty Russell2e04ef72009-07-30 16:03:45 -0600160 * page tables. Hence they return a Guest address.
161 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200162static unsigned long gpgd_addr(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700163{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000164 unsigned int index = vaddr >> (PGDIR_SHIFT);
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200165 return cpu->lg->pgdirs[cpu->cpu_pgd].gpgdir + index * sizeof(pgd_t);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700166}
167
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600168#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -0600169/* Follow the PGD to the PMD. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600170static unsigned long gpmd_addr(pgd_t gpgd, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700171{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000172 unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
173 BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600174 return gpage + pmd_index(vaddr) * sizeof(pmd_t);
175}
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600176
Rusty Russella91d74a2009-07-30 16:03:45 -0600177/* Follow the PMD to the PTE. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600178static unsigned long gpte_addr(struct lg_cpu *cpu,
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600179 pmd_t gpmd, unsigned long vaddr)
180{
181 unsigned long gpage = pmd_pfn(gpmd) << PAGE_SHIFT;
182
183 BUG_ON(!(pmd_flags(gpmd) & _PAGE_PRESENT));
184 return gpage + pte_index(vaddr) * sizeof(pte_t);
185}
186#else
Rusty Russella91d74a2009-07-30 16:03:45 -0600187/* Follow the PGD to the PTE (no mid-level for !PAE). */
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600188static unsigned long gpte_addr(struct lg_cpu *cpu,
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600189 pgd_t gpgd, unsigned long vaddr)
190{
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600191 unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600192
193 BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600194 return gpage + pte_index(vaddr) * sizeof(pte_t);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700195}
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600196#endif
Rusty Russella6bd8e12008-03-28 11:05:53 -0500197/*:*/
198
Rusty Russell2e04ef72009-07-30 16:03:45 -0600199/*M:014
200 * get_pfn is slow: we could probably try to grab batches of pages here as
201 * an optimization (ie. pre-faulting).
202:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700203
Rusty Russell2e04ef72009-07-30 16:03:45 -0600204/*H:350
205 * This routine takes a page number given by the Guest and converts it to
Rusty Russellbff672e2007-07-26 10:41:04 -0700206 * an actual, physical page number. It can fail for several reasons: the
207 * virtual address might not be mapped by the Launcher, the write flag is set
208 * and the page is read-only, or the write flag was set and the page was
209 * shared so had to be copied, but we ran out of memory.
210 *
Rusty Russella6bd8e12008-03-28 11:05:53 -0500211 * This holds a reference to the page, so release_pte() is careful to put that
Rusty Russell2e04ef72009-07-30 16:03:45 -0600212 * back.
213 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700214static unsigned long get_pfn(unsigned long virtpfn, int write)
215{
216 struct page *page;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700217
Rusty Russell71a3f4e2008-08-12 17:52:53 -0500218 /* gup me one page at this address please! */
219 if (get_user_pages_fast(virtpfn << PAGE_SHIFT, 1, write, &page) == 1)
220 return page_to_pfn(page);
221
222 /* This value indicates failure. */
223 return -1UL;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700224}
225
Rusty Russell2e04ef72009-07-30 16:03:45 -0600226/*H:340
227 * Converting a Guest page table entry to a shadow (ie. real) page table
Rusty Russellbff672e2007-07-26 10:41:04 -0700228 * entry can be a little tricky. The flags are (almost) the same, but the
229 * Guest PTE contains a virtual page number: the CPU needs the real page
Rusty Russell2e04ef72009-07-30 16:03:45 -0600230 * number.
231 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200232static pte_t gpte_to_spte(struct lg_cpu *cpu, pte_t gpte, int write)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700233{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000234 unsigned long pfn, base, flags;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700235
Rusty Russell2e04ef72009-07-30 16:03:45 -0600236 /*
237 * The Guest sets the global flag, because it thinks that it is using
Rusty Russellbff672e2007-07-26 10:41:04 -0700238 * PGE. We only told it to use PGE so it would tell us whether it was
239 * flushing a kernel mapping or a userspace mapping. We don't actually
Rusty Russell2e04ef72009-07-30 16:03:45 -0600240 * use the global bit, so throw it away.
241 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000242 flags = (pte_flags(gpte) & ~_PAGE_GLOBAL);
Rusty Russellbff672e2007-07-26 10:41:04 -0700243
Rusty Russell3c6b5bf2007-10-22 11:03:26 +1000244 /* The Guest's pages are offset inside the Launcher. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200245 base = (unsigned long)cpu->lg->mem_base / PAGE_SIZE;
Rusty Russell3c6b5bf2007-10-22 11:03:26 +1000246
Rusty Russell2e04ef72009-07-30 16:03:45 -0600247 /*
248 * We need a temporary "unsigned long" variable to hold the answer from
Rusty Russellbff672e2007-07-26 10:41:04 -0700249 * get_pfn(), because it returns 0xFFFFFFFF on failure, which wouldn't
250 * fit in spte.pfn. get_pfn() finds the real physical number of the
Rusty Russell2e04ef72009-07-30 16:03:45 -0600251 * page, given the virtual number.
252 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000253 pfn = get_pfn(base + pte_pfn(gpte), write);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700254 if (pfn == -1UL) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200255 kill_guest(cpu, "failed to get page %lu", pte_pfn(gpte));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600256 /*
257 * When we destroy the Guest, we'll go through the shadow page
Rusty Russellbff672e2007-07-26 10:41:04 -0700258 * tables and release_pte() them. Make sure we don't think
Rusty Russell2e04ef72009-07-30 16:03:45 -0600259 * this one is valid!
260 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000261 flags = 0;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700262 }
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000263 /* Now we assemble our shadow PTE from the page number and flags. */
264 return pfn_pte(pfn, __pgprot(flags));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700265}
266
Rusty Russellbff672e2007-07-26 10:41:04 -0700267/*H:460 And to complete the chain, release_pte() looks like this: */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000268static void release_pte(pte_t pte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700269{
Rusty Russell2e04ef72009-07-30 16:03:45 -0600270 /*
271 * Remember that get_user_pages_fast() took a reference to the page, in
272 * get_pfn()? We have to put it back now.
273 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000274 if (pte_flags(pte) & _PAGE_PRESENT)
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600275 put_page(pte_page(pte));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700276}
Rusty Russellbff672e2007-07-26 10:41:04 -0700277/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700278
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200279static void check_gpte(struct lg_cpu *cpu, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700280{
Ahmed S. Darwish31f4b462008-02-09 23:24:09 +0100281 if ((pte_flags(gpte) & _PAGE_PSE) ||
282 pte_pfn(gpte) >= cpu->lg->pfn_limit)
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200283 kill_guest(cpu, "bad page table entry");
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700284}
285
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200286static void check_gpgd(struct lg_cpu *cpu, pgd_t gpgd)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700287{
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600288 if ((pgd_flags(gpgd) & ~CHECK_GPGD_MASK) ||
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200289 (pgd_pfn(gpgd) >= cpu->lg->pfn_limit))
290 kill_guest(cpu, "bad page directory entry");
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700291}
292
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600293#ifdef CONFIG_X86_PAE
294static void check_gpmd(struct lg_cpu *cpu, pmd_t gpmd)
295{
296 if ((pmd_flags(gpmd) & ~_PAGE_TABLE) ||
297 (pmd_pfn(gpmd) >= cpu->lg->pfn_limit))
298 kill_guest(cpu, "bad page middle directory entry");
299}
300#endif
301
Rusty Russellbff672e2007-07-26 10:41:04 -0700302/*H:330
Rusty Russelle1e72962007-10-25 15:02:50 +1000303 * (i) Looking up a page table entry when the Guest faults.
Rusty Russellbff672e2007-07-26 10:41:04 -0700304 *
305 * We saw this call in run_guest(): when we see a page fault in the Guest, we
306 * come here. That's because we only set up the shadow page tables lazily as
307 * they're needed, so we get page faults all the time and quietly fix them up
308 * and return to the Guest without it knowing.
309 *
310 * If we fixed up the fault (ie. we mapped the address), this routine returns
Rusty Russell2e04ef72009-07-30 16:03:45 -0600311 * true. Otherwise, it was a real fault and we need to tell the Guest.
312 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300313bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700314{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000315 pgd_t gpgd;
316 pgd_t *spgd;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700317 unsigned long gpte_ptr;
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000318 pte_t gpte;
319 pte_t *spte;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700320
Rusty Russella91d74a2009-07-30 16:03:45 -0600321 /* Mid level for PAE. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600322#ifdef CONFIG_X86_PAE
323 pmd_t *spmd;
324 pmd_t gpmd;
325#endif
326
Rusty Russellbff672e2007-07-26 10:41:04 -0700327 /* First step: get the top-level Guest page table entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200328 gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
Rusty Russellbff672e2007-07-26 10:41:04 -0700329 /* Toplevel not present? We can't map it in. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000330 if (!(pgd_flags(gpgd) & _PAGE_PRESENT))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300331 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700332
Rusty Russellbff672e2007-07-26 10:41:04 -0700333 /* Now look at the matching shadow entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200334 spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000335 if (!(pgd_flags(*spgd) & _PAGE_PRESENT)) {
Rusty Russellbff672e2007-07-26 10:41:04 -0700336 /* No shadow entry: allocate a new shadow PTE page. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700337 unsigned long ptepage = get_zeroed_page(GFP_KERNEL);
Rusty Russell2e04ef72009-07-30 16:03:45 -0600338 /*
339 * This is not really the Guest's fault, but killing it is
340 * simple for this corner case.
341 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700342 if (!ptepage) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200343 kill_guest(cpu, "out of memory allocating pte page");
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300344 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700345 }
Rusty Russellbff672e2007-07-26 10:41:04 -0700346 /* We check that the Guest pgd is OK. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200347 check_gpgd(cpu, gpgd);
Rusty Russell2e04ef72009-07-30 16:03:45 -0600348 /*
349 * And we copy the flags to the shadow PGD entry. The page
350 * number in the shadow PGD is the page we just allocated.
351 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600352 set_pgd(spgd, __pgd(__pa(ptepage) | pgd_flags(gpgd)));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700353 }
354
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600355#ifdef CONFIG_X86_PAE
356 gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
Rusty Russell2e04ef72009-07-30 16:03:45 -0600357 /* Middle level not present? We can't map it in. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600358 if (!(pmd_flags(gpmd) & _PAGE_PRESENT))
359 return false;
360
361 /* Now look at the matching shadow entry. */
362 spmd = spmd_addr(cpu, *spgd, vaddr);
363
364 if (!(pmd_flags(*spmd) & _PAGE_PRESENT)) {
365 /* No shadow entry: allocate a new shadow PTE page. */
366 unsigned long ptepage = get_zeroed_page(GFP_KERNEL);
367
Rusty Russell2e04ef72009-07-30 16:03:45 -0600368 /*
369 * This is not really the Guest's fault, but killing it is
370 * simple for this corner case.
371 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600372 if (!ptepage) {
373 kill_guest(cpu, "out of memory allocating pte page");
374 return false;
375 }
376
377 /* We check that the Guest pmd is OK. */
378 check_gpmd(cpu, gpmd);
379
Rusty Russell2e04ef72009-07-30 16:03:45 -0600380 /*
381 * And we copy the flags to the shadow PMD entry. The page
382 * number in the shadow PMD is the page we just allocated.
383 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600384 set_pmd(spmd, __pmd(__pa(ptepage) | pmd_flags(gpmd)));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600385 }
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600386
Rusty Russell2e04ef72009-07-30 16:03:45 -0600387 /*
388 * OK, now we look at the lower level in the Guest page table: keep its
389 * address, because we might update it later.
390 */
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600391 gpte_ptr = gpte_addr(cpu, gpmd, vaddr);
392#else
Rusty Russell2e04ef72009-07-30 16:03:45 -0600393 /*
394 * OK, now we look at the lower level in the Guest page table: keep its
395 * address, because we might update it later.
396 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600397 gpte_ptr = gpte_addr(cpu, gpgd, vaddr);
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600398#endif
Rusty Russella91d74a2009-07-30 16:03:45 -0600399
400 /* Read the actual PTE value. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200401 gpte = lgread(cpu, gpte_ptr, pte_t);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700402
Rusty Russellbff672e2007-07-26 10:41:04 -0700403 /* If this page isn't in the Guest page tables, we can't page it in. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000404 if (!(pte_flags(gpte) & _PAGE_PRESENT))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300405 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700406
Rusty Russell2e04ef72009-07-30 16:03:45 -0600407 /*
408 * Check they're not trying to write to a page the Guest wants
409 * read-only (bit 2 of errcode == write).
410 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000411 if ((errcode & 2) && !(pte_flags(gpte) & _PAGE_RW))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300412 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700413
Rusty Russelle1e72962007-10-25 15:02:50 +1000414 /* User access to a kernel-only page? (bit 3 == user access) */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000415 if ((errcode & 4) && !(pte_flags(gpte) & _PAGE_USER))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300416 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700417
Rusty Russell2e04ef72009-07-30 16:03:45 -0600418 /*
419 * Check that the Guest PTE flags are OK, and the page number is below
420 * the pfn_limit (ie. not mapping the Launcher binary).
421 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200422 check_gpte(cpu, gpte);
Rusty Russelle1e72962007-10-25 15:02:50 +1000423
Rusty Russellbff672e2007-07-26 10:41:04 -0700424 /* Add the _PAGE_ACCESSED and (for a write) _PAGE_DIRTY flag */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000425 gpte = pte_mkyoung(gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700426 if (errcode & 2)
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000427 gpte = pte_mkdirty(gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700428
Rusty Russellbff672e2007-07-26 10:41:04 -0700429 /* Get the pointer to the shadow PTE entry we're going to set. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600430 spte = spte_addr(cpu, *spgd, vaddr);
Rusty Russell2e04ef72009-07-30 16:03:45 -0600431
432 /*
433 * If there was a valid shadow PTE entry here before, we release it.
434 * This can happen with a write to a previously read-only entry.
435 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700436 release_pte(*spte);
437
Rusty Russell2e04ef72009-07-30 16:03:45 -0600438 /*
439 * If this is a write, we insist that the Guest page is writable (the
440 * final arg to gpte_to_spte()).
441 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000442 if (pte_dirty(gpte))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200443 *spte = gpte_to_spte(cpu, gpte, 1);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000444 else
Rusty Russell2e04ef72009-07-30 16:03:45 -0600445 /*
446 * If this is a read, don't set the "writable" bit in the page
Rusty Russellbff672e2007-07-26 10:41:04 -0700447 * table entry, even if the Guest says it's writable. That way
Rusty Russelle1e72962007-10-25 15:02:50 +1000448 * we will come back here when a write does actually occur, so
Rusty Russell2e04ef72009-07-30 16:03:45 -0600449 * we can update the Guest's _PAGE_DIRTY flag.
450 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600451 set_pte(spte, gpte_to_spte(cpu, pte_wrprotect(gpte), 0));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700452
Rusty Russell2e04ef72009-07-30 16:03:45 -0600453 /*
454 * Finally, we write the Guest PTE entry back: we've set the
455 * _PAGE_ACCESSED and maybe the _PAGE_DIRTY flags.
456 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200457 lgwrite(cpu, gpte_ptr, pte_t, gpte);
Rusty Russellbff672e2007-07-26 10:41:04 -0700458
Rusty Russell2e04ef72009-07-30 16:03:45 -0600459 /*
460 * The fault is fixed, the page table is populated, the mapping
Rusty Russelle1e72962007-10-25 15:02:50 +1000461 * manipulated, the result returned and the code complete. A small
462 * delay and a trace of alliteration are the only indications the Guest
Rusty Russell2e04ef72009-07-30 16:03:45 -0600463 * has that a page fault occurred at all.
464 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300465 return true;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700466}
467
Rusty Russelle1e72962007-10-25 15:02:50 +1000468/*H:360
469 * (ii) Making sure the Guest stack is mapped.
Rusty Russellbff672e2007-07-26 10:41:04 -0700470 *
Rusty Russelle1e72962007-10-25 15:02:50 +1000471 * Remember that direct traps into the Guest need a mapped Guest kernel stack.
472 * pin_stack_pages() calls us here: we could simply call demand_page(), but as
473 * we've seen that logic is quite long, and usually the stack pages are already
474 * mapped, so it's overkill.
Rusty Russellbff672e2007-07-26 10:41:04 -0700475 *
476 * This is a quick version which answers the question: is this virtual address
Rusty Russell2e04ef72009-07-30 16:03:45 -0600477 * mapped by the shadow page tables, and is it writable?
478 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300479static bool page_writable(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700480{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000481 pgd_t *spgd;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700482 unsigned long flags;
483
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600484#ifdef CONFIG_X86_PAE
485 pmd_t *spmd;
486#endif
Rusty Russelle1e72962007-10-25 15:02:50 +1000487 /* Look at the current top level entry: is it present? */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200488 spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000489 if (!(pgd_flags(*spgd) & _PAGE_PRESENT))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300490 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700491
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600492#ifdef CONFIG_X86_PAE
493 spmd = spmd_addr(cpu, *spgd, vaddr);
494 if (!(pmd_flags(*spmd) & _PAGE_PRESENT))
495 return false;
496#endif
497
Rusty Russell2e04ef72009-07-30 16:03:45 -0600498 /*
499 * Check the flags on the pte entry itself: it must be present and
500 * writable.
501 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600502 flags = pte_flags(*(spte_addr(cpu, *spgd, vaddr)));
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000503
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700504 return (flags & (_PAGE_PRESENT|_PAGE_RW)) == (_PAGE_PRESENT|_PAGE_RW);
505}
506
Rusty Russell2e04ef72009-07-30 16:03:45 -0600507/*
508 * So, when pin_stack_pages() asks us to pin a page, we check if it's already
Rusty Russellbff672e2007-07-26 10:41:04 -0700509 * in the page tables, and if not, we call demand_page() with error code 2
Rusty Russell2e04ef72009-07-30 16:03:45 -0600510 * (meaning "write").
511 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200512void pin_page(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700513{
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200514 if (!page_writable(cpu, vaddr) && !demand_page(cpu, vaddr, 2))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200515 kill_guest(cpu, "bad stack page %#lx", vaddr);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700516}
Rusty Russella91d74a2009-07-30 16:03:45 -0600517/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700518
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600519#ifdef CONFIG_X86_PAE
520static void release_pmd(pmd_t *spmd)
521{
522 /* If the entry's not present, there's nothing to release. */
523 if (pmd_flags(*spmd) & _PAGE_PRESENT) {
524 unsigned int i;
525 pte_t *ptepage = __va(pmd_pfn(*spmd) << PAGE_SHIFT);
526 /* For each entry in the page, we might need to release it. */
527 for (i = 0; i < PTRS_PER_PTE; i++)
528 release_pte(ptepage[i]);
529 /* Now we can free the page of PTEs */
530 free_page((long)ptepage);
531 /* And zero out the PMD entry so we never release it twice. */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600532 set_pmd(spmd, __pmd(0));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600533 }
534}
535
536static void release_pgd(pgd_t *spgd)
537{
538 /* If the entry's not present, there's nothing to release. */
539 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
540 unsigned int i;
541 pmd_t *pmdpage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
542
543 for (i = 0; i < PTRS_PER_PMD; i++)
544 release_pmd(&pmdpage[i]);
545
546 /* Now we can free the page of PMDs */
547 free_page((long)pmdpage);
548 /* And zero out the PGD entry so we never release it twice. */
549 set_pgd(spgd, __pgd(0));
550 }
551}
552
553#else /* !CONFIG_X86_PAE */
Rusty Russella91d74a2009-07-30 16:03:45 -0600554/*H:450
555 * If we chase down the release_pgd() code, the non-PAE version looks like
556 * this. The PAE version is almost identical, but instead of calling
557 * release_pte it calls release_pmd(), which looks much like this.
558 */
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600559static void release_pgd(pgd_t *spgd)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700560{
Rusty Russellbff672e2007-07-26 10:41:04 -0700561 /* If the entry's not present, there's nothing to release. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000562 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700563 unsigned int i;
Rusty Russell2e04ef72009-07-30 16:03:45 -0600564 /*
565 * Converting the pfn to find the actual PTE page is easy: turn
Rusty Russellbff672e2007-07-26 10:41:04 -0700566 * the page number into a physical address, then convert to a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600567 * virtual address (easy for kernel pages like this one).
568 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000569 pte_t *ptepage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
Rusty Russellbff672e2007-07-26 10:41:04 -0700570 /* For each entry in the page, we might need to release it. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000571 for (i = 0; i < PTRS_PER_PTE; i++)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700572 release_pte(ptepage[i]);
Rusty Russellbff672e2007-07-26 10:41:04 -0700573 /* Now we can free the page of PTEs */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700574 free_page((long)ptepage);
Rusty Russelle1e72962007-10-25 15:02:50 +1000575 /* And zero out the PGD entry so we never release it twice. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000576 *spgd = __pgd(0);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700577 }
578}
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600579#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -0600580
581/*H:445
582 * We saw flush_user_mappings() twice: once from the flush_user_mappings()
Rusty Russelle1e72962007-10-25 15:02:50 +1000583 * hypercall and once in new_pgdir() when we re-used a top-level pgdir page.
Rusty Russell2e04ef72009-07-30 16:03:45 -0600584 * It simply releases every PTE page from 0 up to the Guest's kernel address.
585 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700586static void flush_user_mappings(struct lguest *lg, int idx)
587{
588 unsigned int i;
Rusty Russellbff672e2007-07-26 10:41:04 -0700589 /* Release every pgd entry up to the kernel's address. */
Rusty Russell47436aa2007-10-22 11:03:36 +1000590 for (i = 0; i < pgd_index(lg->kernel_address); i++)
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600591 release_pgd(lg->pgdirs[idx].pgdir + i);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700592}
593
Rusty Russell2e04ef72009-07-30 16:03:45 -0600594/*H:440
595 * (v) Flushing (throwing away) page tables,
Rusty Russelle1e72962007-10-25 15:02:50 +1000596 *
597 * The Guest has a hypercall to throw away the page tables: it's used when a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600598 * large number of mappings have been changed.
599 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200600void guest_pagetable_flush_user(struct lg_cpu *cpu)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700601{
Rusty Russellbff672e2007-07-26 10:41:04 -0700602 /* Drop the userspace part of the current page table. */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200603 flush_user_mappings(cpu->lg, cpu->cpu_pgd);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700604}
Rusty Russellbff672e2007-07-26 10:41:04 -0700605/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700606
Rusty Russell47436aa2007-10-22 11:03:36 +1000607/* We walk down the guest page tables to get a guest-physical address */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200608unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russell47436aa2007-10-22 11:03:36 +1000609{
610 pgd_t gpgd;
611 pte_t gpte;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600612#ifdef CONFIG_X86_PAE
613 pmd_t gpmd;
614#endif
Rusty Russell47436aa2007-10-22 11:03:36 +1000615 /* First step: get the top-level Guest page table entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200616 gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
Rusty Russell47436aa2007-10-22 11:03:36 +1000617 /* Toplevel not present? We can't map it in. */
Rusty Russell6afbdd02009-03-30 21:55:23 -0600618 if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200619 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell6afbdd02009-03-30 21:55:23 -0600620 return -1UL;
621 }
Rusty Russell47436aa2007-10-22 11:03:36 +1000622
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600623#ifdef CONFIG_X86_PAE
624 gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
625 if (!(pmd_flags(gpmd) & _PAGE_PRESENT))
626 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600627 gpte = lgread(cpu, gpte_addr(cpu, gpmd, vaddr), pte_t);
628#else
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600629 gpte = lgread(cpu, gpte_addr(cpu, gpgd, vaddr), pte_t);
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600630#endif
Rusty Russell47436aa2007-10-22 11:03:36 +1000631 if (!(pte_flags(gpte) & _PAGE_PRESENT))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200632 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell47436aa2007-10-22 11:03:36 +1000633
634 return pte_pfn(gpte) * PAGE_SIZE | (vaddr & ~PAGE_MASK);
635}
636
Rusty Russell2e04ef72009-07-30 16:03:45 -0600637/*
638 * We keep several page tables. This is a simple routine to find the page
Rusty Russellbff672e2007-07-26 10:41:04 -0700639 * table (if any) corresponding to this top-level address the Guest has given
Rusty Russell2e04ef72009-07-30 16:03:45 -0600640 * us.
641 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700642static unsigned int find_pgdir(struct lguest *lg, unsigned long pgtable)
643{
644 unsigned int i;
645 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
Rusty Russell4357bd92008-03-11 09:35:57 -0500646 if (lg->pgdirs[i].pgdir && lg->pgdirs[i].gpgdir == pgtable)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700647 break;
648 return i;
649}
650
Rusty Russell2e04ef72009-07-30 16:03:45 -0600651/*H:435
652 * And this is us, creating the new page directory. If we really do
Rusty Russellbff672e2007-07-26 10:41:04 -0700653 * allocate a new one (and so the kernel parts are not there), we set
Rusty Russell2e04ef72009-07-30 16:03:45 -0600654 * blank_pgdir.
655 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200656static unsigned int new_pgdir(struct lg_cpu *cpu,
Rusty Russellee3db0f2007-10-22 11:03:34 +1000657 unsigned long gpgdir,
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700658 int *blank_pgdir)
659{
660 unsigned int next;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600661#ifdef CONFIG_X86_PAE
662 pmd_t *pmd_table;
663#endif
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700664
Rusty Russell2e04ef72009-07-30 16:03:45 -0600665 /*
666 * We pick one entry at random to throw out. Choosing the Least
667 * Recently Used might be better, but this is easy.
668 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200669 next = random32() % ARRAY_SIZE(cpu->lg->pgdirs);
Rusty Russellbff672e2007-07-26 10:41:04 -0700670 /* If it's never been allocated at all before, try now. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200671 if (!cpu->lg->pgdirs[next].pgdir) {
672 cpu->lg->pgdirs[next].pgdir =
673 (pgd_t *)get_zeroed_page(GFP_KERNEL);
Rusty Russellbff672e2007-07-26 10:41:04 -0700674 /* If the allocation fails, just keep using the one we have */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200675 if (!cpu->lg->pgdirs[next].pgdir)
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200676 next = cpu->cpu_pgd;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600677 else {
678#ifdef CONFIG_X86_PAE
Rusty Russell2e04ef72009-07-30 16:03:45 -0600679 /*
680 * In PAE mode, allocate a pmd page and populate the
681 * last pgd entry.
682 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600683 pmd_table = (pmd_t *)get_zeroed_page(GFP_KERNEL);
684 if (!pmd_table) {
685 free_page((long)cpu->lg->pgdirs[next].pgdir);
686 set_pgd(cpu->lg->pgdirs[next].pgdir, __pgd(0));
687 next = cpu->cpu_pgd;
688 } else {
689 set_pgd(cpu->lg->pgdirs[next].pgdir +
690 SWITCHER_PGD_INDEX,
691 __pgd(__pa(pmd_table) | _PAGE_PRESENT));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600692 /*
693 * This is a blank page, so there are no kernel
694 * mappings: caller must map the stack!
695 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600696 *blank_pgdir = 1;
697 }
698#else
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700699 *blank_pgdir = 1;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600700#endif
701 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700702 }
Rusty Russellbff672e2007-07-26 10:41:04 -0700703 /* Record which Guest toplevel this shadows. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200704 cpu->lg->pgdirs[next].gpgdir = gpgdir;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700705 /* Release all the non-kernel mappings. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200706 flush_user_mappings(cpu->lg, next);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700707
708 return next;
709}
710
Rusty Russell2e04ef72009-07-30 16:03:45 -0600711/*H:430
712 * (iv) Switching page tables
Rusty Russellbff672e2007-07-26 10:41:04 -0700713 *
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600714 * Now we've seen all the page table setting and manipulation, let's see
Rusty Russelle1e72962007-10-25 15:02:50 +1000715 * what happens when the Guest changes page tables (ie. changes the top-level
Rusty Russell2e04ef72009-07-30 16:03:45 -0600716 * pgdir). This occurs on almost every context switch.
717 */
Glauber de Oliveira Costa4665ac82008-01-07 11:05:35 -0200718void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700719{
720 int newpgdir, repin = 0;
721
Rusty Russellbff672e2007-07-26 10:41:04 -0700722 /* Look to see if we have this one already. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200723 newpgdir = find_pgdir(cpu->lg, pgtable);
Rusty Russell2e04ef72009-07-30 16:03:45 -0600724 /*
725 * If not, we allocate or mug an existing one: if it's a fresh one,
726 * repin gets set to 1.
727 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200728 if (newpgdir == ARRAY_SIZE(cpu->lg->pgdirs))
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200729 newpgdir = new_pgdir(cpu, pgtable, &repin);
Rusty Russellbff672e2007-07-26 10:41:04 -0700730 /* Change the current pgd index to the new one. */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200731 cpu->cpu_pgd = newpgdir;
Rusty Russellbff672e2007-07-26 10:41:04 -0700732 /* If it was completely blank, we map in the Guest kernel stack */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700733 if (repin)
Glauber de Oliveira Costa4665ac82008-01-07 11:05:35 -0200734 pin_stack_pages(cpu);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700735}
736
Rusty Russell2e04ef72009-07-30 16:03:45 -0600737/*H:470
738 * Finally, a routine which throws away everything: all PGD entries in all
Rusty Russelle1e72962007-10-25 15:02:50 +1000739 * the shadow page tables, including the Guest's kernel mappings. This is used
Rusty Russell2e04ef72009-07-30 16:03:45 -0600740 * when we destroy the Guest.
741 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700742static void release_all_pagetables(struct lguest *lg)
743{
744 unsigned int i, j;
745
Rusty Russellbff672e2007-07-26 10:41:04 -0700746 /* Every shadow pagetable this Guest has */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700747 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600748 if (lg->pgdirs[i].pgdir) {
749#ifdef CONFIG_X86_PAE
750 pgd_t *spgd;
751 pmd_t *pmdpage;
752 unsigned int k;
753
754 /* Get the last pmd page. */
755 spgd = lg->pgdirs[i].pgdir + SWITCHER_PGD_INDEX;
756 pmdpage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
757
Rusty Russell2e04ef72009-07-30 16:03:45 -0600758 /*
759 * And release the pmd entries of that pmd page,
760 * except for the switcher pmd.
761 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600762 for (k = 0; k < SWITCHER_PMD_INDEX; k++)
763 release_pmd(&pmdpage[k]);
764#endif
Rusty Russellbff672e2007-07-26 10:41:04 -0700765 /* Every PGD entry except the Switcher at the top */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700766 for (j = 0; j < SWITCHER_PGD_INDEX; j++)
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600767 release_pgd(lg->pgdirs[i].pgdir + j);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600768 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700769}
770
Rusty Russell2e04ef72009-07-30 16:03:45 -0600771/*
772 * We also throw away everything when a Guest tells us it's changed a kernel
Rusty Russellbff672e2007-07-26 10:41:04 -0700773 * mapping. Since kernel mappings are in every page table, it's easiest to
Rusty Russelle1e72962007-10-25 15:02:50 +1000774 * throw them all away. This traps the Guest in amber for a while as
Rusty Russell2e04ef72009-07-30 16:03:45 -0600775 * everything faults back in, but it's rare.
776 */
Glauber de Oliveira Costa4665ac82008-01-07 11:05:35 -0200777void guest_pagetable_clear_all(struct lg_cpu *cpu)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700778{
Glauber de Oliveira Costa4665ac82008-01-07 11:05:35 -0200779 release_all_pagetables(cpu->lg);
Rusty Russellbff672e2007-07-26 10:41:04 -0700780 /* We need the Guest kernel stack mapped again. */
Glauber de Oliveira Costa4665ac82008-01-07 11:05:35 -0200781 pin_stack_pages(cpu);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700782}
Rusty Russelle1e72962007-10-25 15:02:50 +1000783/*:*/
Rusty Russell2e04ef72009-07-30 16:03:45 -0600784
785/*M:009
786 * Since we throw away all mappings when a kernel mapping changes, our
Rusty Russelle1e72962007-10-25 15:02:50 +1000787 * performance sucks for guests using highmem. In fact, a guest with
788 * PAGE_OFFSET 0xc0000000 (the default) and more than about 700MB of RAM is
789 * usually slower than a Guest with less memory.
790 *
791 * This, of course, cannot be fixed. It would take some kind of... well, I
Rusty Russell2e04ef72009-07-30 16:03:45 -0600792 * don't know, but the term "puissant code-fu" comes to mind.
793:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700794
Rusty Russell2e04ef72009-07-30 16:03:45 -0600795/*H:420
796 * This is the routine which actually sets the page table entry for then
Rusty Russellbff672e2007-07-26 10:41:04 -0700797 * "idx"'th shadow page table.
798 *
799 * Normally, we can just throw out the old entry and replace it with 0: if they
800 * use it demand_page() will put the new entry in. We need to do this anyway:
801 * The Guest expects _PAGE_ACCESSED to be set on its PTE the first time a page
802 * is read from, and _PAGE_DIRTY when it's written to.
803 *
804 * But Avi Kivity pointed out that most Operating Systems (Linux included) set
805 * these bits on PTEs immediately anyway. This is done to save the CPU from
806 * having to update them, but it helps us the same way: if they set
807 * _PAGE_ACCESSED then we can put a read-only PTE entry in immediately, and if
808 * they set _PAGE_DIRTY then we can put a writable PTE entry in immediately.
809 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200810static void do_set_pte(struct lg_cpu *cpu, int idx,
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000811 unsigned long vaddr, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700812{
Rusty Russelle1e72962007-10-25 15:02:50 +1000813 /* Look up the matching shadow page directory entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200814 pgd_t *spgd = spgd_addr(cpu, idx, vaddr);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600815#ifdef CONFIG_X86_PAE
816 pmd_t *spmd;
817#endif
Rusty Russellbff672e2007-07-26 10:41:04 -0700818
819 /* If the top level isn't present, there's no entry to update. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000820 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600821#ifdef CONFIG_X86_PAE
822 spmd = spmd_addr(cpu, *spgd, vaddr);
823 if (pmd_flags(*spmd) & _PAGE_PRESENT) {
824#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -0600825 /* Otherwise, start by releasing the existing entry. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600826 pte_t *spte = spte_addr(cpu, *spgd, vaddr);
827 release_pte(*spte);
Rusty Russellbff672e2007-07-26 10:41:04 -0700828
Rusty Russell2e04ef72009-07-30 16:03:45 -0600829 /*
830 * If they're setting this entry as dirty or accessed,
831 * we might as well put that entry they've given us in
832 * now. This shaves 10% off a copy-on-write
833 * micro-benchmark.
834 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600835 if (pte_flags(gpte) & (_PAGE_DIRTY | _PAGE_ACCESSED)) {
836 check_gpte(cpu, gpte);
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600837 set_pte(spte,
838 gpte_to_spte(cpu, gpte,
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600839 pte_flags(gpte) & _PAGE_DIRTY));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600840 } else {
841 /*
842 * Otherwise kill it and we can demand_page()
843 * it in later.
844 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600845 set_pte(spte, __pte(0));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600846 }
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600847#ifdef CONFIG_X86_PAE
848 }
849#endif
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700850 }
851}
852
Rusty Russell2e04ef72009-07-30 16:03:45 -0600853/*H:410
854 * Updating a PTE entry is a little trickier.
Rusty Russellbff672e2007-07-26 10:41:04 -0700855 *
856 * We keep track of several different page tables (the Guest uses one for each
857 * process, so it makes sense to cache at least a few). Each of these have
858 * identical kernel parts: ie. every mapping above PAGE_OFFSET is the same for
859 * all processes. So when the page table above that address changes, we update
860 * all the page tables, not just the current one. This is rare.
861 *
Rusty Russella6bd8e12008-03-28 11:05:53 -0500862 * The benefit is that when we have to track a new page table, we can keep all
Rusty Russell2e04ef72009-07-30 16:03:45 -0600863 * the kernel mappings. This speeds up context switch immensely.
864 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200865void guest_set_pte(struct lg_cpu *cpu,
Rusty Russellee3db0f2007-10-22 11:03:34 +1000866 unsigned long gpgdir, unsigned long vaddr, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700867{
Rusty Russell2e04ef72009-07-30 16:03:45 -0600868 /*
869 * Kernel mappings must be changed on all top levels. Slow, but doesn't
870 * happen often.
871 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200872 if (vaddr >= cpu->lg->kernel_address) {
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700873 unsigned int i;
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200874 for (i = 0; i < ARRAY_SIZE(cpu->lg->pgdirs); i++)
875 if (cpu->lg->pgdirs[i].pgdir)
876 do_set_pte(cpu, i, vaddr, gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700877 } else {
Rusty Russellbff672e2007-07-26 10:41:04 -0700878 /* Is this page table one we have a shadow for? */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200879 int pgdir = find_pgdir(cpu->lg, gpgdir);
880 if (pgdir != ARRAY_SIZE(cpu->lg->pgdirs))
Rusty Russellbff672e2007-07-26 10:41:04 -0700881 /* If so, do the update. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200882 do_set_pte(cpu, pgdir, vaddr, gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700883 }
884}
885
Rusty Russellbff672e2007-07-26 10:41:04 -0700886/*H:400
Rusty Russelle1e72962007-10-25 15:02:50 +1000887 * (iii) Setting up a page table entry when the Guest tells us one has changed.
Rusty Russellbff672e2007-07-26 10:41:04 -0700888 *
889 * Just like we did in interrupts_and_traps.c, it makes sense for us to deal
890 * with the other side of page tables while we're here: what happens when the
891 * Guest asks for a page table to be updated?
892 *
893 * We already saw that demand_page() will fill in the shadow page tables when
894 * needed, so we can simply remove shadow page table entries whenever the Guest
895 * tells us they've changed. When the Guest tries to use the new entry it will
896 * fault and demand_page() will fix it up.
897 *
Anand Gadiyarfd589a82009-07-16 17:13:03 +0200898 * So with that in mind here's our code to update a (top-level) PGD entry:
Rusty Russellbff672e2007-07-26 10:41:04 -0700899 */
Matias Zabaljaureguiebe0ba82009-05-30 15:48:08 -0300900void guest_set_pgd(struct lguest *lg, unsigned long gpgdir, u32 idx)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700901{
902 int pgdir;
903
904 if (idx >= SWITCHER_PGD_INDEX)
905 return;
906
Rusty Russellbff672e2007-07-26 10:41:04 -0700907 /* If they're talking about a page table we have a shadow for... */
Rusty Russellee3db0f2007-10-22 11:03:34 +1000908 pgdir = find_pgdir(lg, gpgdir);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700909 if (pgdir < ARRAY_SIZE(lg->pgdirs))
Rusty Russellbff672e2007-07-26 10:41:04 -0700910 /* ... throw it away. */
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600911 release_pgd(lg->pgdirs[pgdir].pgdir + idx);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700912}
Rusty Russella91d74a2009-07-30 16:03:45 -0600913
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600914#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -0600915/* For setting a mid-level, we just throw everything away. It's easy. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600916void guest_set_pmd(struct lguest *lg, unsigned long pmdp, u32 idx)
917{
918 guest_pagetable_clear_all(&lg->cpus[0]);
919}
920#endif
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700921
Rusty Russella91d74a2009-07-30 16:03:45 -0600922/*H:505
923 * To get through boot, we construct simple identity page mappings (which
Rusty Russell2e04ef72009-07-30 16:03:45 -0600924 * set virtual == physical) and linear mappings which will get the Guest far
Rusty Russella91d74a2009-07-30 16:03:45 -0600925 * enough into the boot to create its own. The linear mapping means we
926 * simplify the Guest boot, but it makes assumptions about their PAGE_OFFSET,
927 * as you'll see.
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300928 *
929 * We lay them out of the way, just below the initrd (which is why we need to
Rusty Russell2e04ef72009-07-30 16:03:45 -0600930 * know its size here).
931 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300932static unsigned long setup_pagetables(struct lguest *lg,
933 unsigned long mem,
934 unsigned long initrd_size)
935{
936 pgd_t __user *pgdir;
937 pte_t __user *linear;
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300938 unsigned long mem_base = (unsigned long)lg->mem_base;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600939 unsigned int mapped_pages, i, linear_pages;
940#ifdef CONFIG_X86_PAE
941 pmd_t __user *pmds;
942 unsigned int j;
943 pgd_t pgd;
944 pmd_t pmd;
945#else
946 unsigned int phys_linear;
947#endif
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300948
Rusty Russell2e04ef72009-07-30 16:03:45 -0600949 /*
950 * We have mapped_pages frames to map, so we need linear_pages page
951 * tables to map them.
952 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300953 mapped_pages = mem / PAGE_SIZE;
954 linear_pages = (mapped_pages + PTRS_PER_PTE - 1) / PTRS_PER_PTE;
955
956 /* We put the toplevel page directory page at the top of memory. */
957 pgdir = (pgd_t *)(mem + mem_base - initrd_size - PAGE_SIZE);
958
959 /* Now we use the next linear_pages pages as pte pages */
960 linear = (void *)pgdir - linear_pages * PAGE_SIZE;
961
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600962#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -0600963 /*
964 * And the single mid page goes below that. We only use one, but
965 * that's enough to map 1G, which definitely gets us through boot.
966 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600967 pmds = (void *)linear - PAGE_SIZE;
968#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -0600969 /*
970 * Linear mapping is easy: put every page's address into the
971 * mapping in order.
972 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -0300973 for (i = 0; i < mapped_pages; i++) {
974 pte_t pte;
975 pte = pfn_pte(i, __pgprot(_PAGE_PRESENT|_PAGE_RW|_PAGE_USER));
976 if (copy_to_user(&linear[i], &pte, sizeof(pte)) != 0)
977 return -EFAULT;
978 }
979
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600980#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -0600981 /*
982 * Make the Guest PMD entries point to the corresponding place in the
983 * linear mapping (up to one page worth of PMD).
984 */
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600985 for (i = j = 0; i < mapped_pages && j < PTRS_PER_PMD;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600986 i += PTRS_PER_PTE, j++) {
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600987 pmd = pfn_pmd(((unsigned long)&linear[i] - mem_base)/PAGE_SIZE,
988 __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600989
990 if (copy_to_user(&pmds[j], &pmd, sizeof(pmd)) != 0)
991 return -EFAULT;
992 }
993
Rusty Russella91d74a2009-07-30 16:03:45 -0600994 /* One PGD entry, pointing to that PMD page. */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600995 pgd = __pgd(((unsigned long)pmds - mem_base) | _PAGE_PRESENT);
Rusty Russella91d74a2009-07-30 16:03:45 -0600996 /* Copy it in as the first PGD entry (ie. addresses 0-1G). */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600997 if (copy_to_user(&pgdir[0], &pgd, sizeof(pgd)) != 0)
998 return -EFAULT;
Rusty Russella91d74a2009-07-30 16:03:45 -0600999 /*
Rusty Russellfb100d72009-09-23 22:26:46 -06001000 * And the other PGD entry to make the linear mapping at PAGE_OFFSET
Rusty Russella91d74a2009-07-30 16:03:45 -06001001 */
Rusty Russellfb100d72009-09-23 22:26:46 -06001002 if (copy_to_user(&pgdir[KERNEL_PGD_BOUNDARY], &pgd, sizeof(pgd)))
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001003 return -EFAULT;
1004#else
Rusty Russella91d74a2009-07-30 16:03:45 -06001005 /*
1006 * The top level points to the linear page table pages above.
1007 * We setup the identity and linear mappings here.
1008 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001009 phys_linear = (unsigned long)linear - mem_base;
1010 for (i = 0; i < mapped_pages; i += PTRS_PER_PTE) {
1011 pgd_t pgd;
Rusty Russella91d74a2009-07-30 16:03:45 -06001012 /*
1013 * Create a PGD entry which points to the right part of the
1014 * linear PTE pages.
1015 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001016 pgd = __pgd((phys_linear + i * sizeof(pte_t)) |
1017 (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER));
1018
Rusty Russella91d74a2009-07-30 16:03:45 -06001019 /*
1020 * Copy it into the PGD page at 0 and PAGE_OFFSET.
1021 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001022 if (copy_to_user(&pgdir[i / PTRS_PER_PTE], &pgd, sizeof(pgd))
1023 || copy_to_user(&pgdir[pgd_index(PAGE_OFFSET)
1024 + i / PTRS_PER_PTE],
1025 &pgd, sizeof(pgd)))
1026 return -EFAULT;
1027 }
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001028#endif
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001029
Rusty Russell2e04ef72009-07-30 16:03:45 -06001030 /*
Rusty Russella91d74a2009-07-30 16:03:45 -06001031 * We return the top level (guest-physical) address: we remember where
1032 * this is to write it into lguest_data when the Guest initializes.
Rusty Russell2e04ef72009-07-30 16:03:45 -06001033 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001034 return (unsigned long)pgdir - mem_base;
1035}
1036
Rusty Russell2e04ef72009-07-30 16:03:45 -06001037/*H:500
1038 * (vii) Setting up the page tables initially.
Rusty Russellbff672e2007-07-26 10:41:04 -07001039 *
1040 * When a Guest is first created, the Launcher tells us where the toplevel of
Rusty Russell2e04ef72009-07-30 16:03:45 -06001041 * its first page table is. We set some things up here:
1042 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001043int init_guest_pagetable(struct lguest *lg)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001044{
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001045 u64 mem;
1046 u32 initrd_size;
1047 struct boot_params __user *boot = (struct boot_params *)lg->mem_base;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001048#ifdef CONFIG_X86_PAE
1049 pgd_t *pgd;
1050 pmd_t *pmd_table;
1051#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -06001052 /*
1053 * Get the Guest memory size and the ramdisk size from the boot header
1054 * located at lg->mem_base (Guest address 0).
1055 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001056 if (copy_from_user(&mem, &boot->e820_map[0].size, sizeof(mem))
1057 || get_user(initrd_size, &boot->hdr.ramdisk_size))
1058 return -EFAULT;
1059
Rusty Russell2e04ef72009-07-30 16:03:45 -06001060 /*
1061 * We start on the first shadow page table, and give it a blank PGD
1062 * page.
1063 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001064 lg->pgdirs[0].gpgdir = setup_pagetables(lg, mem, initrd_size);
1065 if (IS_ERR_VALUE(lg->pgdirs[0].gpgdir))
1066 return lg->pgdirs[0].gpgdir;
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -02001067 lg->pgdirs[0].pgdir = (pgd_t *)get_zeroed_page(GFP_KERNEL);
1068 if (!lg->pgdirs[0].pgdir)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001069 return -ENOMEM;
Rusty Russella91d74a2009-07-30 16:03:45 -06001070
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001071#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -06001072 /* For PAE, we also create the initial mid-level. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001073 pgd = lg->pgdirs[0].pgdir;
1074 pmd_table = (pmd_t *) get_zeroed_page(GFP_KERNEL);
1075 if (!pmd_table)
1076 return -ENOMEM;
1077
1078 set_pgd(pgd + SWITCHER_PGD_INDEX,
1079 __pgd(__pa(pmd_table) | _PAGE_PRESENT));
1080#endif
Rusty Russella91d74a2009-07-30 16:03:45 -06001081
1082 /* This is the current page table. */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -02001083 lg->cpus[0].cpu_pgd = 0;
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001084 return 0;
1085}
1086
Rusty Russella91d74a2009-07-30 16:03:45 -06001087/*H:508 When the Guest calls LHCALL_LGUEST_INIT we do more setup. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001088void page_table_guest_data_init(struct lg_cpu *cpu)
Rusty Russell47436aa2007-10-22 11:03:36 +10001089{
1090 /* We get the kernel address: above this is all kernel memory. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001091 if (get_user(cpu->lg->kernel_address,
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001092 &cpu->lg->lguest_data->kernel_address)
Rusty Russell2e04ef72009-07-30 16:03:45 -06001093 /*
1094 * We tell the Guest that it can't use the top 2 or 4 MB
1095 * of virtual addresses used by the Switcher.
1096 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001097 || put_user(RESERVE_MEM * 1024 * 1024,
1098 &cpu->lg->lguest_data->reserve_mem)
1099 || put_user(cpu->lg->pgdirs[0].gpgdir,
1100 &cpu->lg->lguest_data->pgdir))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001101 kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data);
Rusty Russell47436aa2007-10-22 11:03:36 +10001102
Rusty Russell2e04ef72009-07-30 16:03:45 -06001103 /*
1104 * In flush_user_mappings() we loop from 0 to
Rusty Russell47436aa2007-10-22 11:03:36 +10001105 * "pgd_index(lg->kernel_address)". This assumes it won't hit the
Rusty Russell2e04ef72009-07-30 16:03:45 -06001106 * Switcher mappings, so check that now.
1107 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001108#ifdef CONFIG_X86_PAE
1109 if (pgd_index(cpu->lg->kernel_address) == SWITCHER_PGD_INDEX &&
1110 pmd_index(cpu->lg->kernel_address) == SWITCHER_PMD_INDEX)
1111#else
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001112 if (pgd_index(cpu->lg->kernel_address) >= SWITCHER_PGD_INDEX)
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001113#endif
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001114 kill_guest(cpu, "bad kernel address %#lx",
1115 cpu->lg->kernel_address);
Rusty Russell47436aa2007-10-22 11:03:36 +10001116}
1117
Rusty Russellbff672e2007-07-26 10:41:04 -07001118/* When a Guest dies, our cleanup is fairly simple. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001119void free_guest_pagetable(struct lguest *lg)
1120{
1121 unsigned int i;
1122
Rusty Russellbff672e2007-07-26 10:41:04 -07001123 /* Throw away all page table pages. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001124 release_all_pagetables(lg);
Rusty Russellbff672e2007-07-26 10:41:04 -07001125 /* Now free the top levels: free_page() can handle 0 just fine. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001126 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
1127 free_page((long)lg->pgdirs[i].pgdir);
1128}
1129
Rusty Russell2e04ef72009-07-30 16:03:45 -06001130/*H:480
1131 * (vi) Mapping the Switcher when the Guest is about to run.
Rusty Russellbff672e2007-07-26 10:41:04 -07001132 *
Rusty Russelle1e72962007-10-25 15:02:50 +10001133 * The Switcher and the two pages for this CPU need to be visible in the
Rusty Russellbff672e2007-07-26 10:41:04 -07001134 * Guest (and not the pages for other CPUs). We have the appropriate PTE pages
Rusty Russelle1e72962007-10-25 15:02:50 +10001135 * for each CPU already set up, we just need to hook them in now we know which
Rusty Russell2e04ef72009-07-30 16:03:45 -06001136 * Guest is about to run on this CPU.
1137 */
Glauber de Oliveira Costa0c784412008-01-07 11:05:30 -02001138void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001139{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001140 pte_t *switcher_pte_page = __get_cpu_var(switcher_pte_pages);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001141 pte_t regs_pte;
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001142
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001143#ifdef CONFIG_X86_PAE
1144 pmd_t switcher_pmd;
1145 pmd_t *pmd_table;
1146
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001147 switcher_pmd = pfn_pmd(__pa(switcher_pte_page) >> PAGE_SHIFT,
1148 PAGE_KERNEL_EXEC);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001149
Rusty Russella91d74a2009-07-30 16:03:45 -06001150 /* Figure out where the pmd page is, by reading the PGD, and converting
1151 * it to a virtual address. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001152 pmd_table = __va(pgd_pfn(cpu->lg->
1153 pgdirs[cpu->cpu_pgd].pgdir[SWITCHER_PGD_INDEX])
1154 << PAGE_SHIFT);
Rusty Russella91d74a2009-07-30 16:03:45 -06001155 /* Now write it into the shadow page table. */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001156 set_pmd(&pmd_table[SWITCHER_PMD_INDEX], switcher_pmd);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001157#else
1158 pgd_t switcher_pgd;
1159
Rusty Russell2e04ef72009-07-30 16:03:45 -06001160 /*
1161 * Make the last PGD entry for this Guest point to the Switcher's PTE
1162 * page for this CPU (with appropriate flags).
1163 */
Matias Zabaljaureguied1dc772009-05-30 15:35:49 -03001164 switcher_pgd = __pgd(__pa(switcher_pte_page) | __PAGE_KERNEL_EXEC);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001165
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -02001166 cpu->lg->pgdirs[cpu->cpu_pgd].pgdir[SWITCHER_PGD_INDEX] = switcher_pgd;
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001167
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001168#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -06001169 /*
1170 * We also change the Switcher PTE page. When we're running the Guest,
Rusty Russellbff672e2007-07-26 10:41:04 -07001171 * we want the Guest's "regs" page to appear where the first Switcher
1172 * page for this CPU is. This is an optimization: when the Switcher
1173 * saves the Guest registers, it saves them into the first page of this
1174 * CPU's "struct lguest_pages": if we make sure the Guest's register
1175 * page is already mapped there, we don't have to copy them out
Rusty Russell2e04ef72009-07-30 16:03:45 -06001176 * again.
1177 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001178 regs_pte = pfn_pte(__pa(cpu->regs_page) >> PAGE_SHIFT, PAGE_KERNEL);
1179 set_pte(&switcher_pte_page[pte_index((unsigned long)pages)], regs_pte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001180}
Rusty Russellbff672e2007-07-26 10:41:04 -07001181/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001182
1183static void free_switcher_pte_pages(void)
1184{
1185 unsigned int i;
1186
1187 for_each_possible_cpu(i)
1188 free_page((long)switcher_pte_page(i));
1189}
1190
Rusty Russell2e04ef72009-07-30 16:03:45 -06001191/*H:520
1192 * Setting up the Switcher PTE page for given CPU is fairly easy, given
Rusty Russellbff672e2007-07-26 10:41:04 -07001193 * the CPU number and the "struct page"s for the Switcher code itself.
1194 *
Rusty Russell2e04ef72009-07-30 16:03:45 -06001195 * Currently the Switcher is less than a page long, so "pages" is always 1.
1196 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001197static __init void populate_switcher_pte_page(unsigned int cpu,
1198 struct page *switcher_page[],
1199 unsigned int pages)
1200{
1201 unsigned int i;
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001202 pte_t *pte = switcher_pte_page(cpu);
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001203
Rusty Russellbff672e2007-07-26 10:41:04 -07001204 /* The first entries are easy: they map the Switcher code. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001205 for (i = 0; i < pages; i++) {
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001206 set_pte(&pte[i], mk_pte(switcher_page[i],
Matias Zabaljauregui90603d12009-06-12 22:27:06 -06001207 __pgprot(_PAGE_PRESENT|_PAGE_ACCESSED)));
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001208 }
1209
Rusty Russellbff672e2007-07-26 10:41:04 -07001210 /* The only other thing we map is this CPU's pair of pages. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001211 i = pages + cpu*2;
1212
Rusty Russellbff672e2007-07-26 10:41:04 -07001213 /* First page (Guest registers) is writable from the Guest */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001214 set_pte(&pte[i], pfn_pte(page_to_pfn(switcher_page[i]),
Matias Zabaljauregui90603d12009-06-12 22:27:06 -06001215 __pgprot(_PAGE_PRESENT|_PAGE_ACCESSED|_PAGE_RW)));
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001216
Rusty Russell2e04ef72009-07-30 16:03:45 -06001217 /*
1218 * The second page contains the "struct lguest_ro_state", and is
1219 * read-only.
1220 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -06001221 set_pte(&pte[i+1], pfn_pte(page_to_pfn(switcher_page[i+1]),
Matias Zabaljauregui90603d12009-06-12 22:27:06 -06001222 __pgprot(_PAGE_PRESENT|_PAGE_ACCESSED)));
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001223}
1224
Rusty Russell2e04ef72009-07-30 16:03:45 -06001225/*
1226 * We've made it through the page table code. Perhaps our tired brains are
Rusty Russelle1e72962007-10-25 15:02:50 +10001227 * still processing the details, or perhaps we're simply glad it's over.
1228 *
Rusty Russella6bd8e12008-03-28 11:05:53 -05001229 * If nothing else, note that all this complexity in juggling shadow page tables
1230 * in sync with the Guest's page tables is for one reason: for most Guests this
1231 * page table dance determines how bad performance will be. This is why Xen
1232 * uses exotic direct Guest pagetable manipulation, and why both Intel and AMD
1233 * have implemented shadow page table support directly into hardware.
Rusty Russelle1e72962007-10-25 15:02:50 +10001234 *
Rusty Russell2e04ef72009-07-30 16:03:45 -06001235 * There is just one file remaining in the Host.
1236 */
Rusty Russelle1e72962007-10-25 15:02:50 +10001237
Rusty Russell2e04ef72009-07-30 16:03:45 -06001238/*H:510
1239 * At boot or module load time, init_pagetables() allocates and populates
1240 * the Switcher PTE page for each CPU.
1241 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001242__init int init_pagetables(struct page **switcher_page, unsigned int pages)
1243{
1244 unsigned int i;
1245
1246 for_each_possible_cpu(i) {
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001247 switcher_pte_page(i) = (pte_t *)get_zeroed_page(GFP_KERNEL);
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001248 if (!switcher_pte_page(i)) {
1249 free_switcher_pte_pages();
1250 return -ENOMEM;
1251 }
1252 populate_switcher_pte_page(i, switcher_page, pages);
1253 }
1254 return 0;
1255}
Rusty Russellbff672e2007-07-26 10:41:04 -07001256/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001257
Rusty Russellbff672e2007-07-26 10:41:04 -07001258/* Cleaning up simply involves freeing the PTE page for each CPU. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001259void free_pagetables(void)
1260{
1261 free_switcher_pte_pages();
1262}