blob: bfb39bb56ef1b3cd03204a9cd010fa81ec418350 [file] [log] [blame]
Rusty Russell2e04ef72009-07-30 16:03:45 -06001/*P:700
2 * The pagetable code, on the other hand, still shows the scars of
Rusty Russellf938d2c2007-07-26 10:41:02 -07003 * previous encounters. It's functional, and as neat as it can be in the
4 * circumstances, but be wary, for these things are subtle and break easily.
5 * The Guest provides a virtual to physical mapping, but we can neither trust
Rusty Russella6bd8e12008-03-28 11:05:53 -05006 * it nor use it: we verify and convert it here then point the CPU to the
Rusty Russell2e04ef72009-07-30 16:03:45 -06007 * converted Guest pages when running the Guest.
8:*/
Rusty Russellf938d2c2007-07-26 10:41:02 -07009
Rusty Russell6d0cda92013-04-22 14:10:41 +093010/* Copyright (C) Rusty Russell IBM Corporation 2013.
Rusty Russelld7e28ff2007-07-19 01:49:23 -070011 * GPL v2 and any later version */
12#include <linux/mm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/gfp.h>
Rusty Russelld7e28ff2007-07-19 01:49:23 -070014#include <linux/types.h>
15#include <linux/spinlock.h>
16#include <linux/random.h>
17#include <linux/percpu.h>
18#include <asm/tlbflush.h>
Rusty Russell47436aa2007-10-22 11:03:36 +100019#include <asm/uaccess.h>
Rusty Russelld7e28ff2007-07-19 01:49:23 -070020#include "lg.h"
21
Rusty Russell2e04ef72009-07-30 16:03:45 -060022/*M:008
23 * We hold reference to pages, which prevents them from being swapped.
Rusty Russellf56a3842007-07-26 10:41:05 -070024 * It'd be nice to have a callback in the "struct mm_struct" when Linux wants
25 * to swap out. If we had this, and a shrinker callback to trim PTE pages, we
Rusty Russell2e04ef72009-07-30 16:03:45 -060026 * could probably consider launching Guests as non-root.
27:*/
Rusty Russellf56a3842007-07-26 10:41:05 -070028
Rusty Russellbff672e2007-07-26 10:41:04 -070029/*H:300
30 * The Page Table Code
31 *
Rusty Russella91d74a2009-07-30 16:03:45 -060032 * We use two-level page tables for the Guest, or three-level with PAE. If
33 * you're not entirely comfortable with virtual addresses, physical addresses
34 * and page tables then I recommend you review arch/x86/lguest/boot.c's "Page
35 * Table Handling" (with diagrams!).
Rusty Russellbff672e2007-07-26 10:41:04 -070036 *
37 * The Guest keeps page tables, but we maintain the actual ones here: these are
38 * called "shadow" page tables. Which is a very Guest-centric name: these are
39 * the real page tables the CPU uses, although we keep them up to date to
40 * reflect the Guest's. (See what I mean about weird naming? Since when do
41 * shadows reflect anything?)
42 *
43 * Anyway, this is the most complicated part of the Host code. There are seven
44 * parts to this:
Rusty Russelle1e72962007-10-25 15:02:50 +100045 * (i) Looking up a page table entry when the Guest faults,
46 * (ii) Making sure the Guest stack is mapped,
47 * (iii) Setting up a page table entry when the Guest tells us one has changed,
Rusty Russellbff672e2007-07-26 10:41:04 -070048 * (iv) Switching page tables,
Rusty Russelle1e72962007-10-25 15:02:50 +100049 * (v) Flushing (throwing away) page tables,
Rusty Russellbff672e2007-07-26 10:41:04 -070050 * (vi) Mapping the Switcher when the Guest is about to run,
51 * (vii) Setting up the page tables initially.
Rusty Russell2e04ef72009-07-30 16:03:45 -060052:*/
Rusty Russellbff672e2007-07-26 10:41:04 -070053
Rusty Russell2e04ef72009-07-30 16:03:45 -060054/*
Rusty Russella91d74a2009-07-30 16:03:45 -060055 * The Switcher uses the complete top PTE page. That's 1024 PTE entries (4MB)
56 * or 512 PTE entries with PAE (2MB).
Rusty Russell2e04ef72009-07-30 16:03:45 -060057 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100058#define SWITCHER_PGD_INDEX (PTRS_PER_PGD - 1)
Rusty Russelld7e28ff2007-07-19 01:49:23 -070059
Rusty Russell2e04ef72009-07-30 16:03:45 -060060/*
61 * For PAE we need the PMD index as well. We use the last 2MB, so we
62 * will need the last pmd entry of the last pmd page.
63 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060064#ifdef CONFIG_X86_PAE
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060065#define CHECK_GPGD_MASK _PAGE_PRESENT
66#else
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060067#define CHECK_GPGD_MASK _PAGE_TABLE
68#endif
69
Rusty Russell2e04ef72009-07-30 16:03:45 -060070/*H:320
71 * The page table code is curly enough to need helper functions to keep it
Rusty Russella91d74a2009-07-30 16:03:45 -060072 * clear and clean. The kernel itself provides many of them; one advantage
Paul Bollee3d18482013-05-29 09:30:19 +020073 * of insisting that the Guest and Host use the same CONFIG_X86_PAE setting.
Rusty Russellbff672e2007-07-26 10:41:04 -070074 *
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100075 * There are two functions which return pointers to the shadow (aka "real")
Rusty Russellbff672e2007-07-26 10:41:04 -070076 * page tables.
77 *
78 * spgd_addr() takes the virtual address and returns a pointer to the top-level
Rusty Russelle1e72962007-10-25 15:02:50 +100079 * page directory entry (PGD) for that address. Since we keep track of several
80 * page tables, the "i" argument tells us which one we're interested in (it's
Rusty Russell2e04ef72009-07-30 16:03:45 -060081 * usually the current one).
82 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -020083static pgd_t *spgd_addr(struct lg_cpu *cpu, u32 i, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -070084{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +100085 unsigned int index = pgd_index(vaddr);
Rusty Russelld7e28ff2007-07-19 01:49:23 -070086
Rusty Russellbff672e2007-07-26 10:41:04 -070087 /* Return a pointer index'th pgd entry for the i'th page table. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -020088 return &cpu->lg->pgdirs[i].pgdir[index];
Rusty Russelld7e28ff2007-07-19 01:49:23 -070089}
90
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060091#ifdef CONFIG_X86_PAE
Rusty Russell2e04ef72009-07-30 16:03:45 -060092/*
93 * This routine then takes the PGD entry given above, which contains the
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060094 * address of the PMD page. It then returns a pointer to the PMD entry for the
Rusty Russell2e04ef72009-07-30 16:03:45 -060095 * given address.
96 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -060097static pmd_t *spmd_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
98{
99 unsigned int index = pmd_index(vaddr);
100 pmd_t *page;
101
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600102 /* You should never call this if the PGD entry wasn't valid */
103 BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
104 page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
105
106 return &page[index];
107}
108#endif
109
Rusty Russell2e04ef72009-07-30 16:03:45 -0600110/*
111 * This routine then takes the page directory entry returned above, which
Rusty Russelle1e72962007-10-25 15:02:50 +1000112 * contains the address of the page table entry (PTE) page. It then returns a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600113 * pointer to the PTE entry for the given address.
114 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600115static pte_t *spte_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700116{
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600117#ifdef CONFIG_X86_PAE
118 pmd_t *pmd = spmd_addr(cpu, spgd, vaddr);
119 pte_t *page = __va(pmd_pfn(*pmd) << PAGE_SHIFT);
120
121 /* You should never call this if the PMD entry wasn't valid */
122 BUG_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT));
123#else
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000124 pte_t *page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
Rusty Russellbff672e2007-07-26 10:41:04 -0700125 /* You should never call this if the PGD entry wasn't valid */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000126 BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600127#endif
128
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600129 return &page[pte_index(vaddr)];
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700130}
131
Rusty Russell2e04ef72009-07-30 16:03:45 -0600132/*
Rusty Russell9f542882011-07-22 14:39:50 +0930133 * These functions are just like the above, except they access the Guest
Rusty Russell2e04ef72009-07-30 16:03:45 -0600134 * page tables. Hence they return a Guest address.
135 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200136static unsigned long gpgd_addr(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700137{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000138 unsigned int index = vaddr >> (PGDIR_SHIFT);
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200139 return cpu->lg->pgdirs[cpu->cpu_pgd].gpgdir + index * sizeof(pgd_t);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700140}
141
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600142#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -0600143/* Follow the PGD to the PMD. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600144static unsigned long gpmd_addr(pgd_t gpgd, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700145{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000146 unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
147 BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600148 return gpage + pmd_index(vaddr) * sizeof(pmd_t);
149}
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600150
Rusty Russella91d74a2009-07-30 16:03:45 -0600151/* Follow the PMD to the PTE. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600152static unsigned long gpte_addr(struct lg_cpu *cpu,
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600153 pmd_t gpmd, unsigned long vaddr)
154{
155 unsigned long gpage = pmd_pfn(gpmd) << PAGE_SHIFT;
156
157 BUG_ON(!(pmd_flags(gpmd) & _PAGE_PRESENT));
158 return gpage + pte_index(vaddr) * sizeof(pte_t);
159}
160#else
Rusty Russella91d74a2009-07-30 16:03:45 -0600161/* Follow the PGD to the PTE (no mid-level for !PAE). */
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600162static unsigned long gpte_addr(struct lg_cpu *cpu,
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600163 pgd_t gpgd, unsigned long vaddr)
164{
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600165 unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600166
167 BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600168 return gpage + pte_index(vaddr) * sizeof(pte_t);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700169}
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600170#endif
Rusty Russella6bd8e12008-03-28 11:05:53 -0500171/*:*/
172
Rusty Russell9f542882011-07-22 14:39:50 +0930173/*M:007
Rusty Russell2e04ef72009-07-30 16:03:45 -0600174 * get_pfn is slow: we could probably try to grab batches of pages here as
175 * an optimization (ie. pre-faulting).
176:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700177
Rusty Russell2e04ef72009-07-30 16:03:45 -0600178/*H:350
179 * This routine takes a page number given by the Guest and converts it to
Rusty Russellbff672e2007-07-26 10:41:04 -0700180 * an actual, physical page number. It can fail for several reasons: the
181 * virtual address might not be mapped by the Launcher, the write flag is set
182 * and the page is read-only, or the write flag was set and the page was
183 * shared so had to be copied, but we ran out of memory.
184 *
Rusty Russella6bd8e12008-03-28 11:05:53 -0500185 * This holds a reference to the page, so release_pte() is careful to put that
Rusty Russell2e04ef72009-07-30 16:03:45 -0600186 * back.
187 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700188static unsigned long get_pfn(unsigned long virtpfn, int write)
189{
190 struct page *page;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700191
Rusty Russell71a3f4e2008-08-12 17:52:53 -0500192 /* gup me one page at this address please! */
193 if (get_user_pages_fast(virtpfn << PAGE_SHIFT, 1, write, &page) == 1)
194 return page_to_pfn(page);
195
196 /* This value indicates failure. */
197 return -1UL;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700198}
199
Rusty Russell2e04ef72009-07-30 16:03:45 -0600200/*H:340
201 * Converting a Guest page table entry to a shadow (ie. real) page table
Rusty Russellbff672e2007-07-26 10:41:04 -0700202 * entry can be a little tricky. The flags are (almost) the same, but the
203 * Guest PTE contains a virtual page number: the CPU needs the real page
Rusty Russell2e04ef72009-07-30 16:03:45 -0600204 * number.
205 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200206static pte_t gpte_to_spte(struct lg_cpu *cpu, pte_t gpte, int write)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700207{
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000208 unsigned long pfn, base, flags;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700209
Rusty Russell2e04ef72009-07-30 16:03:45 -0600210 /*
211 * The Guest sets the global flag, because it thinks that it is using
Rusty Russellbff672e2007-07-26 10:41:04 -0700212 * PGE. We only told it to use PGE so it would tell us whether it was
213 * flushing a kernel mapping or a userspace mapping. We don't actually
Rusty Russell2e04ef72009-07-30 16:03:45 -0600214 * use the global bit, so throw it away.
215 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000216 flags = (pte_flags(gpte) & ~_PAGE_GLOBAL);
Rusty Russellbff672e2007-07-26 10:41:04 -0700217
Rusty Russell3c6b5bf2007-10-22 11:03:26 +1000218 /* The Guest's pages are offset inside the Launcher. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200219 base = (unsigned long)cpu->lg->mem_base / PAGE_SIZE;
Rusty Russell3c6b5bf2007-10-22 11:03:26 +1000220
Rusty Russell2e04ef72009-07-30 16:03:45 -0600221 /*
222 * We need a temporary "unsigned long" variable to hold the answer from
Rusty Russellbff672e2007-07-26 10:41:04 -0700223 * get_pfn(), because it returns 0xFFFFFFFF on failure, which wouldn't
224 * fit in spte.pfn. get_pfn() finds the real physical number of the
Rusty Russell2e04ef72009-07-30 16:03:45 -0600225 * page, given the virtual number.
226 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000227 pfn = get_pfn(base + pte_pfn(gpte), write);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700228 if (pfn == -1UL) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200229 kill_guest(cpu, "failed to get page %lu", pte_pfn(gpte));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600230 /*
231 * When we destroy the Guest, we'll go through the shadow page
Rusty Russellbff672e2007-07-26 10:41:04 -0700232 * tables and release_pte() them. Make sure we don't think
Rusty Russell2e04ef72009-07-30 16:03:45 -0600233 * this one is valid!
234 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000235 flags = 0;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700236 }
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000237 /* Now we assemble our shadow PTE from the page number and flags. */
238 return pfn_pte(pfn, __pgprot(flags));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700239}
240
Rusty Russellbff672e2007-07-26 10:41:04 -0700241/*H:460 And to complete the chain, release_pte() looks like this: */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000242static void release_pte(pte_t pte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700243{
Rusty Russell2e04ef72009-07-30 16:03:45 -0600244 /*
245 * Remember that get_user_pages_fast() took a reference to the page, in
246 * get_pfn()? We have to put it back now.
247 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000248 if (pte_flags(pte) & _PAGE_PRESENT)
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600249 put_page(pte_page(pte));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700250}
Rusty Russellbff672e2007-07-26 10:41:04 -0700251/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700252
Rusty Russelle1d12602013-04-22 14:10:39 +0930253static bool check_gpte(struct lg_cpu *cpu, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700254{
Ahmed S. Darwish31f4b462008-02-09 23:24:09 +0100255 if ((pte_flags(gpte) & _PAGE_PSE) ||
Rusty Russelle1d12602013-04-22 14:10:39 +0930256 pte_pfn(gpte) >= cpu->lg->pfn_limit) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200257 kill_guest(cpu, "bad page table entry");
Rusty Russelle1d12602013-04-22 14:10:39 +0930258 return false;
259 }
260 return true;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700261}
262
Rusty Russelle1d12602013-04-22 14:10:39 +0930263static bool check_gpgd(struct lg_cpu *cpu, pgd_t gpgd)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700264{
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600265 if ((pgd_flags(gpgd) & ~CHECK_GPGD_MASK) ||
Rusty Russelle1d12602013-04-22 14:10:39 +0930266 (pgd_pfn(gpgd) >= cpu->lg->pfn_limit)) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200267 kill_guest(cpu, "bad page directory entry");
Rusty Russelle1d12602013-04-22 14:10:39 +0930268 return false;
269 }
270 return true;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700271}
272
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600273#ifdef CONFIG_X86_PAE
Rusty Russelle1d12602013-04-22 14:10:39 +0930274static bool check_gpmd(struct lg_cpu *cpu, pmd_t gpmd)
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600275{
276 if ((pmd_flags(gpmd) & ~_PAGE_TABLE) ||
Rusty Russelle1d12602013-04-22 14:10:39 +0930277 (pmd_pfn(gpmd) >= cpu->lg->pfn_limit)) {
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600278 kill_guest(cpu, "bad page middle directory entry");
Rusty Russelle1d12602013-04-22 14:10:39 +0930279 return false;
280 }
281 return true;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600282}
283#endif
284
Rusty Russell17427e02013-04-22 14:10:39 +0930285/*H:331
286 * This is the core routine to walk the shadow page tables and find the page
287 * table entry for a specific address.
288 *
289 * If allocate is set, then we allocate any missing levels, setting the flags
290 * on the new page directory and mid-level directories using the arguments
291 * (which are copied from the Guest's page table entries).
292 */
293static pte_t *find_spte(struct lg_cpu *cpu, unsigned long vaddr, bool allocate,
294 int pgd_flags, int pmd_flags)
295{
296 pgd_t *spgd;
297 /* Mid level for PAE. */
298#ifdef CONFIG_X86_PAE
299 pmd_t *spmd;
300#endif
301
302 /* Get top level entry. */
303 spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
304 if (!(pgd_flags(*spgd) & _PAGE_PRESENT)) {
305 /* No shadow entry: allocate a new shadow PTE page. */
306 unsigned long ptepage;
307
308 /* If they didn't want us to allocate anything, stop. */
309 if (!allocate)
310 return NULL;
311
312 ptepage = get_zeroed_page(GFP_KERNEL);
313 /*
314 * This is not really the Guest's fault, but killing it is
315 * simple for this corner case.
316 */
317 if (!ptepage) {
318 kill_guest(cpu, "out of memory allocating pte page");
319 return NULL;
320 }
321 /*
322 * And we copy the flags to the shadow PGD entry. The page
323 * number in the shadow PGD is the page we just allocated.
324 */
325 set_pgd(spgd, __pgd(__pa(ptepage) | pgd_flags));
326 }
327
328 /*
329 * Intel's Physical Address Extension actually uses three levels of
330 * page tables, so we need to look in the mid-level.
331 */
332#ifdef CONFIG_X86_PAE
333 /* Now look at the mid-level shadow entry. */
334 spmd = spmd_addr(cpu, *spgd, vaddr);
335
336 if (!(pmd_flags(*spmd) & _PAGE_PRESENT)) {
337 /* No shadow entry: allocate a new shadow PTE page. */
338 unsigned long ptepage;
339
340 /* If they didn't want us to allocate anything, stop. */
341 if (!allocate)
342 return NULL;
343
344 ptepage = get_zeroed_page(GFP_KERNEL);
345
346 /*
347 * This is not really the Guest's fault, but killing it is
348 * simple for this corner case.
349 */
350 if (!ptepage) {
351 kill_guest(cpu, "out of memory allocating pmd page");
352 return NULL;
353 }
354
355 /*
356 * And we copy the flags to the shadow PMD entry. The page
357 * number in the shadow PMD is the page we just allocated.
358 */
359 set_pmd(spmd, __pmd(__pa(ptepage) | pmd_flags));
360 }
361#endif
362
363 /* Get the pointer to the shadow PTE entry we're going to set. */
364 return spte_addr(cpu, *spgd, vaddr);
365}
366
Rusty Russellbff672e2007-07-26 10:41:04 -0700367/*H:330
Rusty Russelle1e72962007-10-25 15:02:50 +1000368 * (i) Looking up a page table entry when the Guest faults.
Rusty Russellbff672e2007-07-26 10:41:04 -0700369 *
370 * We saw this call in run_guest(): when we see a page fault in the Guest, we
371 * come here. That's because we only set up the shadow page tables lazily as
372 * they're needed, so we get page faults all the time and quietly fix them up
373 * and return to the Guest without it knowing.
374 *
375 * If we fixed up the fault (ie. we mapped the address), this routine returns
Rusty Russell2e04ef72009-07-30 16:03:45 -0600376 * true. Otherwise, it was a real fault and we need to tell the Guest.
377 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300378bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700379{
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700380 unsigned long gpte_ptr;
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000381 pte_t gpte;
382 pte_t *spte;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600383 pmd_t gpmd;
Rusty Russell17427e02013-04-22 14:10:39 +0930384 pgd_t gpgd;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600385
Rusty Russell68a644d2013-04-22 14:10:37 +0930386 /* We never demand page the Switcher, so trying is a mistake. */
387 if (vaddr >= switcher_addr)
388 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700389
Rusty Russellbff672e2007-07-26 10:41:04 -0700390 /* First step: get the top-level Guest page table entry. */
Rusty Russell5dea1c82011-07-22 14:39:48 +0930391 if (unlikely(cpu->linear_pages)) {
392 /* Faking up a linear mapping. */
393 gpgd = __pgd(CHECK_GPGD_MASK);
394 } else {
395 gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
396 /* Toplevel not present? We can't map it in. */
397 if (!(pgd_flags(gpgd) & _PAGE_PRESENT))
398 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700399
Rusty Russell17427e02013-04-22 14:10:39 +0930400 /*
401 * This kills the Guest if it has weird flags or tries to
402 * refer to a "physical" address outside the bounds.
Rusty Russell2e04ef72009-07-30 16:03:45 -0600403 */
Rusty Russelle1d12602013-04-22 14:10:39 +0930404 if (!check_gpgd(cpu, gpgd))
405 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700406 }
407
Rusty Russell17427e02013-04-22 14:10:39 +0930408 /* This "mid-level" entry is only used for non-linear, PAE mode. */
409 gpmd = __pmd(_PAGE_TABLE);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700410
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600411#ifdef CONFIG_X86_PAE
Rusty Russell17427e02013-04-22 14:10:39 +0930412 if (likely(!cpu->linear_pages)) {
Rusty Russell5dea1c82011-07-22 14:39:48 +0930413 gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
414 /* Middle level not present? We can't map it in. */
415 if (!(pmd_flags(gpmd) & _PAGE_PRESENT))
416 return false;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600417
Rusty Russell17427e02013-04-22 14:10:39 +0930418 /*
419 * This kills the Guest if it has weird flags or tries to
420 * refer to a "physical" address outside the bounds.
Rusty Russell2e04ef72009-07-30 16:03:45 -0600421 */
Rusty Russelle1d12602013-04-22 14:10:39 +0930422 if (!check_gpmd(cpu, gpmd))
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600423 return false;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600424 }
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600425
Rusty Russell2e04ef72009-07-30 16:03:45 -0600426 /*
427 * OK, now we look at the lower level in the Guest page table: keep its
428 * address, because we might update it later.
429 */
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600430 gpte_ptr = gpte_addr(cpu, gpmd, vaddr);
431#else
Rusty Russell2e04ef72009-07-30 16:03:45 -0600432 /*
433 * OK, now we look at the lower level in the Guest page table: keep its
434 * address, because we might update it later.
435 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600436 gpte_ptr = gpte_addr(cpu, gpgd, vaddr);
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600437#endif
Rusty Russella91d74a2009-07-30 16:03:45 -0600438
Rusty Russell5dea1c82011-07-22 14:39:48 +0930439 if (unlikely(cpu->linear_pages)) {
440 /* Linear? Make up a PTE which points to same page. */
441 gpte = __pte((vaddr & PAGE_MASK) | _PAGE_RW | _PAGE_PRESENT);
442 } else {
443 /* Read the actual PTE value. */
444 gpte = lgread(cpu, gpte_ptr, pte_t);
445 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700446
Rusty Russellbff672e2007-07-26 10:41:04 -0700447 /* If this page isn't in the Guest page tables, we can't page it in. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000448 if (!(pte_flags(gpte) & _PAGE_PRESENT))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300449 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700450
Rusty Russell2e04ef72009-07-30 16:03:45 -0600451 /*
452 * Check they're not trying to write to a page the Guest wants
453 * read-only (bit 2 of errcode == write).
454 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000455 if ((errcode & 2) && !(pte_flags(gpte) & _PAGE_RW))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300456 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700457
Rusty Russelle1e72962007-10-25 15:02:50 +1000458 /* User access to a kernel-only page? (bit 3 == user access) */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000459 if ((errcode & 4) && !(pte_flags(gpte) & _PAGE_USER))
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300460 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700461
Rusty Russell2e04ef72009-07-30 16:03:45 -0600462 /*
463 * Check that the Guest PTE flags are OK, and the page number is below
464 * the pfn_limit (ie. not mapping the Launcher binary).
465 */
Rusty Russelle1d12602013-04-22 14:10:39 +0930466 if (!check_gpte(cpu, gpte))
467 return false;
Rusty Russelle1e72962007-10-25 15:02:50 +1000468
Rusty Russellbff672e2007-07-26 10:41:04 -0700469 /* Add the _PAGE_ACCESSED and (for a write) _PAGE_DIRTY flag */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000470 gpte = pte_mkyoung(gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700471 if (errcode & 2)
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000472 gpte = pte_mkdirty(gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700473
Rusty Russellbff672e2007-07-26 10:41:04 -0700474 /* Get the pointer to the shadow PTE entry we're going to set. */
Rusty Russell17427e02013-04-22 14:10:39 +0930475 spte = find_spte(cpu, vaddr, true, pgd_flags(gpgd), pmd_flags(gpmd));
476 if (!spte)
477 return false;
Rusty Russell2e04ef72009-07-30 16:03:45 -0600478
479 /*
480 * If there was a valid shadow PTE entry here before, we release it.
481 * This can happen with a write to a previously read-only entry.
482 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700483 release_pte(*spte);
484
Rusty Russell2e04ef72009-07-30 16:03:45 -0600485 /*
486 * If this is a write, we insist that the Guest page is writable (the
487 * final arg to gpte_to_spte()).
488 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000489 if (pte_dirty(gpte))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200490 *spte = gpte_to_spte(cpu, gpte, 1);
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000491 else
Rusty Russell2e04ef72009-07-30 16:03:45 -0600492 /*
493 * If this is a read, don't set the "writable" bit in the page
Rusty Russellbff672e2007-07-26 10:41:04 -0700494 * table entry, even if the Guest says it's writable. That way
Rusty Russelle1e72962007-10-25 15:02:50 +1000495 * we will come back here when a write does actually occur, so
Rusty Russell2e04ef72009-07-30 16:03:45 -0600496 * we can update the Guest's _PAGE_DIRTY flag.
497 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600498 set_pte(spte, gpte_to_spte(cpu, pte_wrprotect(gpte), 0));
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700499
Rusty Russell2e04ef72009-07-30 16:03:45 -0600500 /*
501 * Finally, we write the Guest PTE entry back: we've set the
502 * _PAGE_ACCESSED and maybe the _PAGE_DIRTY flags.
503 */
Rusty Russell5dea1c82011-07-22 14:39:48 +0930504 if (likely(!cpu->linear_pages))
505 lgwrite(cpu, gpte_ptr, pte_t, gpte);
Rusty Russellbff672e2007-07-26 10:41:04 -0700506
Rusty Russell2e04ef72009-07-30 16:03:45 -0600507 /*
508 * The fault is fixed, the page table is populated, the mapping
Rusty Russelle1e72962007-10-25 15:02:50 +1000509 * manipulated, the result returned and the code complete. A small
510 * delay and a trace of alliteration are the only indications the Guest
Rusty Russell2e04ef72009-07-30 16:03:45 -0600511 * has that a page fault occurred at all.
512 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300513 return true;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700514}
515
Rusty Russelle1e72962007-10-25 15:02:50 +1000516/*H:360
517 * (ii) Making sure the Guest stack is mapped.
Rusty Russellbff672e2007-07-26 10:41:04 -0700518 *
Rusty Russelle1e72962007-10-25 15:02:50 +1000519 * Remember that direct traps into the Guest need a mapped Guest kernel stack.
520 * pin_stack_pages() calls us here: we could simply call demand_page(), but as
521 * we've seen that logic is quite long, and usually the stack pages are already
522 * mapped, so it's overkill.
Rusty Russellbff672e2007-07-26 10:41:04 -0700523 *
524 * This is a quick version which answers the question: is this virtual address
Rusty Russell2e04ef72009-07-30 16:03:45 -0600525 * mapped by the shadow page tables, and is it writable?
526 */
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300527static bool page_writable(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700528{
Rusty Russell17427e02013-04-22 14:10:39 +0930529 pte_t *spte;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700530 unsigned long flags;
531
Rusty Russell68a644d2013-04-22 14:10:37 +0930532 /* You can't put your stack in the Switcher! */
533 if (vaddr >= switcher_addr)
Matias Zabaljaureguidf1693a2009-03-18 13:38:35 -0300534 return false;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700535
Rusty Russell17427e02013-04-22 14:10:39 +0930536 /* If there's no shadow PTE, it's not writable. */
537 spte = find_spte(cpu, vaddr, false, 0, 0);
538 if (!spte)
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600539 return false;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600540
Rusty Russell2e04ef72009-07-30 16:03:45 -0600541 /*
542 * Check the flags on the pte entry itself: it must be present and
543 * writable.
544 */
Rusty Russell17427e02013-04-22 14:10:39 +0930545 flags = pte_flags(*spte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700546 return (flags & (_PAGE_PRESENT|_PAGE_RW)) == (_PAGE_PRESENT|_PAGE_RW);
547}
548
Rusty Russell2e04ef72009-07-30 16:03:45 -0600549/*
550 * So, when pin_stack_pages() asks us to pin a page, we check if it's already
Rusty Russellbff672e2007-07-26 10:41:04 -0700551 * in the page tables, and if not, we call demand_page() with error code 2
Rusty Russell2e04ef72009-07-30 16:03:45 -0600552 * (meaning "write").
553 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200554void pin_page(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700555{
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200556 if (!page_writable(cpu, vaddr) && !demand_page(cpu, vaddr, 2))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200557 kill_guest(cpu, "bad stack page %#lx", vaddr);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700558}
Rusty Russella91d74a2009-07-30 16:03:45 -0600559/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700560
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600561#ifdef CONFIG_X86_PAE
562static void release_pmd(pmd_t *spmd)
563{
564 /* If the entry's not present, there's nothing to release. */
565 if (pmd_flags(*spmd) & _PAGE_PRESENT) {
566 unsigned int i;
567 pte_t *ptepage = __va(pmd_pfn(*spmd) << PAGE_SHIFT);
568 /* For each entry in the page, we might need to release it. */
569 for (i = 0; i < PTRS_PER_PTE; i++)
570 release_pte(ptepage[i]);
571 /* Now we can free the page of PTEs */
572 free_page((long)ptepage);
573 /* And zero out the PMD entry so we never release it twice. */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600574 set_pmd(spmd, __pmd(0));
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600575 }
576}
577
578static void release_pgd(pgd_t *spgd)
579{
580 /* If the entry's not present, there's nothing to release. */
581 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
582 unsigned int i;
583 pmd_t *pmdpage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
584
585 for (i = 0; i < PTRS_PER_PMD; i++)
586 release_pmd(&pmdpage[i]);
587
588 /* Now we can free the page of PMDs */
589 free_page((long)pmdpage);
590 /* And zero out the PGD entry so we never release it twice. */
591 set_pgd(spgd, __pgd(0));
592 }
593}
594
595#else /* !CONFIG_X86_PAE */
Rusty Russella91d74a2009-07-30 16:03:45 -0600596/*H:450
597 * If we chase down the release_pgd() code, the non-PAE version looks like
598 * this. The PAE version is almost identical, but instead of calling
599 * release_pte it calls release_pmd(), which looks much like this.
600 */
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600601static void release_pgd(pgd_t *spgd)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700602{
Rusty Russellbff672e2007-07-26 10:41:04 -0700603 /* If the entry's not present, there's nothing to release. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000604 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700605 unsigned int i;
Rusty Russell2e04ef72009-07-30 16:03:45 -0600606 /*
607 * Converting the pfn to find the actual PTE page is easy: turn
Rusty Russellbff672e2007-07-26 10:41:04 -0700608 * the page number into a physical address, then convert to a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600609 * virtual address (easy for kernel pages like this one).
610 */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000611 pte_t *ptepage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
Rusty Russellbff672e2007-07-26 10:41:04 -0700612 /* For each entry in the page, we might need to release it. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000613 for (i = 0; i < PTRS_PER_PTE; i++)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700614 release_pte(ptepage[i]);
Rusty Russellbff672e2007-07-26 10:41:04 -0700615 /* Now we can free the page of PTEs */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700616 free_page((long)ptepage);
Rusty Russelle1e72962007-10-25 15:02:50 +1000617 /* And zero out the PGD entry so we never release it twice. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000618 *spgd = __pgd(0);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700619 }
620}
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600621#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -0600622
623/*H:445
624 * We saw flush_user_mappings() twice: once from the flush_user_mappings()
Rusty Russelle1e72962007-10-25 15:02:50 +1000625 * hypercall and once in new_pgdir() when we re-used a top-level pgdir page.
Rusty Russell2e04ef72009-07-30 16:03:45 -0600626 * It simply releases every PTE page from 0 up to the Guest's kernel address.
627 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700628static void flush_user_mappings(struct lguest *lg, int idx)
629{
630 unsigned int i;
Rusty Russellbff672e2007-07-26 10:41:04 -0700631 /* Release every pgd entry up to the kernel's address. */
Rusty Russell47436aa2007-10-22 11:03:36 +1000632 for (i = 0; i < pgd_index(lg->kernel_address); i++)
Matias Zabaljauregui90603d12009-06-12 22:27:06 -0600633 release_pgd(lg->pgdirs[idx].pgdir + i);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700634}
635
Rusty Russell2e04ef72009-07-30 16:03:45 -0600636/*H:440
637 * (v) Flushing (throwing away) page tables,
Rusty Russelle1e72962007-10-25 15:02:50 +1000638 *
639 * The Guest has a hypercall to throw away the page tables: it's used when a
Rusty Russell2e04ef72009-07-30 16:03:45 -0600640 * large number of mappings have been changed.
641 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200642void guest_pagetable_flush_user(struct lg_cpu *cpu)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700643{
Rusty Russellbff672e2007-07-26 10:41:04 -0700644 /* Drop the userspace part of the current page table. */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200645 flush_user_mappings(cpu->lg, cpu->cpu_pgd);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700646}
Rusty Russellbff672e2007-07-26 10:41:04 -0700647/*:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700648
Rusty Russell47436aa2007-10-22 11:03:36 +1000649/* We walk down the guest page tables to get a guest-physical address */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200650unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr)
Rusty Russell47436aa2007-10-22 11:03:36 +1000651{
652 pgd_t gpgd;
653 pte_t gpte;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600654#ifdef CONFIG_X86_PAE
655 pmd_t gpmd;
656#endif
Rusty Russell5dea1c82011-07-22 14:39:48 +0930657
658 /* Still not set up? Just map 1:1. */
659 if (unlikely(cpu->linear_pages))
660 return vaddr;
661
Rusty Russell47436aa2007-10-22 11:03:36 +1000662 /* First step: get the top-level Guest page table entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200663 gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
Rusty Russell47436aa2007-10-22 11:03:36 +1000664 /* Toplevel not present? We can't map it in. */
Rusty Russell6afbdd02009-03-30 21:55:23 -0600665 if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200666 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell6afbdd02009-03-30 21:55:23 -0600667 return -1UL;
668 }
Rusty Russell47436aa2007-10-22 11:03:36 +1000669
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600670#ifdef CONFIG_X86_PAE
671 gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
Rusty Russell4623c282013-09-05 17:45:39 +0930672 if (!(pmd_flags(gpmd) & _PAGE_PRESENT)) {
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600673 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell4623c282013-09-05 17:45:39 +0930674 return -1UL;
675 }
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600676 gpte = lgread(cpu, gpte_addr(cpu, gpmd, vaddr), pte_t);
677#else
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600678 gpte = lgread(cpu, gpte_addr(cpu, gpgd, vaddr), pte_t);
Rusty Russell92b4d8d2009-06-12 22:27:08 -0600679#endif
Rusty Russell47436aa2007-10-22 11:03:36 +1000680 if (!(pte_flags(gpte) & _PAGE_PRESENT))
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200681 kill_guest(cpu, "Bad address %#lx", vaddr);
Rusty Russell47436aa2007-10-22 11:03:36 +1000682
683 return pte_pfn(gpte) * PAGE_SIZE | (vaddr & ~PAGE_MASK);
684}
685
Rusty Russell2e04ef72009-07-30 16:03:45 -0600686/*
687 * We keep several page tables. This is a simple routine to find the page
Rusty Russellbff672e2007-07-26 10:41:04 -0700688 * table (if any) corresponding to this top-level address the Guest has given
Rusty Russell2e04ef72009-07-30 16:03:45 -0600689 * us.
690 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700691static unsigned int find_pgdir(struct lguest *lg, unsigned long pgtable)
692{
693 unsigned int i;
694 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
Rusty Russell4357bd92008-03-11 09:35:57 -0500695 if (lg->pgdirs[i].pgdir && lg->pgdirs[i].gpgdir == pgtable)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700696 break;
697 return i;
698}
699
Rusty Russell2e04ef72009-07-30 16:03:45 -0600700/*H:435
701 * And this is us, creating the new page directory. If we really do
Rusty Russellbff672e2007-07-26 10:41:04 -0700702 * allocate a new one (and so the kernel parts are not there), we set
Rusty Russell2e04ef72009-07-30 16:03:45 -0600703 * blank_pgdir.
704 */
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200705static unsigned int new_pgdir(struct lg_cpu *cpu,
Rusty Russellee3db0f2007-10-22 11:03:34 +1000706 unsigned long gpgdir,
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700707 int *blank_pgdir)
708{
709 unsigned int next;
710
Rusty Russell2e04ef72009-07-30 16:03:45 -0600711 /*
712 * We pick one entry at random to throw out. Choosing the Least
713 * Recently Used might be better, but this is easy.
714 */
Akinobu Mita10fdc142013-04-29 16:21:34 -0700715 next = prandom_u32() % ARRAY_SIZE(cpu->lg->pgdirs);
Rusty Russellbff672e2007-07-26 10:41:04 -0700716 /* If it's never been allocated at all before, try now. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200717 if (!cpu->lg->pgdirs[next].pgdir) {
718 cpu->lg->pgdirs[next].pgdir =
719 (pgd_t *)get_zeroed_page(GFP_KERNEL);
Rusty Russellbff672e2007-07-26 10:41:04 -0700720 /* If the allocation fails, just keep using the one we have */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200721 if (!cpu->lg->pgdirs[next].pgdir)
Glauber de Oliveira Costa17136082008-01-07 11:05:37 -0200722 next = cpu->cpu_pgd;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600723 else {
Rusty Russell2e04ef72009-07-30 16:03:45 -0600724 /*
Rusty Russell3412b6a2013-04-22 14:10:40 +0930725 * This is a blank page, so there are no kernel
726 * mappings: caller must map the stack!
Rusty Russell2e04ef72009-07-30 16:03:45 -0600727 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700728 *blank_pgdir = 1;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600729 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700730 }
Rusty Russellbff672e2007-07-26 10:41:04 -0700731 /* Record which Guest toplevel this shadows. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200732 cpu->lg->pgdirs[next].gpgdir = gpgdir;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700733 /* Release all the non-kernel mappings. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200734 flush_user_mappings(cpu->lg, next);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700735
Rusty Russell6d0cda92013-04-22 14:10:41 +0930736 /* This hasn't run on any CPU at all. */
737 cpu->lg->pgdirs[next].last_host_cpu = -1;
738
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700739 return next;
740}
741
Rusty Russell3412b6a2013-04-22 14:10:40 +0930742/*H:501
743 * We do need the Switcher code mapped at all times, so we allocate that
Rusty Russell86935fc2013-04-22 14:10:41 +0930744 * part of the Guest page table here. We map the Switcher code immediately,
745 * but defer mapping of the guest register page and IDT/LDT etc page until
746 * just before we run the guest in map_switcher_in_guest().
747 *
748 * We *could* do this setup in map_switcher_in_guest(), but at that point
749 * we've interrupts disabled, and allocating pages like that is fraught: we
750 * can't sleep if we need to free up some memory.
Rusty Russell3412b6a2013-04-22 14:10:40 +0930751 */
752static bool allocate_switcher_mapping(struct lg_cpu *cpu)
753{
754 int i;
755
756 for (i = 0; i < TOTAL_SWITCHER_PAGES; i++) {
Rusty Russell86935fc2013-04-22 14:10:41 +0930757 pte_t *pte = find_spte(cpu, switcher_addr + i * PAGE_SIZE, true,
758 CHECK_GPGD_MASK, _PAGE_TABLE);
759 if (!pte)
Rusty Russell3412b6a2013-04-22 14:10:40 +0930760 return false;
Rusty Russell86935fc2013-04-22 14:10:41 +0930761
762 /*
763 * Map the switcher page if not already there. It might
764 * already be there because we call allocate_switcher_mapping()
765 * in guest_set_pgd() just in case it did discard our Switcher
766 * mapping, but it probably didn't.
767 */
768 if (i == 0 && !(pte_flags(*pte) & _PAGE_PRESENT)) {
769 /* Get a reference to the Switcher page. */
770 get_page(lg_switcher_pages[0]);
771 /* Create a read-only, exectuable, kernel-style PTE */
772 set_pte(pte,
773 mk_pte(lg_switcher_pages[0], PAGE_KERNEL_RX));
774 }
Rusty Russell3412b6a2013-04-22 14:10:40 +0930775 }
Rusty Russell86935fc2013-04-22 14:10:41 +0930776 cpu->lg->pgdirs[cpu->cpu_pgd].switcher_mapped = true;
Rusty Russell3412b6a2013-04-22 14:10:40 +0930777 return true;
778}
779
Rusty Russell2e04ef72009-07-30 16:03:45 -0600780/*H:470
781 * Finally, a routine which throws away everything: all PGD entries in all
Rusty Russelle1e72962007-10-25 15:02:50 +1000782 * the shadow page tables, including the Guest's kernel mappings. This is used
Rusty Russell2e04ef72009-07-30 16:03:45 -0600783 * when we destroy the Guest.
784 */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700785static void release_all_pagetables(struct lguest *lg)
786{
787 unsigned int i, j;
788
Rusty Russellbff672e2007-07-26 10:41:04 -0700789 /* Every shadow pagetable this Guest has */
Rusty Russell3412b6a2013-04-22 14:10:40 +0930790 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++) {
791 if (!lg->pgdirs[i].pgdir)
792 continue;
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600793
Rusty Russell3412b6a2013-04-22 14:10:40 +0930794 /* Every PGD entry. */
795 for (j = 0; j < PTRS_PER_PGD; j++)
796 release_pgd(lg->pgdirs[i].pgdir + j);
Rusty Russell86935fc2013-04-22 14:10:41 +0930797 lg->pgdirs[i].switcher_mapped = false;
Rusty Russell6d0cda92013-04-22 14:10:41 +0930798 lg->pgdirs[i].last_host_cpu = -1;
Rusty Russell3412b6a2013-04-22 14:10:40 +0930799 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700800}
801
Rusty Russell2e04ef72009-07-30 16:03:45 -0600802/*
803 * We also throw away everything when a Guest tells us it's changed a kernel
Rusty Russellbff672e2007-07-26 10:41:04 -0700804 * mapping. Since kernel mappings are in every page table, it's easiest to
Rusty Russelle1e72962007-10-25 15:02:50 +1000805 * throw them all away. This traps the Guest in amber for a while as
Rusty Russell2e04ef72009-07-30 16:03:45 -0600806 * everything faults back in, but it's rare.
807 */
Glauber de Oliveira Costa4665ac8e2008-01-07 11:05:35 -0200808void guest_pagetable_clear_all(struct lg_cpu *cpu)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700809{
Glauber de Oliveira Costa4665ac8e2008-01-07 11:05:35 -0200810 release_all_pagetables(cpu->lg);
Rusty Russellbff672e2007-07-26 10:41:04 -0700811 /* We need the Guest kernel stack mapped again. */
Glauber de Oliveira Costa4665ac8e2008-01-07 11:05:35 -0200812 pin_stack_pages(cpu);
Rusty Russell3412b6a2013-04-22 14:10:40 +0930813 /* And we need Switcher allocated. */
814 if (!allocate_switcher_mapping(cpu))
815 kill_guest(cpu, "Cannot populate switcher mapping");
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700816}
Rusty Russell5dea1c82011-07-22 14:39:48 +0930817
818/*H:430
819 * (iv) Switching page tables
820 *
821 * Now we've seen all the page table setting and manipulation, let's see
822 * what happens when the Guest changes page tables (ie. changes the top-level
823 * pgdir). This occurs on almost every context switch.
824 */
825void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable)
826{
827 int newpgdir, repin = 0;
828
829 /*
830 * The very first time they call this, we're actually running without
831 * any page tables; we've been making it up. Throw them away now.
832 */
833 if (unlikely(cpu->linear_pages)) {
834 release_all_pagetables(cpu->lg);
835 cpu->linear_pages = false;
836 /* Force allocation of a new pgdir. */
837 newpgdir = ARRAY_SIZE(cpu->lg->pgdirs);
838 } else {
839 /* Look to see if we have this one already. */
840 newpgdir = find_pgdir(cpu->lg, pgtable);
841 }
842
843 /*
844 * If not, we allocate or mug an existing one: if it's a fresh one,
845 * repin gets set to 1.
846 */
847 if (newpgdir == ARRAY_SIZE(cpu->lg->pgdirs))
848 newpgdir = new_pgdir(cpu, pgtable, &repin);
849 /* Change the current pgd index to the new one. */
850 cpu->cpu_pgd = newpgdir;
Rusty Russell3412b6a2013-04-22 14:10:40 +0930851 /*
852 * If it was completely blank, we map in the Guest kernel stack and
853 * the Switcher.
854 */
Rusty Russell5dea1c82011-07-22 14:39:48 +0930855 if (repin)
856 pin_stack_pages(cpu);
Rusty Russell3412b6a2013-04-22 14:10:40 +0930857
Rusty Russell86935fc2013-04-22 14:10:41 +0930858 if (!cpu->lg->pgdirs[cpu->cpu_pgd].switcher_mapped) {
859 if (!allocate_switcher_mapping(cpu))
860 kill_guest(cpu, "Cannot populate switcher mapping");
861 }
Rusty Russell5dea1c82011-07-22 14:39:48 +0930862}
Rusty Russelle1e72962007-10-25 15:02:50 +1000863/*:*/
Rusty Russell2e04ef72009-07-30 16:03:45 -0600864
865/*M:009
866 * Since we throw away all mappings when a kernel mapping changes, our
Rusty Russelle1e72962007-10-25 15:02:50 +1000867 * performance sucks for guests using highmem. In fact, a guest with
868 * PAGE_OFFSET 0xc0000000 (the default) and more than about 700MB of RAM is
869 * usually slower than a Guest with less memory.
870 *
871 * This, of course, cannot be fixed. It would take some kind of... well, I
Rusty Russell2e04ef72009-07-30 16:03:45 -0600872 * don't know, but the term "puissant code-fu" comes to mind.
873:*/
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700874
Rusty Russell2e04ef72009-07-30 16:03:45 -0600875/*H:420
876 * This is the routine which actually sets the page table entry for then
Rusty Russellbff672e2007-07-26 10:41:04 -0700877 * "idx"'th shadow page table.
878 *
879 * Normally, we can just throw out the old entry and replace it with 0: if they
880 * use it demand_page() will put the new entry in. We need to do this anyway:
881 * The Guest expects _PAGE_ACCESSED to be set on its PTE the first time a page
882 * is read from, and _PAGE_DIRTY when it's written to.
883 *
884 * But Avi Kivity pointed out that most Operating Systems (Linux included) set
885 * these bits on PTEs immediately anyway. This is done to save the CPU from
886 * having to update them, but it helps us the same way: if they set
887 * _PAGE_ACCESSED then we can put a read-only PTE entry in immediately, and if
888 * they set _PAGE_DIRTY then we can put a writable PTE entry in immediately.
889 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200890static void do_set_pte(struct lg_cpu *cpu, int idx,
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000891 unsigned long vaddr, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700892{
Rusty Russelle1e72962007-10-25 15:02:50 +1000893 /* Look up the matching shadow page directory entry. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200894 pgd_t *spgd = spgd_addr(cpu, idx, vaddr);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600895#ifdef CONFIG_X86_PAE
896 pmd_t *spmd;
897#endif
Rusty Russellbff672e2007-07-26 10:41:04 -0700898
899 /* If the top level isn't present, there's no entry to update. */
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +1000900 if (pgd_flags(*spgd) & _PAGE_PRESENT) {
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600901#ifdef CONFIG_X86_PAE
902 spmd = spmd_addr(cpu, *spgd, vaddr);
903 if (pmd_flags(*spmd) & _PAGE_PRESENT) {
904#endif
Rusty Russell2e04ef72009-07-30 16:03:45 -0600905 /* Otherwise, start by releasing the existing entry. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600906 pte_t *spte = spte_addr(cpu, *spgd, vaddr);
907 release_pte(*spte);
Rusty Russellbff672e2007-07-26 10:41:04 -0700908
Rusty Russell2e04ef72009-07-30 16:03:45 -0600909 /*
910 * If they're setting this entry as dirty or accessed,
911 * we might as well put that entry they've given us in
912 * now. This shaves 10% off a copy-on-write
913 * micro-benchmark.
914 */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600915 if (pte_flags(gpte) & (_PAGE_DIRTY | _PAGE_ACCESSED)) {
Rusty Russelle1d12602013-04-22 14:10:39 +0930916 if (!check_gpte(cpu, gpte))
917 return;
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600918 set_pte(spte,
919 gpte_to_spte(cpu, gpte,
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600920 pte_flags(gpte) & _PAGE_DIRTY));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600921 } else {
922 /*
923 * Otherwise kill it and we can demand_page()
924 * it in later.
925 */
Rusty Russell4c1ea3d2009-09-23 22:26:45 -0600926 set_pte(spte, __pte(0));
Rusty Russell2e04ef72009-07-30 16:03:45 -0600927 }
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -0600928#ifdef CONFIG_X86_PAE
929 }
930#endif
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700931 }
932}
933
Rusty Russell2e04ef72009-07-30 16:03:45 -0600934/*H:410
935 * Updating a PTE entry is a little trickier.
Rusty Russellbff672e2007-07-26 10:41:04 -0700936 *
937 * We keep track of several different page tables (the Guest uses one for each
938 * process, so it makes sense to cache at least a few). Each of these have
939 * identical kernel parts: ie. every mapping above PAGE_OFFSET is the same for
940 * all processes. So when the page table above that address changes, we update
941 * all the page tables, not just the current one. This is rare.
942 *
Rusty Russella6bd8e12008-03-28 11:05:53 -0500943 * The benefit is that when we have to track a new page table, we can keep all
Rusty Russell2e04ef72009-07-30 16:03:45 -0600944 * the kernel mappings. This speeds up context switch immensely.
945 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200946void guest_set_pte(struct lg_cpu *cpu,
Rusty Russellee3db0f2007-10-22 11:03:34 +1000947 unsigned long gpgdir, unsigned long vaddr, pte_t gpte)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700948{
Rusty Russell68a644d2013-04-22 14:10:37 +0930949 /* We don't let you remap the Switcher; we need it to get back! */
950 if (vaddr >= switcher_addr) {
951 kill_guest(cpu, "attempt to set pte into Switcher pages");
952 return;
953 }
954
Rusty Russell2e04ef72009-07-30 16:03:45 -0600955 /*
956 * Kernel mappings must be changed on all top levels. Slow, but doesn't
957 * happen often.
958 */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200959 if (vaddr >= cpu->lg->kernel_address) {
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700960 unsigned int i;
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200961 for (i = 0; i < ARRAY_SIZE(cpu->lg->pgdirs); i++)
962 if (cpu->lg->pgdirs[i].pgdir)
963 do_set_pte(cpu, i, vaddr, gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700964 } else {
Rusty Russellbff672e2007-07-26 10:41:04 -0700965 /* Is this page table one we have a shadow for? */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200966 int pgdir = find_pgdir(cpu->lg, gpgdir);
967 if (pgdir != ARRAY_SIZE(cpu->lg->pgdirs))
Rusty Russellbff672e2007-07-26 10:41:04 -0700968 /* If so, do the update. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -0200969 do_set_pte(cpu, pgdir, vaddr, gpte);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700970 }
971}
972
Rusty Russellbff672e2007-07-26 10:41:04 -0700973/*H:400
Rusty Russelle1e72962007-10-25 15:02:50 +1000974 * (iii) Setting up a page table entry when the Guest tells us one has changed.
Rusty Russellbff672e2007-07-26 10:41:04 -0700975 *
976 * Just like we did in interrupts_and_traps.c, it makes sense for us to deal
977 * with the other side of page tables while we're here: what happens when the
978 * Guest asks for a page table to be updated?
979 *
980 * We already saw that demand_page() will fill in the shadow page tables when
981 * needed, so we can simply remove shadow page table entries whenever the Guest
982 * tells us they've changed. When the Guest tries to use the new entry it will
983 * fault and demand_page() will fix it up.
984 *
Anand Gadiyarfd589a82009-07-16 17:13:03 +0200985 * So with that in mind here's our code to update a (top-level) PGD entry:
Rusty Russellbff672e2007-07-26 10:41:04 -0700986 */
Matias Zabaljaureguiebe0ba82009-05-30 15:48:08 -0300987void guest_set_pgd(struct lguest *lg, unsigned long gpgdir, u32 idx)
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700988{
989 int pgdir;
990
Rusty Russell3412b6a2013-04-22 14:10:40 +0930991 if (idx > PTRS_PER_PGD) {
992 kill_guest(&lg->cpus[0], "Attempt to set pgd %u/%u",
993 idx, PTRS_PER_PGD);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700994 return;
Rusty Russell3412b6a2013-04-22 14:10:40 +0930995 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700996
Rusty Russellbff672e2007-07-26 10:41:04 -0700997 /* If they're talking about a page table we have a shadow for... */
Rusty Russellee3db0f2007-10-22 11:03:34 +1000998 pgdir = find_pgdir(lg, gpgdir);
Rusty Russell3412b6a2013-04-22 14:10:40 +0930999 if (pgdir < ARRAY_SIZE(lg->pgdirs)) {
Rusty Russellbff672e2007-07-26 10:41:04 -07001000 /* ... throw it away. */
Matias Zabaljauregui90603d12009-06-12 22:27:06 -06001001 release_pgd(lg->pgdirs[pgdir].pgdir + idx);
Rusty Russell3412b6a2013-04-22 14:10:40 +09301002 /* That might have been the Switcher mapping, remap it. */
1003 if (!allocate_switcher_mapping(&lg->cpus[0])) {
1004 kill_guest(&lg->cpus[0],
1005 "Cannot populate switcher mapping");
1006 }
Rusty Russellf616fe42013-05-08 10:06:55 +09301007 lg->pgdirs[pgdir].last_host_cpu = -1;
Rusty Russell3412b6a2013-04-22 14:10:40 +09301008 }
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001009}
Rusty Russella91d74a2009-07-30 16:03:45 -06001010
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001011#ifdef CONFIG_X86_PAE
Rusty Russella91d74a2009-07-30 16:03:45 -06001012/* For setting a mid-level, we just throw everything away. It's easy. */
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001013void guest_set_pmd(struct lguest *lg, unsigned long pmdp, u32 idx)
1014{
1015 guest_pagetable_clear_all(&lg->cpus[0]);
1016}
1017#endif
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001018
Rusty Russell2e04ef72009-07-30 16:03:45 -06001019/*H:500
1020 * (vii) Setting up the page tables initially.
Rusty Russellbff672e2007-07-26 10:41:04 -07001021 *
Rusty Russell5dea1c82011-07-22 14:39:48 +09301022 * When a Guest is first created, set initialize a shadow page table which
1023 * we will populate on future faults. The Guest doesn't have any actual
1024 * pagetables yet, so we set linear_pages to tell demand_page() to fake it
1025 * for the moment.
Rusty Russell3412b6a2013-04-22 14:10:40 +09301026 *
1027 * We do need the Switcher to be mapped at all times, so we allocate that
1028 * part of the Guest page table here.
Rusty Russell2e04ef72009-07-30 16:03:45 -06001029 */
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001030int init_guest_pagetable(struct lguest *lg)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001031{
Rusty Russell5dea1c82011-07-22 14:39:48 +09301032 struct lg_cpu *cpu = &lg->cpus[0];
1033 int allocated = 0;
Matias Zabaljauregui58a24562008-09-29 01:40:07 -03001034
Rusty Russell5dea1c82011-07-22 14:39:48 +09301035 /* lg (and lg->cpus[]) starts zeroed: this allocates a new pgdir */
1036 cpu->cpu_pgd = new_pgdir(cpu, 0, &allocated);
1037 if (!allocated)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001038 return -ENOMEM;
Rusty Russella91d74a2009-07-30 16:03:45 -06001039
Rusty Russell5dea1c82011-07-22 14:39:48 +09301040 /* We start with a linear mapping until the initialize. */
1041 cpu->linear_pages = true;
Rusty Russell3412b6a2013-04-22 14:10:40 +09301042
1043 /* Allocate the page tables for the Switcher. */
1044 if (!allocate_switcher_mapping(cpu)) {
1045 release_all_pagetables(lg);
1046 return -ENOMEM;
1047 }
1048
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001049 return 0;
1050}
1051
Rusty Russella91d74a2009-07-30 16:03:45 -06001052/*H:508 When the Guest calls LHCALL_LGUEST_INIT we do more setup. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001053void page_table_guest_data_init(struct lg_cpu *cpu)
Rusty Russell47436aa2007-10-22 11:03:36 +10001054{
Rusty Russellc215a8b2013-04-22 14:10:37 +09301055 /*
1056 * We tell the Guest that it can't use the virtual addresses
1057 * used by the Switcher. This trick is equivalent to 4GB -
1058 * switcher_addr.
1059 */
1060 u32 top = ~switcher_addr + 1;
1061
Rusty Russell47436aa2007-10-22 11:03:36 +10001062 /* We get the kernel address: above this is all kernel memory. */
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001063 if (get_user(cpu->lg->kernel_address,
Rusty Russellc215a8b2013-04-22 14:10:37 +09301064 &cpu->lg->lguest_data->kernel_address)
Rusty Russell2e04ef72009-07-30 16:03:45 -06001065 /*
Rusty Russellc215a8b2013-04-22 14:10:37 +09301066 * We tell the Guest that it can't use the top virtual
1067 * addresses (used by the Switcher).
Rusty Russell2e04ef72009-07-30 16:03:45 -06001068 */
Rusty Russellc215a8b2013-04-22 14:10:37 +09301069 || put_user(top, &cpu->lg->lguest_data->reserve_mem)) {
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001070 kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data);
Rusty Russell5dea1c82011-07-22 14:39:48 +09301071 return;
1072 }
Rusty Russell47436aa2007-10-22 11:03:36 +10001073
Rusty Russell2e04ef72009-07-30 16:03:45 -06001074 /*
1075 * In flush_user_mappings() we loop from 0 to
Rusty Russell47436aa2007-10-22 11:03:36 +10001076 * "pgd_index(lg->kernel_address)". This assumes it won't hit the
Rusty Russell2e04ef72009-07-30 16:03:45 -06001077 * Switcher mappings, so check that now.
1078 */
Rusty Russell68a644d2013-04-22 14:10:37 +09301079 if (cpu->lg->kernel_address >= switcher_addr)
Glauber de Oliveira Costa382ac6b2008-01-17 19:19:42 -02001080 kill_guest(cpu, "bad kernel address %#lx",
1081 cpu->lg->kernel_address);
Rusty Russell47436aa2007-10-22 11:03:36 +10001082}
1083
Rusty Russellbff672e2007-07-26 10:41:04 -07001084/* When a Guest dies, our cleanup is fairly simple. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001085void free_guest_pagetable(struct lguest *lg)
1086{
1087 unsigned int i;
1088
Rusty Russellbff672e2007-07-26 10:41:04 -07001089 /* Throw away all page table pages. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001090 release_all_pagetables(lg);
Rusty Russellbff672e2007-07-26 10:41:04 -07001091 /* Now free the top levels: free_page() can handle 0 just fine. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001092 for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
1093 free_page((long)lg->pgdirs[i].pgdir);
1094}
1095
Rusty Russell6d0cda92013-04-22 14:10:41 +09301096/*H:481
1097 * This clears the Switcher mappings for cpu #i.
1098 */
1099static void remove_switcher_percpu_map(struct lg_cpu *cpu, unsigned int i)
1100{
1101 unsigned long base = switcher_addr + PAGE_SIZE + i * PAGE_SIZE*2;
1102 pte_t *pte;
1103
1104 /* Clear the mappings for both pages. */
1105 pte = find_spte(cpu, base, false, 0, 0);
1106 release_pte(*pte);
1107 set_pte(pte, __pte(0));
1108
1109 pte = find_spte(cpu, base + PAGE_SIZE, false, 0, 0);
1110 release_pte(*pte);
1111 set_pte(pte, __pte(0));
1112}
1113
Rusty Russell2e04ef72009-07-30 16:03:45 -06001114/*H:480
1115 * (vi) Mapping the Switcher when the Guest is about to run.
Rusty Russellbff672e2007-07-26 10:41:04 -07001116 *
Rusty Russell6d0cda92013-04-22 14:10:41 +09301117 * The Switcher and the two pages for this CPU need to be visible in the Guest
1118 * (and not the pages for other CPUs).
Rusty Russell3412b6a2013-04-22 14:10:40 +09301119 *
Rusty Russell6d0cda92013-04-22 14:10:41 +09301120 * The pages for the pagetables have all been allocated before: we just need
1121 * to make sure the actual PTEs are up-to-date for the CPU we're about to run
1122 * on.
Rusty Russell2e04ef72009-07-30 16:03:45 -06001123 */
Glauber de Oliveira Costa0c784412008-01-07 11:05:30 -02001124void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages)
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001125{
Rusty Russell6d0cda92013-04-22 14:10:41 +09301126 unsigned long base;
Rusty Russell3412b6a2013-04-22 14:10:40 +09301127 struct page *percpu_switcher_page, *regs_page;
1128 pte_t *pte;
Rusty Russell6d0cda92013-04-22 14:10:41 +09301129 struct pgdir *pgdir = &cpu->lg->pgdirs[cpu->cpu_pgd];
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001130
Rusty Russell6d0cda92013-04-22 14:10:41 +09301131 /* Switcher page should always be mapped by now! */
1132 BUG_ON(!pgdir->switcher_mapped);
Matias Zabaljaureguiacdd0b62009-06-12 22:27:07 -06001133
Rusty Russell6d0cda92013-04-22 14:10:41 +09301134 /*
1135 * Remember that we have two pages for each Host CPU, so we can run a
1136 * Guest on each CPU without them interfering. We need to make sure
1137 * those pages are mapped correctly in the Guest, but since we usually
1138 * run on the same CPU, we cache that, and only update the mappings
1139 * when we move.
Rusty Russell2e04ef72009-07-30 16:03:45 -06001140 */
Rusty Russell6d0cda92013-04-22 14:10:41 +09301141 if (pgdir->last_host_cpu == raw_smp_processor_id())
1142 return;
Matias Zabaljaureguidf29f432007-10-22 11:03:33 +10001143
Rusty Russell6d0cda92013-04-22 14:10:41 +09301144 /* -1 means unknown so we remove everything. */
1145 if (pgdir->last_host_cpu == -1) {
1146 unsigned int i;
1147 for_each_possible_cpu(i)
1148 remove_switcher_percpu_map(cpu, i);
1149 } else {
1150 /* We know exactly what CPU mapping to remove. */
1151 remove_switcher_percpu_map(cpu, pgdir->last_host_cpu);
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001152 }
1153
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001154 /*
Rusty Russell3412b6a2013-04-22 14:10:40 +09301155 * When we're running the Guest, we want the Guest's "regs" page to
1156 * appear where the first Switcher page for this CPU is. This is an
1157 * optimization: when the Switcher saves the Guest registers, it saves
1158 * them into the first page of this CPU's "struct lguest_pages": if we
1159 * make sure the Guest's register page is already mapped there, we
1160 * don't have to copy them out again.
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001161 */
Rusty Russell3412b6a2013-04-22 14:10:40 +09301162 /* Find the shadow PTE for this regs page. */
1163 base = switcher_addr + PAGE_SIZE
1164 + raw_smp_processor_id() * sizeof(struct lguest_pages);
1165 pte = find_spte(cpu, base, false, 0, 0);
1166 regs_page = pfn_to_page(__pa(cpu->regs_page) >> PAGE_SHIFT);
1167 get_page(regs_page);
1168 set_pte(pte, mk_pte(regs_page, __pgprot(__PAGE_KERNEL & ~_PAGE_GLOBAL)));
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001169
Rusty Russell2e04ef72009-07-30 16:03:45 -06001170 /*
Rusty Russell3412b6a2013-04-22 14:10:40 +09301171 * We map the second page of the struct lguest_pages read-only in
1172 * the Guest: the IDT, GDT and other things it's not supposed to
1173 * change.
Rusty Russell2e04ef72009-07-30 16:03:45 -06001174 */
Rusty Russell6d0cda92013-04-22 14:10:41 +09301175 pte = find_spte(cpu, base + PAGE_SIZE, false, 0, 0);
Rusty Russell3412b6a2013-04-22 14:10:40 +09301176 percpu_switcher_page
1177 = lg_switcher_pages[1 + raw_smp_processor_id()*2 + 1];
1178 get_page(percpu_switcher_page);
1179 set_pte(pte, mk_pte(percpu_switcher_page,
1180 __pgprot(__PAGE_KERNEL_RO & ~_PAGE_GLOBAL)));
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001181
Rusty Russell6d0cda92013-04-22 14:10:41 +09301182 pgdir->last_host_cpu = raw_smp_processor_id();
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001183}
1184
Rusty Russell6d0cda92013-04-22 14:10:41 +09301185/*H:490
Rusty Russell2e04ef72009-07-30 16:03:45 -06001186 * We've made it through the page table code. Perhaps our tired brains are
Rusty Russelld7e28ff2007-07-19 01:49:23 -07001187 * still processing the details, or perhaps we're simply glad it's over.
1188 *
1189 * If nothing else, note that all this complexity in juggling shadow page tables
1190 * in sync with the Guest's page tables is for one reason: for most Guests this
1191 * page table dance determines how bad performance will be. This is why Xen
1192 * uses exotic direct Guest pagetable manipulation, and why both Intel and AMD
1193 * have implemented shadow page table support directly into hardware.
1194 *
Rusty Russell2e04ef72009-07-30 16:03:45 -06001195 * There is just one file remaining in the Host.
1196 */