blob: 413b19b3d0fe5322ebe5a492d432eb132304aadf [file] [log] [blame]
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001/*
2 * Xen mmu operations
3 *
4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
7 *
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
12 *
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
16 * use.
17 *
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
23 *
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
29 * pagetable.
30 *
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
38 *
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
40 */
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -070041#include <linux/sched.h>
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070042#include <linux/highmem.h>
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070043#include <linux/debugfs.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070044#include <linux/bug.h>
Randy Dunlap44408ad2009-05-12 13:31:40 -070045#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090046#include <linux/gfp.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070047
48#include <asm/pgtable.h>
49#include <asm/tlbflush.h>
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -070050#include <asm/fixmap.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070051#include <asm/mmu_context.h>
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -080052#include <asm/setup.h>
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070053#include <asm/paravirt.h>
Jeremy Fitzhardingecbcd79c2008-07-08 15:06:27 -070054#include <asm/linkage.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070055
56#include <asm/xen/hypercall.h>
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070057#include <asm/xen/hypervisor.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070058
59#include <xen/page.h>
60#include <xen/interface/xen.h>
Stefano Stabellini59151002010-06-17 14:22:52 +010061#include <xen/interface/hvm/hvm_op.h>
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -080062#include <xen/interface/version.h>
63#include <xen/hvc-console.h>
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070064
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -070065#include "multicalls.h"
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -070066#include "mmu.h"
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070067#include "debugfs.h"
68
69#define MMU_UPDATE_HISTO 30
70
71#ifdef CONFIG_XEN_DEBUG_FS
72
73static struct {
74 u32 pgd_update;
75 u32 pgd_update_pinned;
76 u32 pgd_update_batched;
77
78 u32 pud_update;
79 u32 pud_update_pinned;
80 u32 pud_update_batched;
81
82 u32 pmd_update;
83 u32 pmd_update_pinned;
84 u32 pmd_update_batched;
85
86 u32 pte_update;
87 u32 pte_update_pinned;
88 u32 pte_update_batched;
89
90 u32 mmu_update;
91 u32 mmu_update_extended;
92 u32 mmu_update_histo[MMU_UPDATE_HISTO];
93
94 u32 prot_commit;
95 u32 prot_commit_batched;
96
97 u32 set_pte_at;
98 u32 set_pte_at_batched;
99 u32 set_pte_at_pinned;
100 u32 set_pte_at_current;
101 u32 set_pte_at_kernel;
102} mmu_stats;
103
104static u8 zero_stats;
105
106static inline void check_zero(void)
107{
108 if (unlikely(zero_stats)) {
109 memset(&mmu_stats, 0, sizeof(mmu_stats));
110 zero_stats = 0;
111 }
112}
113
114#define ADD_STATS(elem, val) \
115 do { check_zero(); mmu_stats.elem += (val); } while(0)
116
117#else /* !CONFIG_XEN_DEBUG_FS */
118
119#define ADD_STATS(elem, val) do { (void)(val); } while(0)
120
121#endif /* CONFIG_XEN_DEBUG_FS */
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700122
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -0800123
124/*
125 * Identity map, in addition to plain kernel map. This needs to be
126 * large enough to allocate page table pages to allocate the rest.
127 * Each page can map 2MB.
128 */
129static pte_t level1_ident_pgt[PTRS_PER_PTE * 4] __page_aligned_bss;
130
131#ifdef CONFIG_X86_64
132/* l3 pud for userspace vsyscall mapping */
133static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
134#endif /* CONFIG_X86_64 */
135
136/*
137 * Note about cr3 (pagetable base) values:
138 *
139 * xen_cr3 contains the current logical cr3 value; it contains the
140 * last set cr3. This may not be the current effective cr3, because
141 * its update may be being lazily deferred. However, a vcpu looking
142 * at its own cr3 can use this value knowing that it everything will
143 * be self-consistent.
144 *
145 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
146 * hypercall to set the vcpu cr3 is complete (so it may be a little
147 * out of date, but it will never be set early). If one vcpu is
148 * looking at another vcpu's cr3 value, it should use this variable.
149 */
150DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
151DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
152
153
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700154/*
155 * Just beyond the highest usermode address. STACK_TOP_MAX has a
156 * redzone above it, so round it up to a PGD boundary.
157 */
158#define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
159
160
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100161#define P2M_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
Jeremy Fitzhardingecf0923e2008-05-26 23:31:20 +0100162#define TOP_ENTRIES (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100163
Jeremy Fitzhardingecf0923e2008-05-26 23:31:20 +0100164/* Placeholder for holes in the address space */
Jeremy Fitzhardingecbcd79c2008-07-08 15:06:27 -0700165static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE] __page_aligned_data =
Jeremy Fitzhardingecf0923e2008-05-26 23:31:20 +0100166 { [ 0 ... P2M_ENTRIES_PER_PAGE-1 ] = ~0UL };
167
168 /* Array of pointers to pages containing p2m entries */
Jeremy Fitzhardingecbcd79c2008-07-08 15:06:27 -0700169static unsigned long *p2m_top[TOP_ENTRIES] __page_aligned_data =
Jeremy Fitzhardingecf0923e2008-05-26 23:31:20 +0100170 { [ 0 ... TOP_ENTRIES - 1] = &p2m_missing[0] };
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100171
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100172/* Arrays of p2m arrays expressed in mfns used for save/restore */
Jeremy Fitzhardingecbcd79c2008-07-08 15:06:27 -0700173static unsigned long p2m_top_mfn[TOP_ENTRIES] __page_aligned_bss;
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100174
Jeremy Fitzhardingecbcd79c2008-07-08 15:06:27 -0700175static unsigned long p2m_top_mfn_list[TOP_ENTRIES / P2M_ENTRIES_PER_PAGE]
176 __page_aligned_bss;
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100177
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100178static inline unsigned p2m_top_index(unsigned long pfn)
179{
Jeremy Fitzhardinge8006ec32008-05-26 23:31:19 +0100180 BUG_ON(pfn >= MAX_DOMAIN_PAGES);
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100181 return pfn / P2M_ENTRIES_PER_PAGE;
182}
183
184static inline unsigned p2m_index(unsigned long pfn)
185{
186 return pfn % P2M_ENTRIES_PER_PAGE;
187}
188
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100189/* Build the parallel p2m_top_mfn structures */
Ian Campbellfa24ba62009-11-21 11:32:49 +0000190void xen_build_mfn_list_list(void)
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100191{
192 unsigned pfn, idx;
193
Tejf63c2f22008-12-16 11:56:06 -0800194 for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100195 unsigned topidx = p2m_top_index(pfn);
196
197 p2m_top_mfn[topidx] = virt_to_mfn(p2m_top[topidx]);
198 }
199
Tejf63c2f22008-12-16 11:56:06 -0800200 for (idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100201 unsigned topidx = idx * P2M_ENTRIES_PER_PAGE;
202 p2m_top_mfn_list[idx] = virt_to_mfn(&p2m_top_mfn[topidx]);
203 }
Jeremy Fitzhardingecdaead62009-02-27 15:34:59 -0800204}
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100205
Jeremy Fitzhardingecdaead62009-02-27 15:34:59 -0800206void xen_setup_mfn_list_list(void)
207{
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100208 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
209
210 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
211 virt_to_mfn(p2m_top_mfn_list);
212 HYPERVISOR_shared_info->arch.max_pfn = xen_start_info->nr_pages;
213}
214
215/* Set up p2m_top to point to the domain-builder provided p2m pages */
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100216void __init xen_build_dynamic_phys_to_machine(void)
217{
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100218 unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
Jeremy Fitzhardinge8006ec32008-05-26 23:31:19 +0100219 unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100220 unsigned pfn;
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100221
Tejf63c2f22008-12-16 11:56:06 -0800222 for (pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100223 unsigned topidx = p2m_top_index(pfn);
224
225 p2m_top[topidx] = &mfn_list[pfn];
226 }
Jeremy Fitzhardingecdaead62009-02-27 15:34:59 -0800227
228 xen_build_mfn_list_list();
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100229}
230
231unsigned long get_phys_to_machine(unsigned long pfn)
232{
233 unsigned topidx, idx;
234
Jeremy Fitzhardinge8006ec32008-05-26 23:31:19 +0100235 if (unlikely(pfn >= MAX_DOMAIN_PAGES))
236 return INVALID_P2M_ENTRY;
237
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100238 topidx = p2m_top_index(pfn);
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100239 idx = p2m_index(pfn);
240 return p2m_top[topidx][idx];
241}
Ingo Molnar15ce60052008-06-02 13:20:11 +0200242EXPORT_SYMBOL_GPL(get_phys_to_machine);
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100243
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800244/* install a new p2m_top page */
245bool install_p2mtop_page(unsigned long pfn, unsigned long *p)
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100246{
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800247 unsigned topidx = p2m_top_index(pfn);
248 unsigned long **pfnp, *mfnp;
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100249 unsigned i;
250
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800251 pfnp = &p2m_top[topidx];
252 mfnp = &p2m_top_mfn[topidx];
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100253
Tejf63c2f22008-12-16 11:56:06 -0800254 for (i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100255 p[i] = INVALID_P2M_ENTRY;
256
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800257 if (cmpxchg(pfnp, p2m_missing, p) == p2m_missing) {
Jeremy Fitzhardinged5edbc12008-05-26 23:31:22 +0100258 *mfnp = virt_to_mfn(p);
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800259 return true;
260 }
261
262 return false;
263}
264
265static void alloc_p2m(unsigned long pfn)
266{
267 unsigned long *p;
268
269 p = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL);
270 BUG_ON(p == NULL);
271
272 if (!install_p2mtop_page(pfn, p))
273 free_page((unsigned long)p);
274}
275
276/* Try to install p2m mapping; fail if intermediate bits missing */
277bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
278{
279 unsigned topidx, idx;
280
281 if (unlikely(pfn >= MAX_DOMAIN_PAGES)) {
282 BUG_ON(mfn != INVALID_P2M_ENTRY);
283 return true;
284 }
285
286 topidx = p2m_top_index(pfn);
287 if (p2m_top[topidx] == p2m_missing) {
288 if (mfn == INVALID_P2M_ENTRY)
289 return true;
290 return false;
291 }
292
293 idx = p2m_index(pfn);
294 p2m_top[topidx][idx] = mfn;
295
296 return true;
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100297}
298
299void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
300{
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100301 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
302 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
303 return;
304 }
305
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800306 if (unlikely(!__set_phys_to_machine(pfn, mfn))) {
307 alloc_p2m(pfn);
Jeremy Fitzhardinge8006ec32008-05-26 23:31:19 +0100308
Jeremy Fitzhardingee791ca02009-02-26 15:48:33 -0800309 if (!__set_phys_to_machine(pfn, mfn))
310 BUG();
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100311 }
Jeremy Fitzhardinged451bb72008-05-26 23:31:18 +0100312}
313
Jeremy Fitzhardinge9976b392009-02-27 09:19:26 -0800314unsigned long arbitrary_virt_to_mfn(void *vaddr)
315{
316 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
317
318 return PFN_DOWN(maddr.maddr);
319}
320
Jeremy Fitzhardingece803e72008-07-08 15:06:55 -0700321xmaddr_t arbitrary_virt_to_machine(void *vaddr)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700322{
Jeremy Fitzhardingece803e72008-07-08 15:06:55 -0700323 unsigned long address = (unsigned long)vaddr;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100324 unsigned int level;
Chris Lalancette9f32d212008-10-23 17:40:25 -0700325 pte_t *pte;
326 unsigned offset;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700327
Chris Lalancette9f32d212008-10-23 17:40:25 -0700328 /*
329 * if the PFN is in the linear mapped vaddr range, we can just use
330 * the (quick) virt_to_machine() p2m lookup
331 */
332 if (virt_addr_valid(vaddr))
333 return virt_to_machine(vaddr);
334
335 /* otherwise we have to do a (slower) full page-table walk */
336
337 pte = lookup_address(address, &level);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700338 BUG_ON(pte == NULL);
Chris Lalancette9f32d212008-10-23 17:40:25 -0700339 offset = address & ~PAGE_MASK;
Jeremy Fitzhardingeebd879e2008-07-08 15:06:54 -0700340 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700341}
342
343void make_lowmem_page_readonly(void *vaddr)
344{
345 pte_t *pte, ptev;
346 unsigned long address = (unsigned long)vaddr;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100347 unsigned int level;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700348
Ingo Molnarf0646e42008-01-30 13:33:43 +0100349 pte = lookup_address(address, &level);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700350 BUG_ON(pte == NULL);
351
352 ptev = pte_wrprotect(*pte);
353
354 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
355 BUG();
356}
357
358void make_lowmem_page_readwrite(void *vaddr)
359{
360 pte_t *pte, ptev;
361 unsigned long address = (unsigned long)vaddr;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100362 unsigned int level;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700363
Ingo Molnarf0646e42008-01-30 13:33:43 +0100364 pte = lookup_address(address, &level);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700365 BUG_ON(pte == NULL);
366
367 ptev = pte_mkwrite(*pte);
368
369 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
370 BUG();
371}
372
373
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700374static bool xen_page_pinned(void *ptr)
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100375{
376 struct page *page = virt_to_page(ptr);
377
378 return PagePinned(page);
379}
380
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700381static void xen_extend_mmu_update(const struct mmu_update *update)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700382{
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700383 struct multicall_space mcs;
384 struct mmu_update *u;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700385
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700386 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
387
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700388 if (mcs.mc != NULL) {
389 ADD_STATS(mmu_update_extended, 1);
390 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1);
391
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700392 mcs.mc->args[1]++;
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700393
394 if (mcs.mc->args[1] < MMU_UPDATE_HISTO)
395 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1);
396 else
397 ADD_STATS(mmu_update_histo[0], 1);
398 } else {
399 ADD_STATS(mmu_update, 1);
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700400 mcs = __xen_mc_entry(sizeof(*u));
401 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700402 ADD_STATS(mmu_update_histo[1], 1);
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700403 }
404
405 u = mcs.args;
406 *u = *update;
407}
408
409void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
410{
411 struct mmu_update u;
412
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700413 preempt_disable();
414
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700415 xen_mc_batch();
416
Jeremy Fitzhardingece803e72008-07-08 15:06:55 -0700417 /* ptr may be ioremapped for 64-bit pagetable setup */
418 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700419 u.val = pmd_val_ma(val);
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700420 xen_extend_mmu_update(&u);
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700421
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700422 ADD_STATS(pmd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
423
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700424 xen_mc_issue(PARAVIRT_LAZY_MMU);
425
426 preempt_enable();
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700427}
428
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100429void xen_set_pmd(pmd_t *ptr, pmd_t val)
430{
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700431 ADD_STATS(pmd_update, 1);
432
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100433 /* If page is not pinned, we can just update the entry
434 directly */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700435 if (!xen_page_pinned(ptr)) {
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100436 *ptr = val;
437 return;
438 }
439
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700440 ADD_STATS(pmd_update_pinned, 1);
441
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100442 xen_set_pmd_hyper(ptr, val);
443}
444
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700445/*
446 * Associate a virtual page frame with a given physical page frame
447 * and protection flags for that frame.
448 */
449void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
450{
Jeremy Fitzhardinge836fe2f2008-07-08 15:06:58 -0700451 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700452}
453
454void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
455 pte_t *ptep, pte_t pteval)
456{
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700457 ADD_STATS(set_pte_at, 1);
458// ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
459 ADD_STATS(set_pte_at_current, mm == current->mm);
460 ADD_STATS(set_pte_at_kernel, mm == &init_mm);
461
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700462 if (mm == current->mm || mm == &init_mm) {
Jeremy Fitzhardinge8965c1c2007-10-16 11:51:29 -0700463 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700464 struct multicall_space mcs;
465 mcs = xen_mc_entry(0);
466
467 MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700468 ADD_STATS(set_pte_at_batched, 1);
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700469 xen_mc_issue(PARAVIRT_LAZY_MMU);
Jeremy Fitzhardinge2bd50032008-04-02 10:54:10 -0700470 goto out;
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700471 } else
472 if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
Jeremy Fitzhardinge2bd50032008-04-02 10:54:10 -0700473 goto out;
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700474 }
475 xen_set_pte(ptep, pteval);
Jeremy Fitzhardinge2bd50032008-04-02 10:54:10 -0700476
Jeremy Fitzhardinge2829b442009-02-17 23:53:19 -0800477out: return;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700478}
479
Tejf63c2f22008-12-16 11:56:06 -0800480pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
481 unsigned long addr, pte_t *ptep)
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -0700482{
483 /* Just return the pte as-is. We preserve the bits on commit */
484 return *ptep;
485}
486
487void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
488 pte_t *ptep, pte_t pte)
489{
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700490 struct mmu_update u;
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -0700491
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700492 xen_mc_batch();
493
Chris Lalancette9f32d212008-10-23 17:40:25 -0700494 u.ptr = arbitrary_virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700495 u.val = pte_val_ma(pte);
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700496 xen_extend_mmu_update(&u);
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -0700497
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700498 ADD_STATS(prot_commit, 1);
499 ADD_STATS(prot_commit_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
500
Jeremy Fitzhardingee57778a2008-06-16 04:30:02 -0700501 xen_mc_issue(PARAVIRT_LAZY_MMU);
502}
503
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700504/* Assume pteval_t is equivalent to all the other *val_t types. */
505static pteval_t pte_mfn_to_pfn(pteval_t val)
506{
507 if (val & _PAGE_PRESENT) {
Jeremy Fitzhardinge59438c92008-07-21 22:59:42 -0700508 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
Jeremy Fitzhardinge77be1fa2008-07-21 22:59:56 -0700509 pteval_t flags = val & PTE_FLAGS_MASK;
Jeremy Fitzhardinged8355ac2008-07-03 22:10:18 -0700510 val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700511 }
512
513 return val;
514}
515
516static pteval_t pte_pfn_to_mfn(pteval_t val)
517{
518 if (val & _PAGE_PRESENT) {
Jeremy Fitzhardinge59438c92008-07-21 22:59:42 -0700519 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
Jeremy Fitzhardinge77be1fa2008-07-21 22:59:56 -0700520 pteval_t flags = val & PTE_FLAGS_MASK;
Jeremy Fitzhardinged8355ac2008-07-03 22:10:18 -0700521 val = ((pteval_t)pfn_to_mfn(pfn) << PAGE_SHIFT) | flags;
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700522 }
523
524 return val;
525}
526
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700527pteval_t xen_pte_val(pte_t pte)
528{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700529 return pte_mfn_to_pfn(pte.pte);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700530}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800531PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700532
533pgdval_t xen_pgd_val(pgd_t pgd)
534{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700535 return pte_mfn_to_pfn(pgd.pgd);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700536}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800537PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700538
539pte_t xen_make_pte(pteval_t pte)
540{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700541 pte = pte_pfn_to_mfn(pte);
542 return native_make_pte(pte);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700543}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800544PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700545
546pgd_t xen_make_pgd(pgdval_t pgd)
547{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700548 pgd = pte_pfn_to_mfn(pgd);
549 return native_make_pgd(pgd);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700550}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800551PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700552
553pmdval_t xen_pmd_val(pmd_t pmd)
554{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700555 return pte_mfn_to_pfn(pmd.pmd);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700556}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800557PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
Jeremy Fitzhardinge28499142008-05-09 12:05:57 +0100558
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100559void xen_set_pud_hyper(pud_t *ptr, pud_t val)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700560{
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700561 struct mmu_update u;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700562
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700563 preempt_disable();
564
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700565 xen_mc_batch();
566
Jeremy Fitzhardingece803e72008-07-08 15:06:55 -0700567 /* ptr may be ioremapped for 64-bit pagetable setup */
568 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700569 u.val = pud_val_ma(val);
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700570 xen_extend_mmu_update(&u);
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700571
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700572 ADD_STATS(pud_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
573
Jeremy Fitzhardinged66bf8f2007-07-17 18:37:06 -0700574 xen_mc_issue(PARAVIRT_LAZY_MMU);
575
576 preempt_enable();
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700577}
578
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100579void xen_set_pud(pud_t *ptr, pud_t val)
580{
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700581 ADD_STATS(pud_update, 1);
582
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100583 /* If page is not pinned, we can just update the entry
584 directly */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700585 if (!xen_page_pinned(ptr)) {
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100586 *ptr = val;
587 return;
588 }
589
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700590 ADD_STATS(pud_update_pinned, 1);
591
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100592 xen_set_pud_hyper(ptr, val);
593}
594
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700595void xen_set_pte(pte_t *ptep, pte_t pte)
596{
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700597 ADD_STATS(pte_update, 1);
598// ADD_STATS(pte_update_pinned, xen_page_pinned(ptep));
599 ADD_STATS(pte_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
600
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700601#ifdef CONFIG_X86_PAE
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700602 ptep->pte_high = pte.pte_high;
603 smp_wmb();
604 ptep->pte_low = pte.pte_low;
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700605#else
606 *ptep = pte;
607#endif
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700608}
609
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700610#ifdef CONFIG_X86_PAE
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700611void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
612{
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700613 set_64bit((u64 *)ptep, native_pte_val(pte));
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700614}
615
616void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
617{
618 ptep->pte_low = 0;
619 smp_wmb(); /* make sure low gets written first */
620 ptep->pte_high = 0;
621}
622
623void xen_pmd_clear(pmd_t *pmdp)
624{
Jeremy Fitzhardingee2426cf2008-05-31 01:24:27 +0100625 set_pmd(pmdp, __pmd(0));
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700626}
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700627#endif /* CONFIG_X86_PAE */
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700628
Jeremy Fitzhardingeabf33032008-03-17 16:37:07 -0700629pmd_t xen_make_pmd(pmdval_t pmd)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700630{
Jeremy Fitzhardingeebb9cfe2008-06-16 15:01:56 -0700631 pmd = pte_pfn_to_mfn(pmd);
Jeremy Fitzhardinge947a69c2008-03-17 16:37:09 -0700632 return native_make_pmd(pmd);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700633}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800634PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700635
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700636#if PAGETABLE_LEVELS == 4
637pudval_t xen_pud_val(pud_t pud)
638{
639 return pte_mfn_to_pfn(pud.pud);
640}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800641PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700642
643pud_t xen_make_pud(pudval_t pud)
644{
645 pud = pte_pfn_to_mfn(pud);
646
647 return native_make_pud(pud);
648}
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -0800649PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700650
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700651pgd_t *xen_get_user_pgd(pgd_t *pgd)
652{
653 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
654 unsigned offset = pgd - pgd_page;
655 pgd_t *user_ptr = NULL;
656
657 if (offset < pgd_index(USER_LIMIT)) {
658 struct page *page = virt_to_page(pgd_page);
659 user_ptr = (pgd_t *)page->private;
660 if (user_ptr)
661 user_ptr += offset;
662 }
663
664 return user_ptr;
665}
666
667static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700668{
669 struct mmu_update u;
670
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700671 u.ptr = virt_to_machine(ptr).maddr;
672 u.val = pgd_val_ma(val);
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700673 xen_extend_mmu_update(&u);
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700674}
675
676/*
677 * Raw hypercall-based set_pgd, intended for in early boot before
678 * there's a page structure. This implies:
679 * 1. The only existing pagetable is the kernel's
680 * 2. It is always pinned
681 * 3. It has no user pagetable attached to it
682 */
683void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
684{
685 preempt_disable();
686
687 xen_mc_batch();
688
689 __xen_set_pgd_hyper(ptr, val);
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700690
691 xen_mc_issue(PARAVIRT_LAZY_MMU);
692
693 preempt_enable();
694}
695
696void xen_set_pgd(pgd_t *ptr, pgd_t val)
697{
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700698 pgd_t *user_ptr = xen_get_user_pgd(ptr);
699
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700700 ADD_STATS(pgd_update, 1);
701
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700702 /* If page is not pinned, we can just update the entry
703 directly */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700704 if (!xen_page_pinned(ptr)) {
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700705 *ptr = val;
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700706 if (user_ptr) {
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700707 WARN_ON(xen_page_pinned(user_ptr));
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700708 *user_ptr = val;
709 }
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700710 return;
711 }
712
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700713 ADD_STATS(pgd_update_pinned, 1);
714 ADD_STATS(pgd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
715
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700716 /* If it's pinned, then we can at least batch the kernel and
717 user updates together. */
718 xen_mc_batch();
719
720 __xen_set_pgd_hyper(ptr, val);
721 if (user_ptr)
722 __xen_set_pgd_hyper(user_ptr, val);
723
724 xen_mc_issue(PARAVIRT_LAZY_MMU);
Jeremy Fitzhardingef6e58732008-07-08 15:06:38 -0700725}
726#endif /* PAGETABLE_LEVELS == 4 */
727
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700728/*
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700729 * (Yet another) pagetable walker. This one is intended for pinning a
730 * pagetable. This means that it walks a pagetable and calls the
731 * callback function on each page it finds making up the page table,
732 * at every level. It walks the entire pagetable, but it only bothers
733 * pinning pte pages which are below limit. In the normal case this
734 * will be STACK_TOP_MAX, but at boot we need to pin up to
735 * FIXADDR_TOP.
736 *
737 * For 32-bit the important bit is that we don't pin beyond there,
738 * because then we start getting into Xen's ptes.
739 *
740 * For 64-bit, we must skip the Xen hole in the middle of the address
741 * space, just after the big x86-64 virtual hole.
742 */
Ian Campbell86bbc2c2008-11-21 10:21:33 +0000743static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
744 int (*func)(struct mm_struct *mm, struct page *,
745 enum pt_level),
746 unsigned long limit)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700747{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700748 int flush = 0;
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700749 unsigned hole_low, hole_high;
750 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
751 unsigned pgdidx, pudidx, pmdidx;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700752
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700753 /* The limit is the last byte to be touched */
754 limit--;
755 BUG_ON(limit >= FIXADDR_TOP);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700756
757 if (xen_feature(XENFEAT_auto_translated_physmap))
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700758 return 0;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700759
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700760 /*
761 * 64-bit has a great big hole in the middle of the address
762 * space, which contains the Xen mappings. On 32-bit these
763 * will end up making a zero-sized hole and so is a no-op.
764 */
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700765 hole_low = pgd_index(USER_LIMIT);
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700766 hole_high = pgd_index(PAGE_OFFSET);
767
768 pgdidx_limit = pgd_index(limit);
769#if PTRS_PER_PUD > 1
770 pudidx_limit = pud_index(limit);
771#else
772 pudidx_limit = 0;
773#endif
774#if PTRS_PER_PMD > 1
775 pmdidx_limit = pmd_index(limit);
776#else
777 pmdidx_limit = 0;
778#endif
779
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700780 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700781 pud_t *pud;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700782
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700783 if (pgdidx >= hole_low && pgdidx < hole_high)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700784 continue;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700785
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700786 if (!pgd_val(pgd[pgdidx]))
787 continue;
788
789 pud = pud_offset(&pgd[pgdidx], 0);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700790
791 if (PTRS_PER_PUD > 1) /* not folded */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700792 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700793
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700794 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700795 pmd_t *pmd;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700796
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700797 if (pgdidx == pgdidx_limit &&
798 pudidx > pudidx_limit)
799 goto out;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700800
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700801 if (pud_none(pud[pudidx]))
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700802 continue;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700803
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700804 pmd = pmd_offset(&pud[pudidx], 0);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700805
806 if (PTRS_PER_PMD > 1) /* not folded */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700807 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700808
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700809 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
810 struct page *pte;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700811
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700812 if (pgdidx == pgdidx_limit &&
813 pudidx == pudidx_limit &&
814 pmdidx > pmdidx_limit)
815 goto out;
816
817 if (pmd_none(pmd[pmdidx]))
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700818 continue;
819
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700820 pte = pmd_page(pmd[pmdidx]);
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700821 flush |= (*func)(mm, pte, PT_PTE);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700822 }
823 }
824 }
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -0700825
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700826out:
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -0700827 /* Do the top level last, so that the callbacks can use it as
828 a cue to do final things like tlb flushes. */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700829 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700830
831 return flush;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700832}
833
Ian Campbell86bbc2c2008-11-21 10:21:33 +0000834static int xen_pgd_walk(struct mm_struct *mm,
835 int (*func)(struct mm_struct *mm, struct page *,
836 enum pt_level),
837 unsigned long limit)
838{
839 return __xen_pgd_walk(mm, mm->pgd, func, limit);
840}
841
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700842/* If we're using split pte locks, then take the page's lock and
843 return a pointer to it. Otherwise return NULL. */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700844static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700845{
846 spinlock_t *ptl = NULL;
847
Jeremy Fitzhardingef7d0b922008-09-09 15:43:22 -0700848#if USE_SPLIT_PTLOCKS
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700849 ptl = __pte_lockptr(page);
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700850 spin_lock_nest_lock(ptl, &mm->page_table_lock);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700851#endif
852
853 return ptl;
854}
855
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700856static void xen_pte_unlock(void *v)
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700857{
858 spinlock_t *ptl = v;
859 spin_unlock(ptl);
860}
861
862static void xen_do_pin(unsigned level, unsigned long pfn)
863{
864 struct mmuext_op *op;
865 struct multicall_space mcs;
866
867 mcs = __xen_mc_entry(sizeof(*op));
868 op = mcs.args;
869 op->cmd = level;
870 op->arg1.mfn = pfn_to_mfn(pfn);
871 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
872}
873
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700874static int xen_pin_page(struct mm_struct *mm, struct page *page,
875 enum pt_level level)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700876{
Christoph Lameterd60cd462008-04-28 02:12:51 -0700877 unsigned pgfl = TestSetPagePinned(page);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700878 int flush;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700879
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700880 if (pgfl)
881 flush = 0; /* already pinned */
882 else if (PageHighMem(page))
883 /* kmaps need flushing if we found an unpinned
884 highpage */
885 flush = 1;
886 else {
887 void *pt = lowmem_page_address(page);
888 unsigned long pfn = page_to_pfn(page);
889 struct multicall_space mcs = __xen_mc_entry(0);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700890 spinlock_t *ptl;
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700891
892 flush = 0;
893
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -0700894 /*
895 * We need to hold the pagetable lock between the time
896 * we make the pagetable RO and when we actually pin
897 * it. If we don't, then other users may come in and
898 * attempt to update the pagetable by writing it,
899 * which will fail because the memory is RO but not
900 * pinned, so Xen won't do the trap'n'emulate.
901 *
902 * If we're using split pte locks, we can't hold the
903 * entire pagetable's worth of locks during the
904 * traverse, because we may wrap the preempt count (8
905 * bits). The solution is to mark RO and pin each PTE
906 * page while holding the lock. This means the number
907 * of locks we end up holding is never more than a
908 * batch size (~32 entries, at present).
909 *
910 * If we're not using split pte locks, we needn't pin
911 * the PTE pages independently, because we're
912 * protected by the overall pagetable lock.
913 */
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700914 ptl = NULL;
915 if (level == PT_PTE)
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700916 ptl = xen_pte_lock(page, mm);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700917
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700918 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
919 pfn_pte(pfn, PAGE_KERNEL_RO),
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700920 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
921
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -0700922 if (ptl) {
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700923 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
924
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700925 /* Queue a deferred unlock for when this batch
926 is completed. */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -0700927 xen_mc_callback(xen_pte_unlock, ptl);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -0700928 }
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700929 }
930
931 return flush;
932}
933
934/* This is called just after a mm has been created, but it has not
935 been used yet. We need to make sure that its pagetable is all
936 read-only, and can be pinned. */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700937static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700938{
Jeremy Fitzhardinged05fdf32008-10-28 19:23:06 +1100939 vm_unmap_aliases();
940
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700941 xen_mc_batch();
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -0700942
Ian Campbell86bbc2c2008-11-21 10:21:33 +0000943 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
Jeremy Fitzhardinged05fdf32008-10-28 19:23:06 +1100944 /* re-enable interrupts for flushing */
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700945 xen_mc_issue(0);
Jeremy Fitzhardinged05fdf32008-10-28 19:23:06 +1100946
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700947 kmap_flush_unused();
Jeremy Fitzhardinged05fdf32008-10-28 19:23:06 +1100948
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700949 xen_mc_batch();
950 }
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700951
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700952#ifdef CONFIG_X86_64
953 {
954 pgd_t *user_pgd = xen_get_user_pgd(pgd);
955
956 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
957
958 if (user_pgd) {
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700959 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
Tejf63c2f22008-12-16 11:56:06 -0800960 xen_do_pin(MMUEXT_PIN_L4_TABLE,
961 PFN_DOWN(__pa(user_pgd)));
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700962 }
963 }
964#else /* CONFIG_X86_32 */
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700965#ifdef CONFIG_X86_PAE
966 /* Need to make sure unshared kernel PMD is pinnable */
Jeremy Fitzhardinge47cb2ed2008-11-06 13:48:24 -0800967 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700968 PT_PMD);
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -0700969#endif
Jeremy Fitzhardinge28499142008-05-09 12:05:57 +0100970 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -0700971#endif /* CONFIG_X86_64 */
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -0700972 xen_mc_issue(0);
973}
974
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700975static void xen_pgd_pin(struct mm_struct *mm)
976{
977 __xen_pgd_pin(mm, mm->pgd);
978}
979
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100980/*
981 * On save, we need to pin all pagetables to make sure they get their
982 * mfns turned into pfns. Search the list for any unpinned pgds and pin
983 * them (unpinned pgds are not currently in use, probably because the
984 * process is under construction or destruction).
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700985 *
986 * Expected to be called in stop_machine() ("equivalent to taking
987 * every spinlock in the system"), so the locking doesn't really
988 * matter all that much.
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100989 */
990void xen_mm_pin_all(void)
991{
992 unsigned long flags;
993 struct page *page;
994
995 spin_lock_irqsave(&pgd_lock, flags);
996
997 list_for_each_entry(page, &pgd_list, lru) {
998 if (!PagePinned(page)) {
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -0700999 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001000 SetPageSavePinned(page);
1001 }
1002 }
1003
1004 spin_unlock_irqrestore(&pgd_lock, flags);
1005}
1006
Eduardo Habkostc1f2f092008-07-08 15:06:24 -07001007/*
1008 * The init_mm pagetable is really pinned as soon as its created, but
1009 * that's before we have page structures to store the bits. So do all
1010 * the book-keeping now.
1011 */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001012static __init int xen_mark_pinned(struct mm_struct *mm, struct page *page,
1013 enum pt_level level)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001014{
1015 SetPagePinned(page);
1016 return 0;
1017}
1018
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001019static void __init xen_mark_init_mm_pinned(void)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001020{
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001021 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001022}
1023
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001024static int xen_unpin_page(struct mm_struct *mm, struct page *page,
1025 enum pt_level level)
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001026{
Christoph Lameterd60cd462008-04-28 02:12:51 -07001027 unsigned pgfl = TestClearPagePinned(page);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001028
1029 if (pgfl && !PageHighMem(page)) {
1030 void *pt = lowmem_page_address(page);
1031 unsigned long pfn = page_to_pfn(page);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001032 spinlock_t *ptl = NULL;
1033 struct multicall_space mcs;
1034
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -07001035 /*
1036 * Do the converse to pin_page. If we're using split
1037 * pte locks, we must be holding the lock for while
1038 * the pte page is unpinned but still RO to prevent
1039 * concurrent updates from seeing it in this
1040 * partially-pinned state.
1041 */
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001042 if (level == PT_PTE) {
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001043 ptl = xen_pte_lock(page, mm);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001044
Jeremy Fitzhardinge11ad93e2008-08-19 13:32:51 -07001045 if (ptl)
1046 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001047 }
1048
1049 mcs = __xen_mc_entry(0);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001050
1051 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
1052 pfn_pte(pfn, PAGE_KERNEL),
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001053 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
1054
1055 if (ptl) {
1056 /* unlock when batch completed */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -07001057 xen_mc_callback(xen_pte_unlock, ptl);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001058 }
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001059 }
1060
1061 return 0; /* never need to flush on unpin */
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001062}
1063
1064/* Release a pagetables pages back as normal RW */
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001065static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001066{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001067 xen_mc_batch();
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001068
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001069 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001070
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -07001071#ifdef CONFIG_X86_64
1072 {
1073 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1074
1075 if (user_pgd) {
Tejf63c2f22008-12-16 11:56:06 -08001076 xen_do_pin(MMUEXT_UNPIN_TABLE,
1077 PFN_DOWN(__pa(user_pgd)));
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001078 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -07001079 }
1080 }
1081#endif
1082
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -07001083#ifdef CONFIG_X86_PAE
1084 /* Need to make sure unshared kernel PMD is unpinned */
Jeremy Fitzhardinge47cb2ed2008-11-06 13:48:24 -08001085 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001086 PT_PMD);
Jeremy Fitzhardinge5deb30d2008-07-08 15:07:06 -07001087#endif
Jeremy Fitzhardinged6182fb2008-07-08 15:07:13 -07001088
Ian Campbell86bbc2c2008-11-21 10:21:33 +00001089 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001090
1091 xen_mc_issue(0);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001092}
1093
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001094static void xen_pgd_unpin(struct mm_struct *mm)
1095{
1096 __xen_pgd_unpin(mm, mm->pgd);
1097}
1098
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001099/*
1100 * On resume, undo any pinning done at save, so that the rest of the
1101 * kernel doesn't see any unexpected pinned pagetables.
1102 */
1103void xen_mm_unpin_all(void)
1104{
1105 unsigned long flags;
1106 struct page *page;
1107
1108 spin_lock_irqsave(&pgd_lock, flags);
1109
1110 list_for_each_entry(page, &pgd_list, lru) {
1111 if (PageSavePinned(page)) {
1112 BUG_ON(!PagePinned(page));
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001113 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001114 ClearPageSavePinned(page);
1115 }
1116 }
1117
1118 spin_unlock_irqrestore(&pgd_lock, flags);
1119}
1120
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001121void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1122{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001123 spin_lock(&next->page_table_lock);
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001124 xen_pgd_pin(next);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001125 spin_unlock(&next->page_table_lock);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001126}
1127
1128void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1129{
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001130 spin_lock(&mm->page_table_lock);
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001131 xen_pgd_pin(mm);
Jeremy Fitzhardingef4f97b32007-07-17 18:37:05 -07001132 spin_unlock(&mm->page_table_lock);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001133}
1134
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001135
1136#ifdef CONFIG_SMP
1137/* Another cpu may still have their %cr3 pointing at the pagetable, so
1138 we need to repoint it somewhere else before we can unpin it. */
1139static void drop_other_mm_ref(void *info)
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001140{
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001141 struct mm_struct *mm = info;
Jeremy Fitzhardingece87b3d2008-07-08 15:06:40 -07001142 struct mm_struct *active_mm;
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001143
Brian Gerst9eb912d2009-01-19 00:38:57 +09001144 active_mm = percpu_read(cpu_tlbstate.active_mm);
Jeremy Fitzhardingece87b3d2008-07-08 15:06:40 -07001145
1146 if (active_mm == mm)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001147 leave_mm(smp_processor_id());
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001148
1149 /* If this cpu still has a stale cr3 reference, then make sure
1150 it has been flushed. */
Jeremy Fitzhardinge7fd7d832009-02-17 23:24:03 -08001151 if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001152 load_cr3(swapper_pg_dir);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001153}
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001154
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -07001155static void xen_drop_mm_ref(struct mm_struct *mm)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001156{
Mike Travise4d98202008-12-16 17:34:05 -08001157 cpumask_var_t mask;
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001158 unsigned cpu;
1159
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001160 if (current->active_mm == mm) {
1161 if (current->mm == mm)
1162 load_cr3(swapper_pg_dir);
1163 else
1164 leave_mm(smp_processor_id());
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001165 }
1166
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001167 /* Get the "official" set of cpus referring to our pagetable. */
Mike Travise4d98202008-12-16 17:34:05 -08001168 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1169 for_each_online_cpu(cpu) {
Rusty Russell78f1c4d2009-09-24 09:34:51 -06001170 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
Mike Travise4d98202008-12-16 17:34:05 -08001171 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1172 continue;
1173 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1174 }
1175 return;
1176 }
Rusty Russell78f1c4d2009-09-24 09:34:51 -06001177 cpumask_copy(mask, mm_cpumask(mm));
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001178
1179 /* It's possible that a vcpu may have a stale reference to our
1180 cr3, because its in lazy mode, and it hasn't yet flushed
1181 its set of pending hypercalls yet. In this case, we can
1182 look at its actual current cr3 value, and force it to flush
1183 if needed. */
1184 for_each_online_cpu(cpu) {
1185 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
Mike Travise4d98202008-12-16 17:34:05 -08001186 cpumask_set_cpu(cpu, mask);
Jeremy Fitzhardinge9f799912007-10-16 11:51:30 -07001187 }
1188
Mike Travise4d98202008-12-16 17:34:05 -08001189 if (!cpumask_empty(mask))
1190 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1191 free_cpumask_var(mask);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001192}
1193#else
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -07001194static void xen_drop_mm_ref(struct mm_struct *mm)
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001195{
1196 if (current->active_mm == mm)
1197 load_cr3(swapper_pg_dir);
1198}
1199#endif
1200
1201/*
1202 * While a process runs, Xen pins its pagetables, which means that the
1203 * hypervisor forces it to be read-only, and it controls all updates
1204 * to it. This means that all pagetable updates have to go via the
1205 * hypervisor, which is moderately expensive.
1206 *
1207 * Since we're pulling the pagetable down, we switch to use init_mm,
1208 * unpin old process pagetable and mark it all read-write, which
1209 * allows further operations on it to be simple memory accesses.
1210 *
1211 * The only subtle point is that another CPU may be still using the
1212 * pagetable because of lazy tlb flushing. This means we need need to
1213 * switch all CPUs off this pagetable before we can unpin it.
1214 */
1215void xen_exit_mmap(struct mm_struct *mm)
1216{
1217 get_cpu(); /* make sure we don't move around */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -07001218 xen_drop_mm_ref(mm);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001219 put_cpu();
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001220
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -07001221 spin_lock(&mm->page_table_lock);
Jeremy Fitzhardingedf912ea2007-09-25 11:50:00 -07001222
1223 /* pgd may not be pinned in the error exit path of execve */
Jeremy Fitzhardinge7708ad62008-08-19 13:34:22 -07001224 if (xen_page_pinned(mm->pgd))
Jeremy Fitzhardingeeefb47f2008-10-08 13:01:39 -07001225 xen_pgd_unpin(mm);
Jeremy Fitzhardinge74260712007-10-16 11:51:30 -07001226
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -07001227 spin_unlock(&mm->page_table_lock);
Jeremy Fitzhardinge3b827c12007-07-17 18:37:04 -07001228}
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -07001229
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001230static __init void xen_pagetable_setup_start(pgd_t *base)
1231{
1232}
1233
Thomas Gleixnerf1d70622009-08-20 13:13:52 +02001234static void xen_post_allocator_init(void);
1235
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001236static __init void xen_pagetable_setup_done(pgd_t *base)
1237{
1238 xen_setup_shared_info();
Thomas Gleixnerf1d70622009-08-20 13:13:52 +02001239 xen_post_allocator_init();
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001240}
1241
1242static void xen_write_cr2(unsigned long cr2)
1243{
1244 percpu_read(xen_vcpu)->arch.cr2 = cr2;
1245}
1246
1247static unsigned long xen_read_cr2(void)
1248{
1249 return percpu_read(xen_vcpu)->arch.cr2;
1250}
1251
1252unsigned long xen_read_cr2_direct(void)
1253{
1254 return percpu_read(xen_vcpu_info.arch.cr2);
1255}
1256
1257static void xen_flush_tlb(void)
1258{
1259 struct mmuext_op *op;
1260 struct multicall_space mcs;
1261
1262 preempt_disable();
1263
1264 mcs = xen_mc_entry(sizeof(*op));
1265
1266 op = mcs.args;
1267 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1268 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1269
1270 xen_mc_issue(PARAVIRT_LAZY_MMU);
1271
1272 preempt_enable();
1273}
1274
1275static void xen_flush_tlb_single(unsigned long addr)
1276{
1277 struct mmuext_op *op;
1278 struct multicall_space mcs;
1279
1280 preempt_disable();
1281
1282 mcs = xen_mc_entry(sizeof(*op));
1283 op = mcs.args;
1284 op->cmd = MMUEXT_INVLPG_LOCAL;
1285 op->arg1.linear_addr = addr & PAGE_MASK;
1286 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1287
1288 xen_mc_issue(PARAVIRT_LAZY_MMU);
1289
1290 preempt_enable();
1291}
1292
1293static void xen_flush_tlb_others(const struct cpumask *cpus,
1294 struct mm_struct *mm, unsigned long va)
1295{
1296 struct {
1297 struct mmuext_op op;
1298 DECLARE_BITMAP(mask, NR_CPUS);
1299 } *args;
1300 struct multicall_space mcs;
1301
Jeremy Fitzhardingee3f8a742009-03-04 17:36:57 -08001302 if (cpumask_empty(cpus))
1303 return; /* nothing to do */
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001304
1305 mcs = xen_mc_entry(sizeof(*args));
1306 args = mcs.args;
1307 args->op.arg2.vcpumask = to_cpumask(args->mask);
1308
1309 /* Remove us, and any offline CPUS. */
1310 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1311 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001312
1313 if (va == TLB_FLUSH_ALL) {
1314 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1315 } else {
1316 args->op.cmd = MMUEXT_INVLPG_MULTI;
1317 args->op.arg1.linear_addr = va;
1318 }
1319
1320 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1321
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001322 xen_mc_issue(PARAVIRT_LAZY_MMU);
1323}
1324
1325static unsigned long xen_read_cr3(void)
1326{
1327 return percpu_read(xen_cr3);
1328}
1329
1330static void set_current_cr3(void *v)
1331{
1332 percpu_write(xen_current_cr3, (unsigned long)v);
1333}
1334
1335static void __xen_write_cr3(bool kernel, unsigned long cr3)
1336{
1337 struct mmuext_op *op;
1338 struct multicall_space mcs;
1339 unsigned long mfn;
1340
1341 if (cr3)
1342 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1343 else
1344 mfn = 0;
1345
1346 WARN_ON(mfn == 0 && kernel);
1347
1348 mcs = __xen_mc_entry(sizeof(*op));
1349
1350 op = mcs.args;
1351 op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1352 op->arg1.mfn = mfn;
1353
1354 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1355
1356 if (kernel) {
1357 percpu_write(xen_cr3, cr3);
1358
1359 /* Update xen_current_cr3 once the batch has actually
1360 been submitted. */
1361 xen_mc_callback(set_current_cr3, (void *)cr3);
1362 }
1363}
1364
1365static void xen_write_cr3(unsigned long cr3)
1366{
1367 BUG_ON(preemptible());
1368
1369 xen_mc_batch(); /* disables interrupts */
1370
1371 /* Update while interrupts are disabled, so its atomic with
1372 respect to ipis */
1373 percpu_write(xen_cr3, cr3);
1374
1375 __xen_write_cr3(true, cr3);
1376
1377#ifdef CONFIG_X86_64
1378 {
1379 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1380 if (user_pgd)
1381 __xen_write_cr3(false, __pa(user_pgd));
1382 else
1383 __xen_write_cr3(false, 0);
1384 }
1385#endif
1386
1387 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1388}
1389
1390static int xen_pgd_alloc(struct mm_struct *mm)
1391{
1392 pgd_t *pgd = mm->pgd;
1393 int ret = 0;
1394
1395 BUG_ON(PagePinned(virt_to_page(pgd)));
1396
1397#ifdef CONFIG_X86_64
1398 {
1399 struct page *page = virt_to_page(pgd);
1400 pgd_t *user_pgd;
1401
1402 BUG_ON(page->private != 0);
1403
1404 ret = -ENOMEM;
1405
1406 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1407 page->private = (unsigned long)user_pgd;
1408
1409 if (user_pgd != NULL) {
1410 user_pgd[pgd_index(VSYSCALL_START)] =
1411 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1412 ret = 0;
1413 }
1414
1415 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1416 }
1417#endif
1418
1419 return ret;
1420}
1421
1422static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1423{
1424#ifdef CONFIG_X86_64
1425 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1426
1427 if (user_pgd)
1428 free_page((unsigned long)user_pgd);
1429#endif
1430}
1431
Jeremy Fitzhardinge1f4f9312009-02-02 13:58:06 -08001432#ifdef CONFIG_X86_32
1433static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
1434{
1435 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1436 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1437 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1438 pte_val_ma(pte));
1439
1440 return pte;
1441}
1442
1443/* Init-time set_pte while constructing initial pagetables, which
1444 doesn't allow RO pagetable pages to be remapped RW */
1445static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
1446{
1447 pte = mask_rw_pte(ptep, pte);
1448
1449 xen_set_pte(ptep, pte);
1450}
1451#endif
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001452
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001453static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1454{
1455 struct mmuext_op op;
1456 op.cmd = cmd;
1457 op.arg1.mfn = pfn_to_mfn(pfn);
1458 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1459 BUG();
1460}
1461
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001462/* Early in boot, while setting up the initial pagetable, assume
1463 everything is pinned. */
1464static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1465{
1466#ifdef CONFIG_FLATMEM
1467 BUG_ON(mem_map); /* should only be used early */
1468#endif
1469 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001470 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1471}
1472
1473/* Used for pmd and pud */
1474static __init void xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1475{
1476#ifdef CONFIG_FLATMEM
1477 BUG_ON(mem_map); /* should only be used early */
1478#endif
1479 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001480}
1481
1482/* Early release_pte assumes that all pts are pinned, since there's
1483 only init_mm and anything attached to that is pinned. */
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001484static __init void xen_release_pte_init(unsigned long pfn)
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001485{
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001486 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001487 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1488}
1489
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001490static __init void xen_release_pmd_init(unsigned long pfn)
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001491{
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001492 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001493}
1494
1495/* This needs to make sure the new pte page is pinned iff its being
1496 attached to a pinned pagetable. */
1497static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level)
1498{
1499 struct page *page = pfn_to_page(pfn);
1500
1501 if (PagePinned(virt_to_page(mm->pgd))) {
1502 SetPagePinned(page);
1503
1504 vm_unmap_aliases();
1505 if (!PageHighMem(page)) {
1506 make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn)));
1507 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1508 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1509 } else {
1510 /* make sure there are no stray mappings of
1511 this page */
1512 kmap_flush_unused();
1513 }
1514 }
1515}
1516
1517static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1518{
1519 xen_alloc_ptpage(mm, pfn, PT_PTE);
1520}
1521
1522static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1523{
1524 xen_alloc_ptpage(mm, pfn, PT_PMD);
1525}
1526
1527/* This should never happen until we're OK to use struct page */
1528static void xen_release_ptpage(unsigned long pfn, unsigned level)
1529{
1530 struct page *page = pfn_to_page(pfn);
1531
1532 if (PagePinned(page)) {
1533 if (!PageHighMem(page)) {
1534 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1535 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1536 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1537 }
1538 ClearPagePinned(page);
1539 }
1540}
1541
1542static void xen_release_pte(unsigned long pfn)
1543{
1544 xen_release_ptpage(pfn, PT_PTE);
1545}
1546
1547static void xen_release_pmd(unsigned long pfn)
1548{
1549 xen_release_ptpage(pfn, PT_PMD);
1550}
1551
1552#if PAGETABLE_LEVELS == 4
1553static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1554{
1555 xen_alloc_ptpage(mm, pfn, PT_PUD);
1556}
1557
1558static void xen_release_pud(unsigned long pfn)
1559{
1560 xen_release_ptpage(pfn, PT_PUD);
1561}
1562#endif
1563
1564void __init xen_reserve_top(void)
1565{
1566#ifdef CONFIG_X86_32
1567 unsigned long top = HYPERVISOR_VIRT_START;
1568 struct xen_platform_parameters pp;
1569
1570 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1571 top = pp.virt_start;
1572
1573 reserve_top_address(-top);
1574#endif /* CONFIG_X86_32 */
1575}
1576
1577/*
1578 * Like __va(), but returns address in the kernel mapping (which is
1579 * all we have until the physical memory mapping has been set up.
1580 */
1581static void *__ka(phys_addr_t paddr)
1582{
1583#ifdef CONFIG_X86_64
1584 return (void *)(paddr + __START_KERNEL_map);
1585#else
1586 return __va(paddr);
1587#endif
1588}
1589
1590/* Convert a machine address to physical address */
1591static unsigned long m2p(phys_addr_t maddr)
1592{
1593 phys_addr_t paddr;
1594
1595 maddr &= PTE_PFN_MASK;
1596 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1597
1598 return paddr;
1599}
1600
1601/* Convert a machine address to kernel virtual */
1602static void *m2v(phys_addr_t maddr)
1603{
1604 return __ka(m2p(maddr));
1605}
1606
1607static void set_page_prot(void *addr, pgprot_t prot)
1608{
1609 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1610 pte_t pte = pfn_pte(pfn, prot);
1611
1612 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
1613 BUG();
1614}
1615
1616static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1617{
1618 unsigned pmdidx, pteidx;
1619 unsigned ident_pte;
1620 unsigned long pfn;
1621
1622 ident_pte = 0;
1623 pfn = 0;
1624 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1625 pte_t *pte_page;
1626
1627 /* Reuse or allocate a page of ptes */
1628 if (pmd_present(pmd[pmdidx]))
1629 pte_page = m2v(pmd[pmdidx].pmd);
1630 else {
1631 /* Check for free pte pages */
1632 if (ident_pte == ARRAY_SIZE(level1_ident_pgt))
1633 break;
1634
1635 pte_page = &level1_ident_pgt[ident_pte];
1636 ident_pte += PTRS_PER_PTE;
1637
1638 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1639 }
1640
1641 /* Install mappings */
1642 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1643 pte_t pte;
1644
1645 if (pfn > max_pfn_mapped)
1646 max_pfn_mapped = pfn;
1647
1648 if (!pte_none(pte_page[pteidx]))
1649 continue;
1650
1651 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1652 pte_page[pteidx] = pte;
1653 }
1654 }
1655
1656 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1657 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1658
1659 set_page_prot(pmd, PAGE_KERNEL_RO);
1660}
1661
1662#ifdef CONFIG_X86_64
1663static void convert_pfn_mfn(void *v)
1664{
1665 pte_t *pte = v;
1666 int i;
1667
1668 /* All levels are converted the same way, so just treat them
1669 as ptes. */
1670 for (i = 0; i < PTRS_PER_PTE; i++)
1671 pte[i] = xen_make_pte(pte[i].pte);
1672}
1673
1674/*
1675 * Set up the inital kernel pagetable.
1676 *
1677 * We can construct this by grafting the Xen provided pagetable into
1678 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1679 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
1680 * means that only the kernel has a physical mapping to start with -
1681 * but that's enough to get __va working. We need to fill in the rest
1682 * of the physical mapping once some sort of allocator has been set
1683 * up.
1684 */
1685__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1686 unsigned long max_pfn)
1687{
1688 pud_t *l3;
1689 pmd_t *l2;
1690
1691 /* Zap identity mapping */
1692 init_level4_pgt[0] = __pgd(0);
1693
1694 /* Pre-constructed entries are in pfn, so convert to mfn */
1695 convert_pfn_mfn(init_level4_pgt);
1696 convert_pfn_mfn(level3_ident_pgt);
1697 convert_pfn_mfn(level3_kernel_pgt);
1698
1699 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1700 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1701
1702 memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1703 memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1704
1705 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
1706 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
1707 memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1708
1709 /* Set up identity map */
1710 xen_map_identity_early(level2_ident_pgt, max_pfn);
1711
1712 /* Make pagetable pieces RO */
1713 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1714 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1715 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1716 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1717 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1718 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1719
1720 /* Pin down new L4 */
1721 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1722 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1723
1724 /* Unpin Xen-provided one */
1725 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1726
1727 /* Switch over */
1728 pgd = init_level4_pgt;
1729
1730 /*
1731 * At this stage there can be no user pgd, and no page
1732 * structure to attach it to, so make sure we just set kernel
1733 * pgd.
1734 */
1735 xen_mc_batch();
1736 __xen_write_cr3(true, __pa(pgd));
1737 xen_mc_issue(PARAVIRT_LAZY_CPU);
1738
1739 reserve_early(__pa(xen_start_info->pt_base),
1740 __pa(xen_start_info->pt_base +
1741 xen_start_info->nr_pt_frames * PAGE_SIZE),
1742 "XEN PAGETABLES");
1743
1744 return pgd;
1745}
1746#else /* !CONFIG_X86_64 */
1747static pmd_t level2_kernel_pgt[PTRS_PER_PMD] __page_aligned_bss;
1748
1749__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1750 unsigned long max_pfn)
1751{
1752 pmd_t *kernel_pmd;
1753
Jeremy Fitzhardinge93dbda72009-02-26 17:35:44 -08001754 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
1755 xen_start_info->nr_pt_frames * PAGE_SIZE +
1756 512*1024);
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001757
1758 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
1759 memcpy(level2_kernel_pgt, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
1760
1761 xen_map_identity_early(level2_kernel_pgt, max_pfn);
1762
1763 memcpy(swapper_pg_dir, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
1764 set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY],
1765 __pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT));
1766
1767 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1768 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1769 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
1770
1771 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1772
1773 xen_write_cr3(__pa(swapper_pg_dir));
1774
1775 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir)));
1776
Jeremy Fitzhardinge33df4db2009-05-07 11:56:44 -07001777 reserve_early(__pa(xen_start_info->pt_base),
1778 __pa(xen_start_info->pt_base +
1779 xen_start_info->nr_pt_frames * PAGE_SIZE),
1780 "XEN PAGETABLES");
1781
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001782 return swapper_pg_dir;
1783}
1784#endif /* CONFIG_X86_64 */
1785
Masami Hiramatsu3b3809a2009-04-09 10:55:33 -07001786static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001787{
1788 pte_t pte;
1789
1790 phys >>= PAGE_SHIFT;
1791
1792 switch (idx) {
1793 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
1794#ifdef CONFIG_X86_F00F_BUG
1795 case FIX_F00F_IDT:
1796#endif
1797#ifdef CONFIG_X86_32
1798 case FIX_WP_TEST:
1799 case FIX_VDSO:
1800# ifdef CONFIG_HIGHMEM
1801 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
1802# endif
1803#else
1804 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
1805#endif
1806#ifdef CONFIG_X86_LOCAL_APIC
1807 case FIX_APIC_BASE: /* maps dummy local APIC */
1808#endif
Jeremy Fitzhardinge3ecb1b72009-03-07 23:48:41 -08001809 case FIX_TEXT_POKE0:
1810 case FIX_TEXT_POKE1:
1811 /* All local page mappings */
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001812 pte = pfn_pte(phys, prot);
1813 break;
1814
1815 default:
1816 pte = mfn_pte(phys, prot);
1817 break;
1818 }
1819
1820 __native_set_fixmap(idx, pte);
1821
1822#ifdef CONFIG_X86_64
1823 /* Replicate changes to map the vsyscall page into the user
1824 pagetable vsyscall mapping. */
1825 if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) {
1826 unsigned long vaddr = __fix_to_virt(idx);
1827 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
1828 }
1829#endif
1830}
1831
Thomas Gleixnerf1d70622009-08-20 13:13:52 +02001832static __init void xen_post_allocator_init(void)
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001833{
1834 pv_mmu_ops.set_pte = xen_set_pte;
1835 pv_mmu_ops.set_pmd = xen_set_pmd;
1836 pv_mmu_ops.set_pud = xen_set_pud;
1837#if PAGETABLE_LEVELS == 4
1838 pv_mmu_ops.set_pgd = xen_set_pgd;
1839#endif
1840
1841 /* This will work as long as patching hasn't happened yet
1842 (which it hasn't) */
1843 pv_mmu_ops.alloc_pte = xen_alloc_pte;
1844 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
1845 pv_mmu_ops.release_pte = xen_release_pte;
1846 pv_mmu_ops.release_pmd = xen_release_pmd;
1847#if PAGETABLE_LEVELS == 4
1848 pv_mmu_ops.alloc_pud = xen_alloc_pud;
1849 pv_mmu_ops.release_pud = xen_release_pud;
1850#endif
1851
1852#ifdef CONFIG_X86_64
1853 SetPagePinned(virt_to_page(level3_user_vsyscall));
1854#endif
1855 xen_mark_init_mm_pinned();
1856}
1857
Jeremy Fitzhardingeb407fc52009-02-17 23:46:21 -08001858static void xen_leave_lazy_mmu(void)
1859{
Jeremy Fitzhardinge5caecb92009-02-20 23:01:26 -08001860 preempt_disable();
Jeremy Fitzhardingeb407fc52009-02-17 23:46:21 -08001861 xen_mc_flush();
1862 paravirt_leave_lazy_mmu();
Jeremy Fitzhardinge5caecb92009-02-20 23:01:26 -08001863 preempt_enable();
Jeremy Fitzhardingeb407fc52009-02-17 23:46:21 -08001864}
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001865
Thomas Gleixner030cb6c2009-08-20 14:30:02 +02001866static const struct pv_mmu_ops xen_mmu_ops __initdata = {
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001867 .read_cr2 = xen_read_cr2,
1868 .write_cr2 = xen_write_cr2,
1869
1870 .read_cr3 = xen_read_cr3,
1871 .write_cr3 = xen_write_cr3,
1872
1873 .flush_tlb_user = xen_flush_tlb,
1874 .flush_tlb_kernel = xen_flush_tlb,
1875 .flush_tlb_single = xen_flush_tlb_single,
1876 .flush_tlb_others = xen_flush_tlb_others,
1877
1878 .pte_update = paravirt_nop,
1879 .pte_update_defer = paravirt_nop,
1880
1881 .pgd_alloc = xen_pgd_alloc,
1882 .pgd_free = xen_pgd_free,
1883
1884 .alloc_pte = xen_alloc_pte_init,
1885 .release_pte = xen_release_pte_init,
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001886 .alloc_pmd = xen_alloc_pmd_init,
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001887 .alloc_pmd_clone = paravirt_nop,
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001888 .release_pmd = xen_release_pmd_init,
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001889
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001890#ifdef CONFIG_X86_64
1891 .set_pte = xen_set_pte,
1892#else
1893 .set_pte = xen_set_pte_init,
1894#endif
1895 .set_pte_at = xen_set_pte_at,
1896 .set_pmd = xen_set_pmd_hyper,
1897
1898 .ptep_modify_prot_start = __ptep_modify_prot_start,
1899 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
1900
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -08001901 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
1902 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001903
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -08001904 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
1905 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001906
1907#ifdef CONFIG_X86_PAE
1908 .set_pte_atomic = xen_set_pte_atomic,
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001909 .pte_clear = xen_pte_clear,
1910 .pmd_clear = xen_pmd_clear,
1911#endif /* CONFIG_X86_PAE */
1912 .set_pud = xen_set_pud_hyper,
1913
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -08001914 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
1915 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001916
1917#if PAGETABLE_LEVELS == 4
Jeremy Fitzhardingeda5de7c2009-01-28 14:35:07 -08001918 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
1919 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001920 .set_pgd = xen_set_pgd_hyper,
1921
Jeremy Fitzhardingeb96229b2009-03-17 13:30:55 -07001922 .alloc_pud = xen_alloc_pmd_init,
1923 .release_pud = xen_release_pmd_init,
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001924#endif /* PAGETABLE_LEVELS == 4 */
1925
1926 .activate_mm = xen_activate_mm,
1927 .dup_mmap = xen_dup_mmap,
1928 .exit_mmap = xen_exit_mmap,
1929
1930 .lazy_mode = {
1931 .enter = paravirt_enter_lazy_mmu,
Jeremy Fitzhardingeb407fc52009-02-17 23:46:21 -08001932 .leave = xen_leave_lazy_mmu,
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001933 },
1934
1935 .set_fixmap = xen_set_fixmap,
1936};
1937
Thomas Gleixner030cb6c2009-08-20 14:30:02 +02001938void __init xen_init_mmu_ops(void)
1939{
1940 x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start;
1941 x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done;
1942 pv_mmu_ops = xen_mmu_ops;
1943}
Jeremy Fitzhardinge319f3ba2009-01-28 14:35:01 -08001944
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001945#ifdef CONFIG_XEN_PVHVM
Stefano Stabellini59151002010-06-17 14:22:52 +01001946static void xen_hvm_exit_mmap(struct mm_struct *mm)
1947{
1948 struct xen_hvm_pagetable_dying a;
1949 int rc;
1950
1951 a.domid = DOMID_SELF;
1952 a.gpa = __pa(mm->pgd);
1953 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
1954 WARN_ON_ONCE(rc < 0);
1955}
1956
1957static int is_pagetable_dying_supported(void)
1958{
1959 struct xen_hvm_pagetable_dying a;
1960 int rc = 0;
1961
1962 a.domid = DOMID_SELF;
1963 a.gpa = 0x00;
1964 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
1965 if (rc < 0) {
1966 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
1967 return 0;
1968 }
1969 return 1;
1970}
1971
1972void __init xen_hvm_init_mmu_ops(void)
1973{
1974 if (is_pagetable_dying_supported())
1975 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
1976}
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001977#endif
Stefano Stabellini59151002010-06-17 14:22:52 +01001978
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -07001979#ifdef CONFIG_XEN_DEBUG_FS
1980
1981static struct dentry *d_mmu_debug;
1982
1983static int __init xen_mmu_debugfs(void)
1984{
1985 struct dentry *d_xen = xen_init_debugfs();
1986
1987 if (d_xen == NULL)
1988 return -ENOMEM;
1989
1990 d_mmu_debug = debugfs_create_dir("mmu", d_xen);
1991
1992 debugfs_create_u8("zero_stats", 0644, d_mmu_debug, &zero_stats);
1993
1994 debugfs_create_u32("pgd_update", 0444, d_mmu_debug, &mmu_stats.pgd_update);
1995 debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug,
1996 &mmu_stats.pgd_update_pinned);
1997 debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug,
1998 &mmu_stats.pgd_update_pinned);
1999
2000 debugfs_create_u32("pud_update", 0444, d_mmu_debug, &mmu_stats.pud_update);
2001 debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug,
2002 &mmu_stats.pud_update_pinned);
2003 debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug,
2004 &mmu_stats.pud_update_pinned);
2005
2006 debugfs_create_u32("pmd_update", 0444, d_mmu_debug, &mmu_stats.pmd_update);
2007 debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug,
2008 &mmu_stats.pmd_update_pinned);
2009 debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug,
2010 &mmu_stats.pmd_update_pinned);
2011
2012 debugfs_create_u32("pte_update", 0444, d_mmu_debug, &mmu_stats.pte_update);
2013// debugfs_create_u32("pte_update_pinned", 0444, d_mmu_debug,
2014// &mmu_stats.pte_update_pinned);
2015 debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug,
2016 &mmu_stats.pte_update_pinned);
2017
2018 debugfs_create_u32("mmu_update", 0444, d_mmu_debug, &mmu_stats.mmu_update);
2019 debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug,
2020 &mmu_stats.mmu_update_extended);
2021 xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug,
2022 mmu_stats.mmu_update_histo, 20);
2023
2024 debugfs_create_u32("set_pte_at", 0444, d_mmu_debug, &mmu_stats.set_pte_at);
2025 debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug,
2026 &mmu_stats.set_pte_at_batched);
2027 debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug,
2028 &mmu_stats.set_pte_at_current);
2029 debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug,
2030 &mmu_stats.set_pte_at_kernel);
2031
2032 debugfs_create_u32("prot_commit", 0444, d_mmu_debug, &mmu_stats.prot_commit);
2033 debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug,
2034 &mmu_stats.prot_commit_batched);
2035
2036 return 0;
2037}
2038fs_initcall(xen_mmu_debugfs);
2039
2040#endif /* CONFIG_XEN_DEBUG_FS */