blob: ddc81a06edb94dc2ce306eb93cb136f59b6edfef [file] [log] [blame]
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -08001/*
2 * Xen leaves the responsibility for maintaining p2m mappings to the
3 * guests themselves, but it must also access and update the p2m array
4 * during suspend/resume when all the pages are reallocated.
5 *
6 * The p2m table is logically a flat array, but we implement it as a
7 * three-level tree to allow the address space to be sparse.
8 *
9 * Xen
10 * |
11 * p2m_top p2m_top_mfn
12 * / \ / \
13 * p2m_mid p2m_mid p2m_mid_mfn p2m_mid_mfn
14 * / \ / \ / /
15 * p2m p2m p2m p2m p2m p2m p2m ...
16 *
17 * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p.
18 *
19 * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the
20 * maximum representable pseudo-physical address space is:
21 * P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages
22 *
23 * P2M_PER_PAGE depends on the architecture, as a mfn is always
24 * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
25 * 512 and 1024 entries respectively.
26 */
27
28#include <linux/init.h>
29#include <linux/module.h>
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +000030#include <linux/list.h>
31#include <linux/hash.h>
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +000032#include <linux/sched.h>
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -080033
34#include <asm/cache.h>
35#include <asm/setup.h>
36
37#include <asm/xen/page.h>
38#include <asm/xen/hypercall.h>
39#include <asm/xen/hypervisor.h>
40
41#include "xen-ops.h"
42
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +000043static void __init m2p_override_init(void);
44
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -080045unsigned long xen_max_p2m_pfn __read_mostly;
46
47#define P2M_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
48#define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *))
49#define P2M_TOP_PER_PAGE (PAGE_SIZE / sizeof(unsigned long **))
50
51#define MAX_P2M_PFN (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
52
53/* Placeholders for holes in the address space */
54static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
55static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);
56static RESERVE_BRK_ARRAY(unsigned long, p2m_mid_missing_mfn, P2M_MID_PER_PAGE);
57
58static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE);
59static RESERVE_BRK_ARRAY(unsigned long, p2m_top_mfn, P2M_TOP_PER_PAGE);
60static RESERVE_BRK_ARRAY(unsigned long *, p2m_top_mfn_p, P2M_TOP_PER_PAGE);
61
62RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
63RESERVE_BRK(p2m_mid_mfn, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
64
65static inline unsigned p2m_top_index(unsigned long pfn)
66{
67 BUG_ON(pfn >= MAX_P2M_PFN);
68 return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);
69}
70
71static inline unsigned p2m_mid_index(unsigned long pfn)
72{
73 return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;
74}
75
76static inline unsigned p2m_index(unsigned long pfn)
77{
78 return pfn % P2M_PER_PAGE;
79}
80
81static void p2m_top_init(unsigned long ***top)
82{
83 unsigned i;
84
85 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
86 top[i] = p2m_mid_missing;
87}
88
89static void p2m_top_mfn_init(unsigned long *top)
90{
91 unsigned i;
92
93 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
94 top[i] = virt_to_mfn(p2m_mid_missing_mfn);
95}
96
97static void p2m_top_mfn_p_init(unsigned long **top)
98{
99 unsigned i;
100
101 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
102 top[i] = p2m_mid_missing_mfn;
103}
104
105static void p2m_mid_init(unsigned long **mid)
106{
107 unsigned i;
108
109 for (i = 0; i < P2M_MID_PER_PAGE; i++)
110 mid[i] = p2m_missing;
111}
112
113static void p2m_mid_mfn_init(unsigned long *mid)
114{
115 unsigned i;
116
117 for (i = 0; i < P2M_MID_PER_PAGE; i++)
118 mid[i] = virt_to_mfn(p2m_missing);
119}
120
121static void p2m_init(unsigned long *p2m)
122{
123 unsigned i;
124
125 for (i = 0; i < P2M_MID_PER_PAGE; i++)
126 p2m[i] = INVALID_P2M_ENTRY;
127}
128
129/*
130 * Build the parallel p2m_top_mfn and p2m_mid_mfn structures
131 *
132 * This is called both at boot time, and after resuming from suspend:
133 * - At boot time we're called very early, and must use extend_brk()
134 * to allocate memory.
135 *
136 * - After resume we're called from within stop_machine, but the mfn
137 * tree should alreay be completely allocated.
138 */
139void xen_build_mfn_list_list(void)
140{
141 unsigned long pfn;
142
143 /* Pre-initialize p2m_top_mfn to be completely missing */
144 if (p2m_top_mfn == NULL) {
145 p2m_mid_missing_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
146 p2m_mid_mfn_init(p2m_mid_missing_mfn);
147
148 p2m_top_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
149 p2m_top_mfn_p_init(p2m_top_mfn_p);
150
151 p2m_top_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
152 p2m_top_mfn_init(p2m_top_mfn);
153 } else {
154 /* Reinitialise, mfn's all change after migration */
155 p2m_mid_mfn_init(p2m_mid_missing_mfn);
156 }
157
158 for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) {
159 unsigned topidx = p2m_top_index(pfn);
160 unsigned mididx = p2m_mid_index(pfn);
161 unsigned long **mid;
162 unsigned long *mid_mfn_p;
163
164 mid = p2m_top[topidx];
165 mid_mfn_p = p2m_top_mfn_p[topidx];
166
167 /* Don't bother allocating any mfn mid levels if
168 * they're just missing, just update the stored mfn,
169 * since all could have changed over a migrate.
170 */
171 if (mid == p2m_mid_missing) {
172 BUG_ON(mididx);
173 BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
174 p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn);
175 pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;
176 continue;
177 }
178
179 if (mid_mfn_p == p2m_mid_missing_mfn) {
180 /*
181 * XXX boot-time only! We should never find
182 * missing parts of the mfn tree after
183 * runtime. extend_brk() will BUG if we call
184 * it too late.
185 */
186 mid_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
187 p2m_mid_mfn_init(mid_mfn_p);
188
189 p2m_top_mfn_p[topidx] = mid_mfn_p;
190 }
191
192 p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
193 mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]);
194 }
195}
196
197void xen_setup_mfn_list_list(void)
198{
199 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
200
201 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
202 virt_to_mfn(p2m_top_mfn);
203 HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
204}
205
206/* Set up p2m_top to point to the domain-builder provided p2m pages */
207void __init xen_build_dynamic_phys_to_machine(void)
208{
209 unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
210 unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
211 unsigned long pfn;
212
213 xen_max_p2m_pfn = max_pfn;
214
215 p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
216 p2m_init(p2m_missing);
217
218 p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
219 p2m_mid_init(p2m_mid_missing);
220
221 p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE);
222 p2m_top_init(p2m_top);
223
224 /*
225 * The domain builder gives us a pre-constructed p2m array in
226 * mfn_list for all the pages initially given to us, so we just
227 * need to graft that into our tree structure.
228 */
229 for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) {
230 unsigned topidx = p2m_top_index(pfn);
231 unsigned mididx = p2m_mid_index(pfn);
232
233 if (p2m_top[topidx] == p2m_mid_missing) {
234 unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
235 p2m_mid_init(mid);
236
237 p2m_top[topidx] = mid;
238 }
239
Stefan Bader8e1b4cf2011-01-20 15:38:23 +0100240 /*
241 * As long as the mfn_list has enough entries to completely
242 * fill a p2m page, pointing into the array is ok. But if
243 * not the entries beyond the last pfn will be undefined.
244 * And guessing that the 'what-ever-there-is' does not take it
245 * too kindly when changing it to invalid markers, a new page
246 * is allocated, initialized and filled with the valid part.
247 */
248 if (unlikely(pfn + P2M_PER_PAGE > max_pfn)) {
249 unsigned long p2midx;
250 unsigned long *p2m = extend_brk(PAGE_SIZE, PAGE_SIZE);
251 p2m_init(p2m);
252
253 for (p2midx = 0; pfn + p2midx < max_pfn; p2midx++) {
254 p2m[p2midx] = mfn_list[pfn + p2midx];
255 }
256 p2m_top[topidx][mididx] = p2m;
257 } else
258 p2m_top[topidx][mididx] = &mfn_list[pfn];
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800259 }
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000260
261 m2p_override_init();
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800262}
263
264unsigned long get_phys_to_machine(unsigned long pfn)
265{
266 unsigned topidx, mididx, idx;
267
268 if (unlikely(pfn >= MAX_P2M_PFN))
269 return INVALID_P2M_ENTRY;
270
271 topidx = p2m_top_index(pfn);
272 mididx = p2m_mid_index(pfn);
273 idx = p2m_index(pfn);
274
275 return p2m_top[topidx][mididx][idx];
276}
277EXPORT_SYMBOL_GPL(get_phys_to_machine);
278
279static void *alloc_p2m_page(void)
280{
281 return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT);
282}
283
284static void free_p2m_page(void *p)
285{
286 free_page((unsigned long)p);
287}
288
289/*
290 * Fully allocate the p2m structure for a given pfn. We need to check
291 * that both the top and mid levels are allocated, and make sure the
292 * parallel mfn tree is kept in sync. We may race with other cpus, so
293 * the new pages are installed with cmpxchg; if we lose the race then
294 * simply free the page we allocated and use the one that's there.
295 */
296static bool alloc_p2m(unsigned long pfn)
297{
298 unsigned topidx, mididx;
299 unsigned long ***top_p, **mid;
300 unsigned long *top_mfn_p, *mid_mfn;
301
302 topidx = p2m_top_index(pfn);
303 mididx = p2m_mid_index(pfn);
304
305 top_p = &p2m_top[topidx];
306 mid = *top_p;
307
308 if (mid == p2m_mid_missing) {
309 /* Mid level is missing, allocate a new one */
310 mid = alloc_p2m_page();
311 if (!mid)
312 return false;
313
314 p2m_mid_init(mid);
315
316 if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
317 free_p2m_page(mid);
318 }
319
320 top_mfn_p = &p2m_top_mfn[topidx];
321 mid_mfn = p2m_top_mfn_p[topidx];
322
323 BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p);
324
325 if (mid_mfn == p2m_mid_missing_mfn) {
326 /* Separately check the mid mfn level */
327 unsigned long missing_mfn;
328 unsigned long mid_mfn_mfn;
329
330 mid_mfn = alloc_p2m_page();
331 if (!mid_mfn)
332 return false;
333
334 p2m_mid_mfn_init(mid_mfn);
335
336 missing_mfn = virt_to_mfn(p2m_mid_missing_mfn);
337 mid_mfn_mfn = virt_to_mfn(mid_mfn);
338 if (cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn) != missing_mfn)
339 free_p2m_page(mid_mfn);
340 else
341 p2m_top_mfn_p[topidx] = mid_mfn;
342 }
343
344 if (p2m_top[topidx][mididx] == p2m_missing) {
345 /* p2m leaf page is missing */
346 unsigned long *p2m;
347
348 p2m = alloc_p2m_page();
349 if (!p2m)
350 return false;
351
352 p2m_init(p2m);
353
354 if (cmpxchg(&mid[mididx], p2m_missing, p2m) != p2m_missing)
355 free_p2m_page(p2m);
356 else
357 mid_mfn[mididx] = virt_to_mfn(p2m);
358 }
359
360 return true;
361}
362
363/* Try to install p2m mapping; fail if intermediate bits missing */
364bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
365{
366 unsigned topidx, mididx, idx;
367
368 if (unlikely(pfn >= MAX_P2M_PFN)) {
369 BUG_ON(mfn != INVALID_P2M_ENTRY);
370 return true;
371 }
372
373 topidx = p2m_top_index(pfn);
374 mididx = p2m_mid_index(pfn);
375 idx = p2m_index(pfn);
376
377 if (p2m_top[topidx][mididx] == p2m_missing)
378 return mfn == INVALID_P2M_ENTRY;
379
380 p2m_top[topidx][mididx][idx] = mfn;
381
382 return true;
383}
384
385bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
386{
387 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
388 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
389 return true;
390 }
391
392 if (unlikely(!__set_phys_to_machine(pfn, mfn))) {
393 if (!alloc_p2m(pfn))
394 return false;
395
396 if (!__set_phys_to_machine(pfn, mfn))
397 return false;
398 }
399
400 return true;
401}
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000402
403#define M2P_OVERRIDE_HASH_SHIFT 10
404#define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT)
405
406static RESERVE_BRK_ARRAY(struct list_head, m2p_overrides, M2P_OVERRIDE_HASH);
407static DEFINE_SPINLOCK(m2p_override_lock);
408
409static void __init m2p_override_init(void)
410{
411 unsigned i;
412
413 m2p_overrides = extend_brk(sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH,
414 sizeof(unsigned long));
415
416 for (i = 0; i < M2P_OVERRIDE_HASH; i++)
417 INIT_LIST_HEAD(&m2p_overrides[i]);
418}
419
420static unsigned long mfn_hash(unsigned long mfn)
421{
422 return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
423}
424
425/* Add an MFN override for a particular page */
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000426int m2p_add_override(unsigned long mfn, struct page *page)
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000427{
428 unsigned long flags;
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000429 unsigned long pfn;
430 unsigned long address;
431 unsigned level;
432 pte_t *ptep = NULL;
433
434 pfn = page_to_pfn(page);
435 if (!PageHighMem(page)) {
436 address = (unsigned long)__va(pfn << PAGE_SHIFT);
437 ptep = lookup_address(address, &level);
438
439 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
440 "m2p_add_override: pfn %lx not mapped", pfn))
441 return -EINVAL;
442 }
443
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000444 page->private = mfn;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000445 page->index = pfn_to_mfn(pfn);
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000446
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000447 __set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000448 if (!PageHighMem(page))
449 /* Just zap old mapping for now */
450 pte_clear(&init_mm, address, ptep);
451
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000452 spin_lock_irqsave(&m2p_override_lock, flags);
453 list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]);
454 spin_unlock_irqrestore(&m2p_override_lock, flags);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000455
456 return 0;
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000457}
458
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000459int m2p_remove_override(struct page *page)
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000460{
461 unsigned long flags;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000462 unsigned long mfn;
463 unsigned long pfn;
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000464 unsigned long address;
465 unsigned level;
466 pte_t *ptep = NULL;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000467
468 pfn = page_to_pfn(page);
469 mfn = get_phys_to_machine(pfn);
470 if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT))
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000471 return -EINVAL;
472
473 if (!PageHighMem(page)) {
474 address = (unsigned long)__va(pfn << PAGE_SHIFT);
475 ptep = lookup_address(address, &level);
476
477 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
478 "m2p_remove_override: pfn %lx not mapped", pfn))
479 return -EINVAL;
480 }
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000481
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000482 spin_lock_irqsave(&m2p_override_lock, flags);
483 list_del(&page->lru);
484 spin_unlock_irqrestore(&m2p_override_lock, flags);
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000485 __set_phys_to_machine(pfn, page->index);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000486
487 if (!PageHighMem(page))
488 set_pte_at(&init_mm, address, ptep,
489 pfn_pte(pfn, PAGE_KERNEL));
490 /* No tlb flush necessary because the caller already
491 * left the pte unmapped. */
492
493 return 0;
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000494}
495
496struct page *m2p_find_override(unsigned long mfn)
497{
498 unsigned long flags;
499 struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)];
500 struct page *p, *ret;
501
502 ret = NULL;
503
504 spin_lock_irqsave(&m2p_override_lock, flags);
505
506 list_for_each_entry(p, bucket, lru) {
507 if (p->private == mfn) {
508 ret = p;
509 break;
510 }
511 }
512
513 spin_unlock_irqrestore(&m2p_override_lock, flags);
514
515 return ret;
516}
517
518unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
519{
520 struct page *p = m2p_find_override(mfn);
521 unsigned long ret = pfn;
522
523 if (p)
524 ret = page_to_pfn(p);
525
526 return ret;
527}
Konrad Rzeszutek Wilke1b478e2011-01-11 15:09:16 -0500528EXPORT_SYMBOL_GPL(m2p_find_override_pfn);