blob: 58efeb9d5440b75d2fd6d3cb5880f4ddd2729130 [file] [log] [blame]
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -08001/*
2 * Xen leaves the responsibility for maintaining p2m mappings to the
3 * guests themselves, but it must also access and update the p2m array
4 * during suspend/resume when all the pages are reallocated.
5 *
6 * The p2m table is logically a flat array, but we implement it as a
7 * three-level tree to allow the address space to be sparse.
8 *
9 * Xen
10 * |
11 * p2m_top p2m_top_mfn
12 * / \ / \
13 * p2m_mid p2m_mid p2m_mid_mfn p2m_mid_mfn
14 * / \ / \ / /
15 * p2m p2m p2m p2m p2m p2m p2m ...
16 *
17 * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p.
18 *
19 * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the
20 * maximum representable pseudo-physical address space is:
21 * P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages
22 *
23 * P2M_PER_PAGE depends on the architecture, as a mfn is always
24 * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
25 * 512 and 1024 entries respectively.
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -050026 *
27 * In short, these structures contain the Machine Frame Number (MFN) of the PFN.
28 *
29 * However not all entries are filled with MFNs. Specifically for all other
30 * leaf entries, or for the top root, or middle one, for which there is a void
31 * entry, we assume it is "missing". So (for example)
32 * pfn_to_mfn(0x90909090)=INVALID_P2M_ENTRY.
33 *
34 * We also have the possibility of setting 1-1 mappings on certain regions, so
35 * that:
36 * pfn_to_mfn(0xc0000)=0xc0000
37 *
38 * The benefit of this is, that we can assume for non-RAM regions (think
39 * PCI BARs, or ACPI spaces), we can create mappings easily b/c we
40 * get the PFN value to match the MFN.
41 *
42 * For this to work efficiently we have one new page p2m_identity and
43 * allocate (via reserved_brk) any other pages we need to cover the sides
44 * (1GB or 4MB boundary violations). All entries in p2m_identity are set to
45 * INVALID_P2M_ENTRY type (Xen toolstack only recognizes that and MFNs,
46 * no other fancy value).
47 *
48 * On lookup we spot that the entry points to p2m_identity and return the
49 * identity value instead of dereferencing and returning INVALID_P2M_ENTRY.
50 * If the entry points to an allocated page, we just proceed as before and
51 * return the PFN. If the PFN has IDENTITY_FRAME_BIT set we unmask that in
52 * appropriate functions (pfn_to_mfn).
53 *
54 * The reason for having the IDENTITY_FRAME_BIT instead of just returning the
55 * PFN is that we could find ourselves where pfn_to_mfn(pfn)==pfn for a
56 * non-identity pfn. To protect ourselves against we elect to set (and get) the
57 * IDENTITY_FRAME_BIT on all identity mapped PFNs.
58 *
59 * This simplistic diagram is used to explain the more subtle piece of code.
60 * There is also a digram of the P2M at the end that can help.
61 * Imagine your E820 looking as so:
62 *
63 * 1GB 2GB
64 * /-------------------+---------\/----\ /----------\ /---+-----\
65 * | System RAM | Sys RAM ||ACPI| | reserved | | Sys RAM |
66 * \-------------------+---------/\----/ \----------/ \---+-----/
67 * ^- 1029MB ^- 2001MB
68 *
69 * [1029MB = 263424 (0x40500), 2001MB = 512256 (0x7D100),
70 * 2048MB = 524288 (0x80000)]
71 *
72 * And dom0_mem=max:3GB,1GB is passed in to the guest, meaning memory past 1GB
73 * is actually not present (would have to kick the balloon driver to put it in).
74 *
75 * When we are told to set the PFNs for identity mapping (see patch: "xen/setup:
76 * Set identity mapping for non-RAM E820 and E820 gaps.") we pass in the start
77 * of the PFN and the end PFN (263424 and 512256 respectively). The first step
78 * is to reserve_brk a top leaf page if the p2m[1] is missing. The top leaf page
79 * covers 512^2 of page estate (1GB) and in case the start or end PFN is not
80 * aligned on 512^2*PAGE_SIZE (1GB) we loop on aligned 1GB PFNs from start pfn
81 * to end pfn. We reserve_brk top leaf pages if they are missing (means they
82 * point to p2m_mid_missing).
83 *
84 * With the E820 example above, 263424 is not 1GB aligned so we allocate a
85 * reserve_brk page which will cover the PFNs estate from 0x40000 to 0x80000.
86 * Each entry in the allocate page is "missing" (points to p2m_missing).
87 *
88 * Next stage is to determine if we need to do a more granular boundary check
89 * on the 4MB (or 2MB depending on architecture) off the start and end pfn's.
90 * We check if the start pfn and end pfn violate that boundary check, and if
91 * so reserve_brk a middle (p2m[x][y]) leaf page. This way we have a much finer
92 * granularity of setting which PFNs are missing and which ones are identity.
93 * In our example 263424 and 512256 both fail the check so we reserve_brk two
94 * pages. Populate them with INVALID_P2M_ENTRY (so they both have "missing"
95 * values) and assign them to p2m[1][2] and p2m[1][488] respectively.
96 *
97 * At this point we would at minimum reserve_brk one page, but could be up to
98 * three. Each call to set_phys_range_identity has at maximum a three page
99 * cost. If we were to query the P2M at this stage, all those entries from
100 * start PFN through end PFN (so 1029MB -> 2001MB) would return
101 * INVALID_P2M_ENTRY ("missing").
102 *
103 * The next step is to walk from the start pfn to the end pfn setting
104 * the IDENTITY_FRAME_BIT on each PFN. This is done in set_phys_range_identity.
105 * If we find that the middle leaf is pointing to p2m_missing we can swap it
106 * over to p2m_identity - this way covering 4MB (or 2MB) PFN space. At this
107 * point we do not need to worry about boundary aligment (so no need to
108 * reserve_brk a middle page, figure out which PFNs are "missing" and which
109 * ones are identity), as that has been done earlier. If we find that the
110 * middle leaf is not occupied by p2m_identity or p2m_missing, we dereference
111 * that page (which covers 512 PFNs) and set the appropriate PFN with
112 * IDENTITY_FRAME_BIT. In our example 263424 and 512256 end up there, and we
113 * set from p2m[1][2][256->511] and p2m[1][488][0->256] with
114 * IDENTITY_FRAME_BIT set.
115 *
116 * All other regions that are void (or not filled) either point to p2m_missing
117 * (considered missing) or have the default value of INVALID_P2M_ENTRY (also
118 * considered missing). In our case, p2m[1][2][0->255] and p2m[1][488][257->511]
119 * contain the INVALID_P2M_ENTRY value and are considered "missing."
120 *
121 * This is what the p2m ends up looking (for the E820 above) with this
122 * fabulous drawing:
123 *
124 * p2m /--------------\
125 * /-----\ | &mfn_list[0],| /-----------------\
126 * | 0 |------>| &mfn_list[1],| /---------------\ | ~0, ~0, .. |
127 * |-----| | ..., ~0, ~0 | | ~0, ~0, [x]---+----->| IDENTITY [@256] |
128 * | 1 |---\ \--------------/ | [p2m_identity]+\ | IDENTITY [@257] |
129 * |-----| \ | [p2m_identity]+\\ | .... |
130 * | 2 |--\ \-------------------->| ... | \\ \----------------/
131 * |-----| \ \---------------/ \\
132 * | 3 |\ \ \\ p2m_identity
133 * |-----| \ \-------------------->/---------------\ /-----------------\
134 * | .. +->+ | [p2m_identity]+-->| ~0, ~0, ~0, ... |
135 * \-----/ / | [p2m_identity]+-->| ..., ~0 |
136 * / /---------------\ | .... | \-----------------/
137 * / | IDENTITY[@0] | /-+-[x], ~0, ~0.. |
138 * / | IDENTITY[@256]|<----/ \---------------/
139 * / | ~0, ~0, .... |
140 * | \---------------/
141 * |
142 * p2m_missing p2m_missing
143 * /------------------\ /------------\
144 * | [p2m_mid_missing]+---->| ~0, ~0, ~0 |
145 * | [p2m_mid_missing]+---->| ..., ~0 |
146 * \------------------/ \------------/
147 *
148 * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT)
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800149 */
150
151#include <linux/init.h>
152#include <linux/module.h>
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000153#include <linux/list.h>
154#include <linux/hash.h>
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000155#include <linux/sched.h>
Konrad Rzeszutek Wilk2222e712010-12-22 08:57:30 -0500156#include <linux/seq_file.h>
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800157
158#include <asm/cache.h>
159#include <asm/setup.h>
160
161#include <asm/xen/page.h>
162#include <asm/xen/hypercall.h>
163#include <asm/xen/hypervisor.h>
164
165#include "xen-ops.h"
166
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000167static void __init m2p_override_init(void);
168
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800169unsigned long xen_max_p2m_pfn __read_mostly;
170
171#define P2M_PER_PAGE (PAGE_SIZE / sizeof(unsigned long))
172#define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *))
173#define P2M_TOP_PER_PAGE (PAGE_SIZE / sizeof(unsigned long **))
174
175#define MAX_P2M_PFN (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
176
177/* Placeholders for holes in the address space */
178static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
179static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);
180static RESERVE_BRK_ARRAY(unsigned long, p2m_mid_missing_mfn, P2M_MID_PER_PAGE);
181
182static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE);
183static RESERVE_BRK_ARRAY(unsigned long, p2m_top_mfn, P2M_TOP_PER_PAGE);
184static RESERVE_BRK_ARRAY(unsigned long *, p2m_top_mfn_p, P2M_TOP_PER_PAGE);
185
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500186static RESERVE_BRK_ARRAY(unsigned long, p2m_identity, P2M_PER_PAGE);
187
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800188RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
189RESERVE_BRK(p2m_mid_mfn, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
190
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500191/* We might hit two boundary violations at the start and end, at max each
192 * boundary violation will require three middle nodes. */
193RESERVE_BRK(p2m_mid_identity, PAGE_SIZE * 2 * 3);
194
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800195static inline unsigned p2m_top_index(unsigned long pfn)
196{
197 BUG_ON(pfn >= MAX_P2M_PFN);
198 return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);
199}
200
201static inline unsigned p2m_mid_index(unsigned long pfn)
202{
203 return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;
204}
205
206static inline unsigned p2m_index(unsigned long pfn)
207{
208 return pfn % P2M_PER_PAGE;
209}
210
211static void p2m_top_init(unsigned long ***top)
212{
213 unsigned i;
214
215 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
216 top[i] = p2m_mid_missing;
217}
218
219static void p2m_top_mfn_init(unsigned long *top)
220{
221 unsigned i;
222
223 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
224 top[i] = virt_to_mfn(p2m_mid_missing_mfn);
225}
226
227static void p2m_top_mfn_p_init(unsigned long **top)
228{
229 unsigned i;
230
231 for (i = 0; i < P2M_TOP_PER_PAGE; i++)
232 top[i] = p2m_mid_missing_mfn;
233}
234
235static void p2m_mid_init(unsigned long **mid)
236{
237 unsigned i;
238
239 for (i = 0; i < P2M_MID_PER_PAGE; i++)
240 mid[i] = p2m_missing;
241}
242
243static void p2m_mid_mfn_init(unsigned long *mid)
244{
245 unsigned i;
246
247 for (i = 0; i < P2M_MID_PER_PAGE; i++)
248 mid[i] = virt_to_mfn(p2m_missing);
249}
250
251static void p2m_init(unsigned long *p2m)
252{
253 unsigned i;
254
255 for (i = 0; i < P2M_MID_PER_PAGE; i++)
256 p2m[i] = INVALID_P2M_ENTRY;
257}
258
259/*
260 * Build the parallel p2m_top_mfn and p2m_mid_mfn structures
261 *
262 * This is called both at boot time, and after resuming from suspend:
263 * - At boot time we're called very early, and must use extend_brk()
264 * to allocate memory.
265 *
266 * - After resume we're called from within stop_machine, but the mfn
267 * tree should alreay be completely allocated.
268 */
Ian Campbell44b46c32011-02-11 16:37:41 +0000269void __ref xen_build_mfn_list_list(void)
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800270{
271 unsigned long pfn;
272
273 /* Pre-initialize p2m_top_mfn to be completely missing */
274 if (p2m_top_mfn == NULL) {
275 p2m_mid_missing_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
276 p2m_mid_mfn_init(p2m_mid_missing_mfn);
277
278 p2m_top_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
279 p2m_top_mfn_p_init(p2m_top_mfn_p);
280
281 p2m_top_mfn = extend_brk(PAGE_SIZE, PAGE_SIZE);
282 p2m_top_mfn_init(p2m_top_mfn);
283 } else {
284 /* Reinitialise, mfn's all change after migration */
285 p2m_mid_mfn_init(p2m_mid_missing_mfn);
286 }
287
288 for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) {
289 unsigned topidx = p2m_top_index(pfn);
290 unsigned mididx = p2m_mid_index(pfn);
291 unsigned long **mid;
292 unsigned long *mid_mfn_p;
293
294 mid = p2m_top[topidx];
295 mid_mfn_p = p2m_top_mfn_p[topidx];
296
297 /* Don't bother allocating any mfn mid levels if
298 * they're just missing, just update the stored mfn,
299 * since all could have changed over a migrate.
300 */
301 if (mid == p2m_mid_missing) {
302 BUG_ON(mididx);
303 BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
304 p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn);
305 pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;
306 continue;
307 }
308
309 if (mid_mfn_p == p2m_mid_missing_mfn) {
310 /*
311 * XXX boot-time only! We should never find
312 * missing parts of the mfn tree after
313 * runtime. extend_brk() will BUG if we call
314 * it too late.
315 */
316 mid_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
317 p2m_mid_mfn_init(mid_mfn_p);
318
319 p2m_top_mfn_p[topidx] = mid_mfn_p;
320 }
321
322 p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
323 mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]);
324 }
325}
326
327void xen_setup_mfn_list_list(void)
328{
329 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
330
331 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
332 virt_to_mfn(p2m_top_mfn);
333 HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
334}
335
336/* Set up p2m_top to point to the domain-builder provided p2m pages */
337void __init xen_build_dynamic_phys_to_machine(void)
338{
339 unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
340 unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
341 unsigned long pfn;
342
343 xen_max_p2m_pfn = max_pfn;
344
345 p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
346 p2m_init(p2m_missing);
347
348 p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
349 p2m_mid_init(p2m_mid_missing);
350
351 p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE);
352 p2m_top_init(p2m_top);
353
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500354 p2m_identity = extend_brk(PAGE_SIZE, PAGE_SIZE);
355 p2m_init(p2m_identity);
356
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800357 /*
358 * The domain builder gives us a pre-constructed p2m array in
359 * mfn_list for all the pages initially given to us, so we just
360 * need to graft that into our tree structure.
361 */
362 for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) {
363 unsigned topidx = p2m_top_index(pfn);
364 unsigned mididx = p2m_mid_index(pfn);
365
366 if (p2m_top[topidx] == p2m_mid_missing) {
367 unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
368 p2m_mid_init(mid);
369
370 p2m_top[topidx] = mid;
371 }
372
Stefan Bader8e1b4cf2011-01-20 15:38:23 +0100373 /*
374 * As long as the mfn_list has enough entries to completely
375 * fill a p2m page, pointing into the array is ok. But if
376 * not the entries beyond the last pfn will be undefined.
Stefan Bader8e1b4cf2011-01-20 15:38:23 +0100377 */
378 if (unlikely(pfn + P2M_PER_PAGE > max_pfn)) {
379 unsigned long p2midx;
Stefan Bader8e1b4cf2011-01-20 15:38:23 +0100380
Stefan Badercf04d122011-01-27 10:03:14 -0500381 p2midx = max_pfn % P2M_PER_PAGE;
382 for ( ; p2midx < P2M_PER_PAGE; p2midx++)
383 mfn_list[pfn + p2midx] = INVALID_P2M_ENTRY;
384 }
385 p2m_top[topidx][mididx] = &mfn_list[pfn];
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800386 }
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000387
388 m2p_override_init();
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800389}
390
391unsigned long get_phys_to_machine(unsigned long pfn)
392{
393 unsigned topidx, mididx, idx;
394
395 if (unlikely(pfn >= MAX_P2M_PFN))
396 return INVALID_P2M_ENTRY;
397
398 topidx = p2m_top_index(pfn);
399 mididx = p2m_mid_index(pfn);
400 idx = p2m_index(pfn);
401
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500402 /*
403 * The INVALID_P2M_ENTRY is filled in both p2m_*identity
404 * and in p2m_*missing, so returning the INVALID_P2M_ENTRY
405 * would be wrong.
406 */
407 if (p2m_top[topidx][mididx] == p2m_identity)
408 return IDENTITY_FRAME(pfn);
409
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800410 return p2m_top[topidx][mididx][idx];
411}
412EXPORT_SYMBOL_GPL(get_phys_to_machine);
413
414static void *alloc_p2m_page(void)
415{
416 return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT);
417}
418
419static void free_p2m_page(void *p)
420{
421 free_page((unsigned long)p);
422}
423
424/*
425 * Fully allocate the p2m structure for a given pfn. We need to check
426 * that both the top and mid levels are allocated, and make sure the
427 * parallel mfn tree is kept in sync. We may race with other cpus, so
428 * the new pages are installed with cmpxchg; if we lose the race then
429 * simply free the page we allocated and use the one that's there.
430 */
431static bool alloc_p2m(unsigned long pfn)
432{
433 unsigned topidx, mididx;
434 unsigned long ***top_p, **mid;
435 unsigned long *top_mfn_p, *mid_mfn;
436
437 topidx = p2m_top_index(pfn);
438 mididx = p2m_mid_index(pfn);
439
440 top_p = &p2m_top[topidx];
441 mid = *top_p;
442
443 if (mid == p2m_mid_missing) {
444 /* Mid level is missing, allocate a new one */
445 mid = alloc_p2m_page();
446 if (!mid)
447 return false;
448
449 p2m_mid_init(mid);
450
451 if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
452 free_p2m_page(mid);
453 }
454
455 top_mfn_p = &p2m_top_mfn[topidx];
456 mid_mfn = p2m_top_mfn_p[topidx];
457
458 BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p);
459
460 if (mid_mfn == p2m_mid_missing_mfn) {
461 /* Separately check the mid mfn level */
462 unsigned long missing_mfn;
463 unsigned long mid_mfn_mfn;
464
465 mid_mfn = alloc_p2m_page();
466 if (!mid_mfn)
467 return false;
468
469 p2m_mid_mfn_init(mid_mfn);
470
471 missing_mfn = virt_to_mfn(p2m_mid_missing_mfn);
472 mid_mfn_mfn = virt_to_mfn(mid_mfn);
473 if (cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn) != missing_mfn)
474 free_p2m_page(mid_mfn);
475 else
476 p2m_top_mfn_p[topidx] = mid_mfn;
477 }
478
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500479 if (p2m_top[topidx][mididx] == p2m_identity ||
480 p2m_top[topidx][mididx] == p2m_missing) {
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800481 /* p2m leaf page is missing */
482 unsigned long *p2m;
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500483 unsigned long *p2m_orig = p2m_top[topidx][mididx];
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800484
485 p2m = alloc_p2m_page();
486 if (!p2m)
487 return false;
488
489 p2m_init(p2m);
490
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500491 if (cmpxchg(&mid[mididx], p2m_orig, p2m) != p2m_orig)
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800492 free_p2m_page(p2m);
493 else
494 mid_mfn[mididx] = virt_to_mfn(p2m);
495 }
496
497 return true;
498}
499
Randy Dunlapb83c6e52011-03-24 13:34:32 -0700500static bool __init __early_alloc_p2m(unsigned long pfn)
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500501{
502 unsigned topidx, mididx, idx;
503
504 topidx = p2m_top_index(pfn);
505 mididx = p2m_mid_index(pfn);
506 idx = p2m_index(pfn);
507
508 /* Pfff.. No boundary cross-over, lets get out. */
509 if (!idx)
510 return false;
511
512 WARN(p2m_top[topidx][mididx] == p2m_identity,
513 "P2M[%d][%d] == IDENTITY, should be MISSING (or alloced)!\n",
514 topidx, mididx);
515
516 /*
517 * Could be done by xen_build_dynamic_phys_to_machine..
518 */
519 if (p2m_top[topidx][mididx] != p2m_missing)
520 return false;
521
522 /* Boundary cross-over for the edges: */
523 if (idx) {
524 unsigned long *p2m = extend_brk(PAGE_SIZE, PAGE_SIZE);
Konrad Rzeszutek Wilk8c595082011-04-01 15:18:48 -0400525 unsigned long *mid_mfn_p;
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500526
527 p2m_init(p2m);
528
529 p2m_top[topidx][mididx] = p2m;
530
Konrad Rzeszutek Wilk8c595082011-04-01 15:18:48 -0400531 /* For save/restore we need to MFN of the P2M saved */
532
533 mid_mfn_p = p2m_top_mfn_p[topidx];
534 WARN(mid_mfn_p[mididx] != virt_to_mfn(p2m_missing),
535 "P2M_TOP_P[%d][%d] != MFN of p2m_missing!\n",
536 topidx, mididx);
537 mid_mfn_p[mididx] = virt_to_mfn(p2m);
538
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500539 }
540 return idx != 0;
541}
Randy Dunlapb83c6e52011-03-24 13:34:32 -0700542unsigned long __init set_phys_range_identity(unsigned long pfn_s,
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500543 unsigned long pfn_e)
544{
545 unsigned long pfn;
546
547 if (unlikely(pfn_s >= MAX_P2M_PFN || pfn_e >= MAX_P2M_PFN))
548 return 0;
549
550 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
551 return pfn_e - pfn_s;
552
553 if (pfn_s > pfn_e)
554 return 0;
555
556 for (pfn = (pfn_s & ~(P2M_MID_PER_PAGE * P2M_PER_PAGE - 1));
557 pfn < ALIGN(pfn_e, (P2M_MID_PER_PAGE * P2M_PER_PAGE));
558 pfn += P2M_MID_PER_PAGE * P2M_PER_PAGE)
559 {
560 unsigned topidx = p2m_top_index(pfn);
Konrad Rzeszutek Wilk8c595082011-04-01 15:18:48 -0400561 unsigned long *mid_mfn_p;
562 unsigned long **mid;
563
564 mid = p2m_top[topidx];
565 mid_mfn_p = p2m_top_mfn_p[topidx];
566 if (mid == p2m_mid_missing) {
567 mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500568
569 p2m_mid_init(mid);
570
571 p2m_top[topidx] = mid;
Konrad Rzeszutek Wilk8c595082011-04-01 15:18:48 -0400572
573 BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
574 }
575 /* And the save/restore P2M tables.. */
576 if (mid_mfn_p == p2m_mid_missing_mfn) {
577 mid_mfn_p = extend_brk(PAGE_SIZE, PAGE_SIZE);
578 p2m_mid_mfn_init(mid_mfn_p);
579
580 p2m_top_mfn_p[topidx] = mid_mfn_p;
581 p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
582 /* Note: we don't set mid_mfn_p[midix] here,
583 * look in __early_alloc_p2m */
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500584 }
585 }
586
587 __early_alloc_p2m(pfn_s);
588 __early_alloc_p2m(pfn_e);
589
590 for (pfn = pfn_s; pfn < pfn_e; pfn++)
591 if (!__set_phys_to_machine(pfn, IDENTITY_FRAME(pfn)))
592 break;
593
594 if (!WARN((pfn - pfn_s) != (pfn_e - pfn_s),
595 "Identity mapping failed. We are %ld short of 1-1 mappings!\n",
596 (pfn_e - pfn_s) - (pfn - pfn_s)))
597 printk(KERN_DEBUG "1-1 mapping on %lx->%lx\n", pfn_s, pfn);
598
599 return pfn - pfn_s;
600}
601
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800602/* Try to install p2m mapping; fail if intermediate bits missing */
603bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
604{
605 unsigned topidx, mididx, idx;
606
Konrad Rzeszutek Wilk6eaa4122011-01-18 20:09:41 -0500607 if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
608 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
609 return true;
610 }
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800611 if (unlikely(pfn >= MAX_P2M_PFN)) {
612 BUG_ON(mfn != INVALID_P2M_ENTRY);
613 return true;
614 }
615
616 topidx = p2m_top_index(pfn);
617 mididx = p2m_mid_index(pfn);
618 idx = p2m_index(pfn);
619
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500620 /* For sparse holes were the p2m leaf has real PFN along with
621 * PCI holes, stick in the PFN as the MFN value.
622 */
623 if (mfn != INVALID_P2M_ENTRY && (mfn & IDENTITY_FRAME_BIT)) {
624 if (p2m_top[topidx][mididx] == p2m_identity)
625 return true;
626
627 /* Swap over from MISSING to IDENTITY if needed. */
628 if (p2m_top[topidx][mididx] == p2m_missing) {
Konrad Rzeszutek Wilkc7617792011-01-18 20:17:10 -0500629 WARN_ON(cmpxchg(&p2m_top[topidx][mididx], p2m_missing,
630 p2m_identity) != p2m_missing);
Konrad Rzeszutek Wilkf4cec352011-01-18 20:15:21 -0500631 return true;
632 }
633 }
634
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800635 if (p2m_top[topidx][mididx] == p2m_missing)
636 return mfn == INVALID_P2M_ENTRY;
637
638 p2m_top[topidx][mididx][idx] = mfn;
639
640 return true;
641}
642
643bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
644{
Jeremy Fitzhardingeb5eafe92010-12-06 16:29:22 -0800645 if (unlikely(!__set_phys_to_machine(pfn, mfn))) {
646 if (!alloc_p2m(pfn))
647 return false;
648
649 if (!__set_phys_to_machine(pfn, mfn))
650 return false;
651 }
652
653 return true;
654}
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000655
656#define M2P_OVERRIDE_HASH_SHIFT 10
657#define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT)
658
659static RESERVE_BRK_ARRAY(struct list_head, m2p_overrides, M2P_OVERRIDE_HASH);
660static DEFINE_SPINLOCK(m2p_override_lock);
661
662static void __init m2p_override_init(void)
663{
664 unsigned i;
665
666 m2p_overrides = extend_brk(sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH,
667 sizeof(unsigned long));
668
669 for (i = 0; i < M2P_OVERRIDE_HASH; i++)
670 INIT_LIST_HEAD(&m2p_overrides[i]);
671}
672
673static unsigned long mfn_hash(unsigned long mfn)
674{
675 return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
676}
677
678/* Add an MFN override for a particular page */
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500679int m2p_add_override(unsigned long mfn, struct page *page, bool clear_pte)
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000680{
681 unsigned long flags;
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000682 unsigned long pfn;
Ian Campbell6b08cfe2011-02-11 15:23:58 +0000683 unsigned long uninitialized_var(address);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000684 unsigned level;
685 pte_t *ptep = NULL;
686
687 pfn = page_to_pfn(page);
688 if (!PageHighMem(page)) {
689 address = (unsigned long)__va(pfn << PAGE_SHIFT);
690 ptep = lookup_address(address, &level);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000691 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
692 "m2p_add_override: pfn %lx not mapped", pfn))
693 return -EINVAL;
694 }
695
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000696 page->private = mfn;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000697 page->index = pfn_to_mfn(pfn);
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000698
Daniel De Graafb2542442011-03-24 08:10:07 -0400699 if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn))))
700 return -ENOMEM;
701
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500702 if (clear_pte && !PageHighMem(page))
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000703 /* Just zap old mapping for now */
704 pte_clear(&init_mm, address, ptep);
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000705 spin_lock_irqsave(&m2p_override_lock, flags);
706 list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]);
707 spin_unlock_irqrestore(&m2p_override_lock, flags);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000708
709 return 0;
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000710}
Konrad Rzeszutek Wilk8a917072011-04-20 11:54:10 -0400711EXPORT_SYMBOL_GPL(m2p_add_override);
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500712int m2p_remove_override(struct page *page, bool clear_pte)
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000713{
714 unsigned long flags;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000715 unsigned long mfn;
716 unsigned long pfn;
Ian Campbell6b08cfe2011-02-11 15:23:58 +0000717 unsigned long uninitialized_var(address);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000718 unsigned level;
719 pte_t *ptep = NULL;
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000720
721 pfn = page_to_pfn(page);
722 mfn = get_phys_to_machine(pfn);
723 if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT))
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000724 return -EINVAL;
725
726 if (!PageHighMem(page)) {
727 address = (unsigned long)__va(pfn << PAGE_SHIFT);
728 ptep = lookup_address(address, &level);
729
730 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
731 "m2p_remove_override: pfn %lx not mapped", pfn))
732 return -EINVAL;
733 }
Stefano Stabellini9b705f02010-12-10 14:52:45 +0000734
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000735 spin_lock_irqsave(&m2p_override_lock, flags);
736 list_del(&page->lru);
737 spin_unlock_irqrestore(&m2p_override_lock, flags);
Daniel De Graafb2542442011-03-24 08:10:07 -0400738 set_phys_to_machine(pfn, page->index);
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000739
Konrad Rzeszutek Wilkcf8d9162011-02-28 17:58:48 -0500740 if (clear_pte && !PageHighMem(page))
Jeremy Fitzhardinge87f1d402010-12-13 14:42:30 +0000741 set_pte_at(&init_mm, address, ptep,
742 pfn_pte(pfn, PAGE_KERNEL));
743 /* No tlb flush necessary because the caller already
744 * left the pte unmapped. */
745
746 return 0;
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000747}
Konrad Rzeszutek Wilk8a917072011-04-20 11:54:10 -0400748EXPORT_SYMBOL_GPL(m2p_remove_override);
Jeremy Fitzhardinge448f28312010-12-15 13:19:33 +0000749
750struct page *m2p_find_override(unsigned long mfn)
751{
752 unsigned long flags;
753 struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)];
754 struct page *p, *ret;
755
756 ret = NULL;
757
758 spin_lock_irqsave(&m2p_override_lock, flags);
759
760 list_for_each_entry(p, bucket, lru) {
761 if (p->private == mfn) {
762 ret = p;
763 break;
764 }
765 }
766
767 spin_unlock_irqrestore(&m2p_override_lock, flags);
768
769 return ret;
770}
771
772unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
773{
774 struct page *p = m2p_find_override(mfn);
775 unsigned long ret = pfn;
776
777 if (p)
778 ret = page_to_pfn(p);
779
780 return ret;
781}
Konrad Rzeszutek Wilke1b478e2011-01-11 15:09:16 -0500782EXPORT_SYMBOL_GPL(m2p_find_override_pfn);
Konrad Rzeszutek Wilk2222e712010-12-22 08:57:30 -0500783
784#ifdef CONFIG_XEN_DEBUG_FS
785
786int p2m_dump_show(struct seq_file *m, void *v)
787{
788 static const char * const level_name[] = { "top", "middle",
789 "entry", "abnormal" };
790 static const char * const type_name[] = { "identity", "missing",
791 "pfn", "abnormal"};
792#define TYPE_IDENTITY 0
793#define TYPE_MISSING 1
794#define TYPE_PFN 2
795#define TYPE_UNKNOWN 3
796 unsigned long pfn, prev_pfn_type = 0, prev_pfn_level = 0;
797 unsigned int uninitialized_var(prev_level);
798 unsigned int uninitialized_var(prev_type);
799
800 if (!p2m_top)
801 return 0;
802
803 for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn++) {
804 unsigned topidx = p2m_top_index(pfn);
805 unsigned mididx = p2m_mid_index(pfn);
806 unsigned idx = p2m_index(pfn);
807 unsigned lvl, type;
808
809 lvl = 4;
810 type = TYPE_UNKNOWN;
811 if (p2m_top[topidx] == p2m_mid_missing) {
812 lvl = 0; type = TYPE_MISSING;
813 } else if (p2m_top[topidx] == NULL) {
814 lvl = 0; type = TYPE_UNKNOWN;
815 } else if (p2m_top[topidx][mididx] == NULL) {
816 lvl = 1; type = TYPE_UNKNOWN;
817 } else if (p2m_top[topidx][mididx] == p2m_identity) {
818 lvl = 1; type = TYPE_IDENTITY;
819 } else if (p2m_top[topidx][mididx] == p2m_missing) {
820 lvl = 1; type = TYPE_MISSING;
821 } else if (p2m_top[topidx][mididx][idx] == 0) {
822 lvl = 2; type = TYPE_UNKNOWN;
823 } else if (p2m_top[topidx][mididx][idx] == IDENTITY_FRAME(pfn)) {
824 lvl = 2; type = TYPE_IDENTITY;
825 } else if (p2m_top[topidx][mididx][idx] == INVALID_P2M_ENTRY) {
826 lvl = 2; type = TYPE_MISSING;
827 } else if (p2m_top[topidx][mididx][idx] == pfn) {
828 lvl = 2; type = TYPE_PFN;
829 } else if (p2m_top[topidx][mididx][idx] != pfn) {
830 lvl = 2; type = TYPE_PFN;
831 }
832 if (pfn == 0) {
833 prev_level = lvl;
834 prev_type = type;
835 }
836 if (pfn == MAX_DOMAIN_PAGES-1) {
837 lvl = 3;
838 type = TYPE_UNKNOWN;
839 }
840 if (prev_type != type) {
841 seq_printf(m, " [0x%lx->0x%lx] %s\n",
842 prev_pfn_type, pfn, type_name[prev_type]);
843 prev_pfn_type = pfn;
844 prev_type = type;
845 }
846 if (prev_level != lvl) {
847 seq_printf(m, " [0x%lx->0x%lx] level %s\n",
848 prev_pfn_level, pfn, level_name[prev_level]);
849 prev_pfn_level = pfn;
850 prev_level = lvl;
851 }
852 }
853 return 0;
854#undef TYPE_IDENTITY
855#undef TYPE_MISSING
856#undef TYPE_PFN
857#undef TYPE_UNKNOWN
858}
859#endif