blob: 29d96d168bc097195e9cbe66ab723c90929e1116 [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_PGTABLE_H
2#define _ASM_X86_PGTABLE_H
Jeremy Fitzhardinge6c386652008-01-30 13:32:55 +01003
Ingo Molnarc47c1b12009-02-09 11:57:45 +01004#include <asm/page.h>
5
Jeremy Fitzhardinge8d19c992009-02-08 18:46:18 -08006#include <asm/pgtable_types.h>
Suresh Siddhab2bc2732008-09-23 14:00:36 -07007
venkatesh.pallipadi@intel.com8a7b12f2008-12-18 11:41:31 -08008/*
9 * Macro to mark a page protection value as UC-
10 */
11#define pgprot_noncached(prot) \
12 ((boot_cpu_data.x86 > 3) \
13 ? (__pgprot(pgprot_val(prot) | _PAGE_CACHE_UC_MINUS)) \
14 : (prot))
15
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010016#ifndef __ASSEMBLY__
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +010017
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010018/*
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010019 * ZERO_PAGE is a global shared page that is always zero: used
20 * for zero-mapped memory areas etc..
21 */
Joe Perches3cbaeaf2008-03-23 01:03:12 -070022extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010023#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
24
Jeremy Fitzhardingee3ed9102008-01-30 13:34:11 +010025extern spinlock_t pgd_lock;
26extern struct list_head pgd_list;
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010027
Jeremy Fitzhardinge54321d92009-02-11 10:20:05 -080028#ifdef CONFIG_PARAVIRT
29#include <asm/paravirt.h>
30#else /* !CONFIG_PARAVIRT */
31#define set_pte(ptep, pte) native_set_pte(ptep, pte)
32#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
33
Jeremy Fitzhardinge54321d92009-02-11 10:20:05 -080034#define set_pte_atomic(ptep, pte) \
35 native_set_pte_atomic(ptep, pte)
36
37#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
38
39#ifndef __PAGETABLE_PUD_FOLDED
40#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
41#define pgd_clear(pgd) native_pgd_clear(pgd)
42#endif
43
44#ifndef set_pud
45# define set_pud(pudp, pud) native_set_pud(pudp, pud)
46#endif
47
48#ifndef __PAGETABLE_PMD_FOLDED
49#define pud_clear(pud) native_pud_clear(pud)
50#endif
51
52#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
53#define pmd_clear(pmd) native_pmd_clear(pmd)
54
55#define pte_update(mm, addr, ptep) do { } while (0)
56#define pte_update_defer(mm, addr, ptep) do { } while (0)
57
58static inline void __init paravirt_pagetable_setup_start(pgd_t *base)
59{
60 native_pagetable_setup_start(base);
61}
62
63static inline void __init paravirt_pagetable_setup_done(pgd_t *base)
64{
65 native_pagetable_setup_done(base);
66}
67
68#define pgd_val(x) native_pgd_val(x)
69#define __pgd(x) native_make_pgd(x)
70
71#ifndef __PAGETABLE_PUD_FOLDED
72#define pud_val(x) native_pud_val(x)
73#define __pud(x) native_make_pud(x)
74#endif
75
76#ifndef __PAGETABLE_PMD_FOLDED
77#define pmd_val(x) native_pmd_val(x)
78#define __pmd(x) native_make_pmd(x)
79#endif
80
81#define pte_val(x) native_pte_val(x)
82#define __pte(x) native_make_pte(x)
83
84#endif /* CONFIG_PARAVIRT */
85
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010086/*
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010087 * The following only work if pte_present() is true.
88 * Undefined behaviour if not..
89 */
Joe Perches3cbaeaf2008-03-23 01:03:12 -070090static inline int pte_dirty(pte_t pte)
91{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +010092 return pte_flags(pte) & _PAGE_DIRTY;
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010093}
94
Joe Perches3cbaeaf2008-03-23 01:03:12 -070095static inline int pte_young(pte_t pte)
96{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +010097 return pte_flags(pte) & _PAGE_ACCESSED;
Joe Perches3cbaeaf2008-03-23 01:03:12 -070098}
99
100static inline int pte_write(pte_t pte)
101{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100102 return pte_flags(pte) & _PAGE_RW;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700103}
104
105static inline int pte_file(pte_t pte)
106{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100107 return pte_flags(pte) & _PAGE_FILE;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700108}
109
110static inline int pte_huge(pte_t pte)
111{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100112 return pte_flags(pte) & _PAGE_PSE;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700113}
114
115static inline int pte_global(pte_t pte)
116{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100117 return pte_flags(pte) & _PAGE_GLOBAL;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700118}
119
120static inline int pte_exec(pte_t pte)
121{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100122 return !(pte_flags(pte) & _PAGE_NX);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700123}
124
Nick Piggin7e675132008-04-28 02:13:00 -0700125static inline int pte_special(pte_t pte)
126{
Jan Beulich606ee442008-09-17 16:48:17 +0100127 return pte_flags(pte) & _PAGE_SPECIAL;
Nick Piggin7e675132008-04-28 02:13:00 -0700128}
129
Hugh Dickins91030ca2008-09-09 16:42:45 +0100130static inline unsigned long pte_pfn(pte_t pte)
131{
132 return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
133}
134
135#define pte_page(pte) pfn_to_page(pte_pfn(pte))
136
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700137static inline int pmd_large(pmd_t pte)
138{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800139 return (pmd_flags(pte) & (_PAGE_PSE | _PAGE_PRESENT)) ==
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700140 (_PAGE_PSE | _PAGE_PRESENT);
141}
142
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800143static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
144{
145 pteval_t v = native_pte_val(pte);
146
147 return native_make_pte(v | set);
148}
149
150static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
151{
152 pteval_t v = native_pte_val(pte);
153
154 return native_make_pte(v & ~clear);
155}
156
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700157static inline pte_t pte_mkclean(pte_t pte)
158{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800159 return pte_clear_flags(pte, _PAGE_DIRTY);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700160}
161
162static inline pte_t pte_mkold(pte_t pte)
163{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800164 return pte_clear_flags(pte, _PAGE_ACCESSED);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700165}
166
167static inline pte_t pte_wrprotect(pte_t pte)
168{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800169 return pte_clear_flags(pte, _PAGE_RW);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700170}
171
172static inline pte_t pte_mkexec(pte_t pte)
173{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800174 return pte_clear_flags(pte, _PAGE_NX);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700175}
176
177static inline pte_t pte_mkdirty(pte_t pte)
178{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800179 return pte_set_flags(pte, _PAGE_DIRTY);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700180}
181
182static inline pte_t pte_mkyoung(pte_t pte)
183{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800184 return pte_set_flags(pte, _PAGE_ACCESSED);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700185}
186
187static inline pte_t pte_mkwrite(pte_t pte)
188{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800189 return pte_set_flags(pte, _PAGE_RW);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700190}
191
192static inline pte_t pte_mkhuge(pte_t pte)
193{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800194 return pte_set_flags(pte, _PAGE_PSE);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700195}
196
197static inline pte_t pte_clrhuge(pte_t pte)
198{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800199 return pte_clear_flags(pte, _PAGE_PSE);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700200}
201
202static inline pte_t pte_mkglobal(pte_t pte)
203{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800204 return pte_set_flags(pte, _PAGE_GLOBAL);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700205}
206
207static inline pte_t pte_clrglobal(pte_t pte)
208{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800209 return pte_clear_flags(pte, _PAGE_GLOBAL);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700210}
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +0100211
Nick Piggin7e675132008-04-28 02:13:00 -0700212static inline pte_t pte_mkspecial(pte_t pte)
213{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800214 return pte_set_flags(pte, _PAGE_SPECIAL);
Nick Piggin7e675132008-04-28 02:13:00 -0700215}
216
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800217/*
218 * Mask out unsupported bits in a present pgprot. Non-present pgprots
219 * can use those bits for other purposes, so leave them be.
220 */
221static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
222{
223 pgprotval_t protval = pgprot_val(pgprot);
224
225 if (protval & _PAGE_PRESENT)
226 protval &= __supported_pte_mask;
227
228 return protval;
229}
230
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100231static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
232{
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800233 return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
234 massage_pgprot(pgprot));
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100235}
236
237static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
238{
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800239 return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
240 massage_pgprot(pgprot));
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100241}
242
Ingo Molnar38472312008-01-30 13:32:57 +0100243static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
244{
245 pteval_t val = pte_val(pte);
246
247 /*
248 * Chop off the NX bit (if present), and add the NX portion of
249 * the newprot (if present):
250 */
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -0700251 val &= _PAGE_CHG_MASK;
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800252 val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK;
Ingo Molnar38472312008-01-30 13:32:57 +0100253
254 return __pte(val);
255}
256
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -0700257/* mprotect needs to preserve PAT bits when updating vm_page_prot */
258#define pgprot_modify pgprot_modify
259static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
260{
261 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
262 pgprotval_t addbits = pgprot_val(newprot);
263 return __pgprot(preservebits | addbits);
264}
265
Jeremy Fitzhardinge77be1fa2008-07-21 22:59:56 -0700266#define pte_pgprot(x) __pgprot(pte_flags(x) & PTE_FLAGS_MASK)
Andi Kleenc6ca18e2008-01-30 13:33:51 +0100267
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800268#define canon_pgprot(p) __pgprot(massage_pgprot(p))
Andi Kleen1e8e23b2008-01-30 13:33:53 +0100269
venkatesh.pallipadi@intel.comafc7d202009-01-09 16:13:10 -0800270static inline int is_new_memtype_allowed(unsigned long flags,
271 unsigned long new_flags)
272{
273 /*
274 * Certain new memtypes are not allowed with certain
275 * requested memtype:
276 * - request is uncached, return cannot be write-back
277 * - request is write-combine, return cannot be write-back
278 */
279 if ((flags == _PAGE_CACHE_UC_MINUS &&
280 new_flags == _PAGE_CACHE_WB) ||
281 (flags == _PAGE_CACHE_WC &&
282 new_flags == _PAGE_CACHE_WB)) {
283 return 0;
284 }
285
286 return 1;
287}
288
Tejun Heo458a3e62009-02-24 11:57:21 +0900289pmd_t *populate_extra_pmd(unsigned long vaddr);
290pte_t *populate_extra_pte(unsigned long vaddr);
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +0100291#endif /* __ASSEMBLY__ */
292
Thomas Gleixner96a388d2007-10-11 11:20:03 +0200293#ifdef CONFIG_X86_32
294# include "pgtable_32.h"
295#else
296# include "pgtable_64.h"
297#endif
Jeremy Fitzhardinge6c386652008-01-30 13:32:55 +0100298
Jeremy Fitzhardingeaca159d2009-02-05 11:30:54 -0800299#ifndef __ASSEMBLY__
Jeremy Fitzhardingef4769612009-02-05 11:30:55 -0800300#include <linux/mm_types.h>
Jeremy Fitzhardingeaca159d2009-02-05 11:30:54 -0800301
Jeremy Fitzhardingea034a012009-02-05 11:30:43 -0800302static inline int pte_none(pte_t pte)
303{
304 return !pte.pte;
305}
306
Jeremy Fitzhardinge8de01da2009-02-05 11:30:44 -0800307#define __HAVE_ARCH_PTE_SAME
308static inline int pte_same(pte_t a, pte_t b)
309{
310 return a.pte == b.pte;
311}
312
Jeremy Fitzhardinge7c683852009-02-05 11:30:45 -0800313static inline int pte_present(pte_t a)
314{
315 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
316}
317
Jeremy Fitzhardinge649e8ef2009-02-05 11:30:50 -0800318static inline int pmd_present(pmd_t pmd)
319{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800320 return pmd_flags(pmd) & _PAGE_PRESENT;
Jeremy Fitzhardinge649e8ef2009-02-05 11:30:50 -0800321}
322
Jeremy Fitzhardinge4fea8012009-02-05 11:30:51 -0800323static inline int pmd_none(pmd_t pmd)
324{
325 /* Only check low word on 32-bit platforms, since it might be
326 out of sync with upper half. */
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800327 return (unsigned long)native_pmd_val(pmd) == 0;
Jeremy Fitzhardinge4fea8012009-02-05 11:30:51 -0800328}
329
Jeremy Fitzhardinge3ffb3562009-02-05 11:30:59 -0800330static inline unsigned long pmd_page_vaddr(pmd_t pmd)
331{
332 return (unsigned long)__va(pmd_val(pmd) & PTE_PFN_MASK);
333}
334
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100335/*
336 * Currently stuck as a macro due to indirect forward reference to
337 * linux/mmzone.h's __section_mem_map_addr() definition:
338 */
339#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
Jeremy Fitzhardinge20063ca2009-02-05 11:31:00 -0800340
Jeremy Fitzhardingee24d7ee2009-02-05 11:31:01 -0800341/*
342 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
343 *
344 * this macro returns the index of the entry in the pmd page which would
345 * control the given virtual address
346 */
347static inline unsigned pmd_index(unsigned long address)
348{
349 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
350}
351
Jeremy Fitzhardinge97e28172009-02-05 11:31:05 -0800352/*
353 * Conversion functions: convert a page and protection to a page entry,
354 * and a page entry and page directory to the page they refer to.
355 *
356 * (Currently stuck as a macro because of indirect forward reference
357 * to linux/mm.h:page_to_nid())
358 */
359#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
360
Jeremy Fitzhardinge346309c2009-02-05 11:31:06 -0800361/*
362 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
363 *
364 * this function returns the index of the entry in the pte page which would
365 * control the given virtual address
366 */
367static inline unsigned pte_index(unsigned long address)
368{
369 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
370}
371
Jeremy Fitzhardinge3fbc2442009-02-05 11:31:07 -0800372static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
373{
374 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
375}
376
Jeremy Fitzhardinge99510232009-02-05 11:31:11 -0800377static inline int pmd_bad(pmd_t pmd)
378{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800379 return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
Jeremy Fitzhardinge99510232009-02-05 11:31:11 -0800380}
381
Jeremy Fitzhardingecc290ca2009-02-05 11:31:12 -0800382static inline unsigned long pages_to_mb(unsigned long npg)
383{
384 return npg >> (20 - PAGE_SHIFT);
385}
386
Jeremy Fitzhardinge6cf71502009-02-05 11:31:15 -0800387#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \
388 remap_pfn_range(vma, vaddr, pfn, size, prot)
389
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800390#if PAGETABLE_LEVELS > 2
Jeremy Fitzhardingedeb79cf2009-02-05 11:31:13 -0800391static inline int pud_none(pud_t pud)
392{
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800393 return native_pud_val(pud) == 0;
Jeremy Fitzhardingedeb79cf2009-02-05 11:31:13 -0800394}
395
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800396static inline int pud_present(pud_t pud)
397{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800398 return pud_flags(pud) & _PAGE_PRESENT;
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800399}
Jeremy Fitzhardinge6fff47e2009-02-05 11:30:53 -0800400
401static inline unsigned long pud_page_vaddr(pud_t pud)
402{
403 return (unsigned long)__va((unsigned long)pud_val(pud) & PTE_PFN_MASK);
404}
Jeremy Fitzhardingef4769612009-02-05 11:30:55 -0800405
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100406/*
407 * Currently stuck as a macro due to indirect forward reference to
408 * linux/mmzone.h's __section_mem_map_addr() definition:
409 */
410#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
Jeremy Fitzhardinge01ade20d5a2009-02-05 11:31:02 -0800411
412/* Find an entry in the second-level page table.. */
413static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
414{
415 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
416}
Jeremy Fitzhardinge3180fba2009-02-05 11:31:04 -0800417
418static inline unsigned long pmd_pfn(pmd_t pmd)
419{
420 return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
421}
Jeremy Fitzhardinge3f6cbef2009-02-05 11:31:08 -0800422
423static inline int pud_large(pud_t pud)
424{
Jeremy Fitzhardingee2f5bda2009-02-09 00:09:52 -0800425 return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
Jeremy Fitzhardinge3f6cbef2009-02-05 11:31:08 -0800426 (_PAGE_PSE | _PAGE_PRESENT);
427}
Jeremy Fitzhardingea61bb292009-02-05 11:31:10 -0800428
429static inline int pud_bad(pud_t pud)
430{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800431 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
Jeremy Fitzhardingea61bb292009-02-05 11:31:10 -0800432}
Jeremy Fitzhardingee2f5bda2009-02-09 00:09:52 -0800433#else
434static inline int pud_large(pud_t pud)
435{
436 return 0;
437}
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800438#endif /* PAGETABLE_LEVELS > 2 */
439
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800440#if PAGETABLE_LEVELS > 3
441static inline int pgd_present(pgd_t pgd)
442{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800443 return pgd_flags(pgd) & _PAGE_PRESENT;
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800444}
Jeremy Fitzhardingec5f040b2009-02-05 11:30:52 -0800445
446static inline unsigned long pgd_page_vaddr(pgd_t pgd)
447{
448 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
449}
Jeremy Fitzhardinge777cba12009-02-05 11:30:56 -0800450
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100451/*
452 * Currently stuck as a macro due to indirect forward reference to
453 * linux/mmzone.h's __section_mem_map_addr() definition:
454 */
455#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
Jeremy Fitzhardinge7cfb8102009-02-05 11:30:57 -0800456
457/* to find an entry in a page-table-directory. */
458static inline unsigned pud_index(unsigned long address)
459{
460 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
461}
Jeremy Fitzhardinge3d081b12009-02-05 11:30:58 -0800462
463static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
464{
465 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);
466}
Jeremy Fitzhardinge30f10312009-02-05 11:31:09 -0800467
468static inline int pgd_bad(pgd_t pgd)
469{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800470 return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
Jeremy Fitzhardinge30f10312009-02-05 11:31:09 -0800471}
Jeremy Fitzhardinge7325cc22009-02-05 11:31:14 -0800472
473static inline int pgd_none(pgd_t pgd)
474{
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800475 return !native_pgd_val(pgd);
Jeremy Fitzhardinge7325cc22009-02-05 11:31:14 -0800476}
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800477#endif /* PAGETABLE_LEVELS > 3 */
478
Thomas Gleixner96a388d2007-10-11 11:20:03 +0200479#endif /* __ASSEMBLY__ */
480
Jeremy Fitzhardingefb15a9b2008-06-25 00:19:06 -0400481/*
482 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
483 *
484 * this macro returns the index of the entry in the pgd page which would
485 * control the given virtual address
486 */
487#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
488
489/*
490 * pgd_offset() returns a (pgd_t *)
491 * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
492 */
493#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address)))
494/*
495 * a shortcut which implies the use of the kernel's pgd, instead
496 * of a process's
497 */
498#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
499
500
Jeremy Fitzhardinge68db0652008-03-17 16:37:13 -0700501#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
502#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
503
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100504#ifndef __ASSEMBLY__
505
Jeremy Fitzhardinge48916452008-01-30 13:32:58 +0100506/* local pte updates need not use xchg for locking */
507static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
508{
509 pte_t res = *ptep;
510
511 /* Pure native function needs no input for mm, addr */
512 native_pte_clear(NULL, 0, ptep);
513 return res;
514}
515
516static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
517 pte_t *ptep , pte_t pte)
518{
519 native_set_pte(ptep, pte);
520}
521
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100522#ifndef CONFIG_PARAVIRT
523/*
524 * Rules for using pte_update - it must be called after any PTE update which
525 * has not been done using the set_pte / clear_pte interfaces. It is used by
526 * shadow mode hypervisors to resynchronize the shadow page tables. Kernel PTE
527 * updates should either be sets, clears, or set_pte_atomic for P->P
528 * transitions, which means this hook should only be called for user PTEs.
529 * This hook implies a P->P protection or access change has taken place, which
530 * requires a subsequent TLB flush. The notification can optionally be delayed
531 * until the TLB flush event by using the pte_update_defer form of the
532 * interface, but care must be taken to assure that the flush happens while
533 * still holding the same page table lock so that the shadow and primary pages
534 * do not become out of sync on SMP.
535 */
536#define pte_update(mm, addr, ptep) do { } while (0)
537#define pte_update_defer(mm, addr, ptep) do { } while (0)
538#endif
539
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100540/*
541 * We only update the dirty/accessed state if we set
542 * the dirty bit by hand in the kernel, since the hardware
543 * will do the accessed bit for us, and we don't want to
544 * race with other CPU's that might be updating the dirty
545 * bit at the same time.
546 */
Jeremy Fitzhardingebea41802008-06-25 00:18:57 -0400547struct vm_area_struct;
548
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100549#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -0700550extern int ptep_set_access_flags(struct vm_area_struct *vma,
551 unsigned long address, pte_t *ptep,
552 pte_t entry, int dirty);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100553
554#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
Jeremy Fitzhardingef9fbf1a2008-03-17 16:37:04 -0700555extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
556 unsigned long addr, pte_t *ptep);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100557
558#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
Jeremy Fitzhardingec20311e2008-03-17 16:37:05 -0700559extern int ptep_clear_flush_young(struct vm_area_struct *vma,
560 unsigned long address, pte_t *ptep);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100561
562#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700563static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
564 pte_t *ptep)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100565{
566 pte_t pte = native_ptep_get_and_clear(ptep);
567 pte_update(mm, addr, ptep);
568 return pte;
569}
570
571#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700572static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
573 unsigned long addr, pte_t *ptep,
574 int full)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100575{
576 pte_t pte;
577 if (full) {
578 /*
579 * Full address destruction in progress; paravirt does not
580 * care about updates and native needs no locking
581 */
582 pte = native_local_ptep_get_and_clear(ptep);
583 } else {
584 pte = ptep_get_and_clear(mm, addr, ptep);
585 }
586 return pte;
587}
588
589#define __HAVE_ARCH_PTEP_SET_WRPROTECT
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700590static inline void ptep_set_wrprotect(struct mm_struct *mm,
591 unsigned long addr, pte_t *ptep)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100592{
Jeremy Fitzhardinged8d89822008-01-30 13:32:58 +0100593 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100594 pte_update(mm, addr, ptep);
595}
596
Jeremy Fitzhardinge85958b42008-03-17 16:37:14 -0700597/*
598 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
599 *
600 * dst - pointer to pgd range anwhere on a pgd page
601 * src - ""
602 * count - the number of pgds to copy.
603 *
604 * dst and src can be on the same page, but the range must not overlap,
605 * and must not cross a page boundary.
606 */
607static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
608{
609 memcpy(dst, src, count * sizeof(pgd_t));
610}
611
612
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100613#include <asm-generic/pgtable.h>
614#endif /* __ASSEMBLY__ */
615
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700616#endif /* _ASM_X86_PGTABLE_H */