Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 1 | #ifndef _ASM_POWERPC_PGTABLE_H |
| 2 | #define _ASM_POWERPC_PGTABLE_H |
Arnd Bergmann | 88ced03 | 2005-12-16 22:43:46 +0100 | [diff] [blame] | 3 | #ifdef __KERNEL__ |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 4 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 5 | #ifndef __ASSEMBLY__ |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 6 | #include <linux/mmdebug.h> |
Scott Wood | 1c98025 | 2014-08-08 18:40:42 -0500 | [diff] [blame^] | 7 | #include <linux/mmzone.h> |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 8 | #include <asm/processor.h> /* For TASK_SIZE */ |
| 9 | #include <asm/mmu.h> |
| 10 | #include <asm/page.h> |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 11 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 12 | struct mm_struct; |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 13 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 14 | #endif /* !__ASSEMBLY__ */ |
| 15 | |
David Gibson | f88df14 | 2007-04-30 16:30:56 +1000 | [diff] [blame] | 16 | #if defined(CONFIG_PPC64) |
| 17 | # include <asm/pgtable-ppc64.h> |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 18 | #else |
David Gibson | f88df14 | 2007-04-30 16:30:56 +1000 | [diff] [blame] | 19 | # include <asm/pgtable-ppc32.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #endif |
| 21 | |
Aneesh Kumar K.V | cc3665a | 2013-04-28 09:37:27 +0000 | [diff] [blame] | 22 | /* |
| 23 | * We save the slot number & secondary bit in the second half of the |
| 24 | * PTE page. We use the 8 bytes per each pte entry. |
| 25 | */ |
| 26 | #define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8) |
| 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | #ifndef __ASSEMBLY__ |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 29 | |
Aneesh Kumar K.V | 78f1dbd | 2012-09-10 02:52:57 +0000 | [diff] [blame] | 30 | #include <asm/tlbflush.h> |
| 31 | |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 32 | /* Generic accessors to PTE bits */ |
| 33 | static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } |
| 34 | static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } |
| 35 | static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } |
| 36 | static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } |
| 37 | static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 38 | static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } |
| 39 | static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); } |
| 40 | |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 41 | #ifdef CONFIG_NUMA_BALANCING |
| 42 | |
| 43 | static inline int pte_present(pte_t pte) |
| 44 | { |
| 45 | return pte_val(pte) & (_PAGE_PRESENT | _PAGE_NUMA); |
| 46 | } |
| 47 | |
Mel Gorman | c46a7c8 | 2014-06-04 16:06:30 -0700 | [diff] [blame] | 48 | #define pte_present_nonuma pte_present_nonuma |
| 49 | static inline int pte_present_nonuma(pte_t pte) |
| 50 | { |
| 51 | return pte_val(pte) & (_PAGE_PRESENT); |
| 52 | } |
| 53 | |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 54 | #define pte_numa pte_numa |
| 55 | static inline int pte_numa(pte_t pte) |
| 56 | { |
| 57 | return (pte_val(pte) & |
| 58 | (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA; |
| 59 | } |
| 60 | |
| 61 | #define pte_mknonnuma pte_mknonnuma |
| 62 | static inline pte_t pte_mknonnuma(pte_t pte) |
| 63 | { |
| 64 | pte_val(pte) &= ~_PAGE_NUMA; |
| 65 | pte_val(pte) |= _PAGE_PRESENT | _PAGE_ACCESSED; |
| 66 | return pte; |
| 67 | } |
| 68 | |
| 69 | #define pte_mknuma pte_mknuma |
| 70 | static inline pte_t pte_mknuma(pte_t pte) |
| 71 | { |
| 72 | /* |
| 73 | * We should not set _PAGE_NUMA on non present ptes. Also clear the |
| 74 | * present bit so that hash_page will return 1 and we collect this |
| 75 | * as numa fault. |
| 76 | */ |
| 77 | if (pte_present(pte)) { |
| 78 | pte_val(pte) |= _PAGE_NUMA; |
| 79 | pte_val(pte) &= ~_PAGE_PRESENT; |
| 80 | } else |
| 81 | VM_BUG_ON(1); |
| 82 | return pte; |
| 83 | } |
| 84 | |
Aneesh Kumar K.V | 56eecdb | 2014-02-12 09:13:38 +0530 | [diff] [blame] | 85 | #define ptep_set_numa ptep_set_numa |
| 86 | static inline void ptep_set_numa(struct mm_struct *mm, unsigned long addr, |
| 87 | pte_t *ptep) |
| 88 | { |
| 89 | if ((pte_val(*ptep) & _PAGE_PRESENT) == 0) |
| 90 | VM_BUG_ON(1); |
| 91 | |
| 92 | pte_update(mm, addr, ptep, _PAGE_PRESENT, _PAGE_NUMA, 0); |
| 93 | return; |
| 94 | } |
| 95 | |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 96 | #define pmd_numa pmd_numa |
| 97 | static inline int pmd_numa(pmd_t pmd) |
| 98 | { |
| 99 | return pte_numa(pmd_pte(pmd)); |
| 100 | } |
| 101 | |
Aneesh Kumar K.V | 56eecdb | 2014-02-12 09:13:38 +0530 | [diff] [blame] | 102 | #define pmdp_set_numa pmdp_set_numa |
| 103 | static inline void pmdp_set_numa(struct mm_struct *mm, unsigned long addr, |
| 104 | pmd_t *pmdp) |
| 105 | { |
| 106 | if ((pmd_val(*pmdp) & _PAGE_PRESENT) == 0) |
| 107 | VM_BUG_ON(1); |
| 108 | |
| 109 | pmd_hugepage_update(mm, addr, pmdp, _PAGE_PRESENT, _PAGE_NUMA); |
| 110 | return; |
| 111 | } |
| 112 | |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 113 | #define pmd_mknonnuma pmd_mknonnuma |
| 114 | static inline pmd_t pmd_mknonnuma(pmd_t pmd) |
| 115 | { |
| 116 | return pte_pmd(pte_mknonnuma(pmd_pte(pmd))); |
| 117 | } |
| 118 | |
| 119 | #define pmd_mknuma pmd_mknuma |
| 120 | static inline pmd_t pmd_mknuma(pmd_t pmd) |
| 121 | { |
| 122 | return pte_pmd(pte_mknuma(pmd_pte(pmd))); |
| 123 | } |
| 124 | |
| 125 | # else |
| 126 | |
| 127 | static inline int pte_present(pte_t pte) |
| 128 | { |
| 129 | return pte_val(pte) & _PAGE_PRESENT; |
| 130 | } |
| 131 | #endif /* CONFIG_NUMA_BALANCING */ |
| 132 | |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 133 | /* Conversion functions: convert a page and protection to a page entry, |
| 134 | * and a page entry and page directory to the page they refer to. |
| 135 | * |
| 136 | * Even if PTEs can be unsigned long long, a PFN is always an unsigned |
| 137 | * long for now. |
| 138 | */ |
| 139 | static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { |
| 140 | return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | |
| 141 | pgprot_val(pgprot)); } |
| 142 | static inline unsigned long pte_pfn(pte_t pte) { |
| 143 | return pte_val(pte) >> PTE_RPN_SHIFT; } |
| 144 | |
| 145 | /* Keep these as a macros to avoid include dependency mess */ |
| 146 | #define pte_page(x) pfn_to_page(pte_pfn(x)) |
| 147 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) |
| 148 | |
| 149 | /* Generic modifiers for PTE bits */ |
| 150 | static inline pte_t pte_wrprotect(pte_t pte) { |
| 151 | pte_val(pte) &= ~(_PAGE_RW | _PAGE_HWWRITE); return pte; } |
| 152 | static inline pte_t pte_mkclean(pte_t pte) { |
| 153 | pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_HWWRITE); return pte; } |
| 154 | static inline pte_t pte_mkold(pte_t pte) { |
| 155 | pte_val(pte) &= ~_PAGE_ACCESSED; return pte; } |
| 156 | static inline pte_t pte_mkwrite(pte_t pte) { |
| 157 | pte_val(pte) |= _PAGE_RW; return pte; } |
| 158 | static inline pte_t pte_mkdirty(pte_t pte) { |
| 159 | pte_val(pte) |= _PAGE_DIRTY; return pte; } |
| 160 | static inline pte_t pte_mkyoung(pte_t pte) { |
| 161 | pte_val(pte) |= _PAGE_ACCESSED; return pte; } |
| 162 | static inline pte_t pte_mkspecial(pte_t pte) { |
| 163 | pte_val(pte) |= _PAGE_SPECIAL; return pte; } |
| 164 | static inline pte_t pte_mkhuge(pte_t pte) { |
| 165 | return pte; } |
| 166 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) |
| 167 | { |
| 168 | pte_val(pte) = (pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot); |
| 169 | return pte; |
| 170 | } |
| 171 | |
| 172 | |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 173 | /* Insert a PTE, top-level function is out of line. It uses an inline |
| 174 | * low level function in the respective pgtable-* files |
| 175 | */ |
| 176 | extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, |
| 177 | pte_t pte); |
| 178 | |
| 179 | /* This low level function performs the actual PTE insertion |
| 180 | * Setting the PTE depends on the MMU type and other factors. It's |
| 181 | * an horrible mess that I'm not going to try to clean up now but |
| 182 | * I'm keeping it in one place rather than spread around |
| 183 | */ |
| 184 | static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 185 | pte_t *ptep, pte_t pte, int percpu) |
| 186 | { |
| 187 | #if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT) |
| 188 | /* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the |
| 189 | * helper pte_update() which does an atomic update. We need to do that |
| 190 | * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a |
| 191 | * per-CPU PTE such as a kmap_atomic, we do a simple update preserving |
| 192 | * the hash bits instead (ie, same as the non-SMP case) |
| 193 | */ |
| 194 | if (percpu) |
| 195 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 196 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 197 | else |
| 198 | pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); |
| 199 | |
Paul Mackerras | 1660e9d | 2009-08-17 14:36:32 +1000 | [diff] [blame] | 200 | #elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) |
| 201 | /* Second case is 32-bit with 64-bit PTE. In this case, we |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 202 | * can just store as long as we do the two halves in the right order |
| 203 | * with a barrier in between. This is possible because we take care, |
| 204 | * in the hash code, to pre-invalidate if the PTE was already hashed, |
| 205 | * which synchronizes us with any concurrent invalidation. |
| 206 | * In the percpu case, we also fallback to the simple update preserving |
| 207 | * the hash bits |
| 208 | */ |
| 209 | if (percpu) { |
| 210 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 211 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 212 | return; |
| 213 | } |
| 214 | #if _PAGE_HASHPTE != 0 |
| 215 | if (pte_val(*ptep) & _PAGE_HASHPTE) |
| 216 | flush_hash_entry(mm, ptep, addr); |
| 217 | #endif |
| 218 | __asm__ __volatile__("\ |
| 219 | stw%U0%X0 %2,%0\n\ |
| 220 | eieio\n\ |
| 221 | stw%U0%X0 %L2,%1" |
| 222 | : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) |
| 223 | : "r" (pte) : "memory"); |
| 224 | |
| 225 | #elif defined(CONFIG_PPC_STD_MMU_32) |
| 226 | /* Third case is 32-bit hash table in UP mode, we need to preserve |
| 227 | * the _PAGE_HASHPTE bit since we may not have invalidated the previous |
| 228 | * translation in the hash yet (done in a subsequent flush_tlb_xxx()) |
| 229 | * and see we need to keep track that this PTE needs invalidating |
| 230 | */ |
| 231 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 232 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 233 | |
| 234 | #else |
| 235 | /* Anything else just stores the PTE normally. That covers all 64-bit |
Paul Mackerras | 1660e9d | 2009-08-17 14:36:32 +1000 | [diff] [blame] | 236 | * cases, and 32-bit non-hash with 32-bit PTEs. |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 237 | */ |
| 238 | *ptep = pte; |
| 239 | #endif |
| 240 | } |
| 241 | |
| 242 | |
| 243 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
| 244 | extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, |
| 245 | pte_t *ptep, pte_t entry, int dirty); |
| 246 | |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 247 | /* |
| 248 | * Macro to mark a page protection value as "uncacheable". |
| 249 | */ |
| 250 | |
| 251 | #define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ |
| 252 | _PAGE_WRITETHRU) |
| 253 | |
| 254 | #define pgprot_noncached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 255 | _PAGE_NO_CACHE | _PAGE_GUARDED)) |
| 256 | |
| 257 | #define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 258 | _PAGE_NO_CACHE)) |
| 259 | |
| 260 | #define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 261 | _PAGE_COHERENT)) |
| 262 | |
| 263 | #define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 264 | _PAGE_COHERENT | _PAGE_WRITETHRU)) |
| 265 | |
Geoff Thorpe | 09c188c | 2011-10-27 02:58:45 +0000 | [diff] [blame] | 266 | #define pgprot_cached_noncoherent(prot) \ |
| 267 | (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) |
| 268 | |
Anton Blanchard | fe3cc0d9 | 2011-02-28 20:00:47 +0000 | [diff] [blame] | 269 | #define pgprot_writecombine pgprot_noncached_wc |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 270 | |
| 271 | struct file; |
| 272 | extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
| 273 | unsigned long size, pgprot_t vma_prot); |
| 274 | #define __HAVE_PHYS_MEM_ACCESS_PROT |
| 275 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 276 | /* |
| 277 | * ZERO_PAGE is a global shared page that is always zero: used |
| 278 | * for zero-mapped memory areas etc.. |
| 279 | */ |
| 280 | extern unsigned long empty_zero_page[]; |
| 281 | #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) |
| 282 | |
| 283 | extern pgd_t swapper_pg_dir[]; |
| 284 | |
Scott Wood | 1c98025 | 2014-08-08 18:40:42 -0500 | [diff] [blame^] | 285 | void limit_zone_pfn(enum zone_type zone, unsigned long max_pfn); |
| 286 | int dma_pfn_limit_to_zone(u64 pfn_limit); |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 287 | extern void paging_init(void); |
| 288 | |
| 289 | /* |
| 290 | * kern_addr_valid is intended to indicate whether an address is a valid |
| 291 | * kernel address. Most 32-bit archs define it as always true (like this) |
| 292 | * but most 64-bit archs actually perform a test. What should we do here? |
| 293 | */ |
| 294 | #define kern_addr_valid(addr) (1) |
| 295 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | #include <asm-generic/pgtable.h> |
Benjamin Herrenschmidt | 1e3519f | 2008-07-25 16:21:11 +1000 | [diff] [blame] | 297 | |
| 298 | |
| 299 | /* |
| 300 | * This gets called at the end of handling a page fault, when |
| 301 | * the kernel has put a new PTE into the page table for the process. |
| 302 | * We use it to ensure coherency between the i-cache and d-cache |
| 303 | * for the page which has just been mapped in. |
| 304 | * On machines which use an MMU hash table, we use this to put a |
| 305 | * corresponding HPTE into the hash table ahead of time, instead of |
| 306 | * waiting for the inevitable extra hash-table miss exception. |
| 307 | */ |
Russell King | 4b3073e | 2009-12-18 16:40:18 +0000 | [diff] [blame] | 308 | extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); |
Benjamin Herrenschmidt | 1e3519f | 2008-07-25 16:21:11 +1000 | [diff] [blame] | 309 | |
David Gibson | a4fe3ce | 2009-10-26 19:24:31 +0000 | [diff] [blame] | 310 | extern int gup_hugepd(hugepd_t *hugepd, unsigned pdshift, unsigned long addr, |
| 311 | unsigned long end, int write, struct page **pages, int *nr); |
| 312 | |
Aneesh Kumar K.V | e2b3d20 | 2013-04-28 09:37:30 +0000 | [diff] [blame] | 313 | extern int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, |
| 314 | unsigned long end, int write, struct page **pages, int *nr); |
Aneesh Kumar K.V | 074c2ea | 2013-06-20 14:30:15 +0530 | [diff] [blame] | 315 | #ifndef CONFIG_TRANSPARENT_HUGEPAGE |
| 316 | #define pmd_large(pmd) 0 |
| 317 | #define has_transparent_hugepage() 0 |
| 318 | #endif |
Aneesh Kumar K.V | 2940999 | 2013-06-20 14:30:16 +0530 | [diff] [blame] | 319 | pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, |
| 320 | unsigned *shift); |
Bharat Bhushan | f5e3fe0 | 2013-11-15 11:01:15 +0530 | [diff] [blame] | 321 | |
| 322 | static inline pte_t *lookup_linux_ptep(pgd_t *pgdir, unsigned long hva, |
| 323 | unsigned long *pte_sizep) |
| 324 | { |
| 325 | pte_t *ptep; |
| 326 | unsigned long ps = *pte_sizep; |
| 327 | unsigned int shift; |
| 328 | |
| 329 | ptep = find_linux_pte_or_hugepte(pgdir, hva, &shift); |
| 330 | if (!ptep) |
| 331 | return NULL; |
| 332 | if (shift) |
| 333 | *pte_sizep = 1ul << shift; |
| 334 | else |
| 335 | *pte_sizep = PAGE_SIZE; |
| 336 | |
| 337 | if (ps > *pte_sizep) |
| 338 | return NULL; |
| 339 | |
| 340 | return ptep; |
| 341 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 342 | #endif /* __ASSEMBLY__ */ |
| 343 | |
Arnd Bergmann | 88ced03 | 2005-12-16 22:43:46 +0100 | [diff] [blame] | 344 | #endif /* __KERNEL__ */ |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 345 | #endif /* _ASM_POWERPC_PGTABLE_H */ |