Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 1 | #ifndef _ASM_POWERPC_PGTABLE_H |
| 2 | #define _ASM_POWERPC_PGTABLE_H |
Arnd Bergmann | 88ced03 | 2005-12-16 22:43:46 +0100 | [diff] [blame] | 3 | #ifdef __KERNEL__ |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 4 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 5 | #ifndef __ASSEMBLY__ |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 6 | #include <linux/mmdebug.h> |
Scott Wood | 1c98025 | 2014-08-08 18:40:42 -0500 | [diff] [blame] | 7 | #include <linux/mmzone.h> |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 8 | #include <asm/processor.h> /* For TASK_SIZE */ |
| 9 | #include <asm/mmu.h> |
| 10 | #include <asm/page.h> |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 11 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 12 | struct mm_struct; |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 13 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 14 | #endif /* !__ASSEMBLY__ */ |
| 15 | |
David Gibson | f88df14 | 2007-04-30 16:30:56 +1000 | [diff] [blame] | 16 | #if defined(CONFIG_PPC64) |
| 17 | # include <asm/pgtable-ppc64.h> |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 18 | #else |
David Gibson | f88df14 | 2007-04-30 16:30:56 +1000 | [diff] [blame] | 19 | # include <asm/pgtable-ppc32.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #endif |
| 21 | |
Aneesh Kumar K.V | cc3665a | 2013-04-28 09:37:27 +0000 | [diff] [blame] | 22 | /* |
| 23 | * We save the slot number & secondary bit in the second half of the |
| 24 | * PTE page. We use the 8 bytes per each pte entry. |
| 25 | */ |
| 26 | #define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8) |
| 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | #ifndef __ASSEMBLY__ |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 29 | |
Aneesh Kumar K.V | 78f1dbd | 2012-09-10 02:52:57 +0000 | [diff] [blame] | 30 | #include <asm/tlbflush.h> |
| 31 | |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 32 | /* Generic accessors to PTE bits */ |
| 33 | static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } |
| 34 | static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } |
| 35 | static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } |
| 36 | static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } |
| 37 | static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 38 | static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } |
| 39 | static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); } |
| 40 | |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 41 | #ifdef CONFIG_NUMA_BALANCING |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 42 | static inline int pte_present(pte_t pte) |
| 43 | { |
Mel Gorman | 6a33979 | 2014-10-09 15:26:33 -0700 | [diff] [blame] | 44 | return pte_val(pte) & _PAGE_NUMA_MASK; |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 45 | } |
| 46 | |
Mel Gorman | c46a7c8 | 2014-06-04 16:06:30 -0700 | [diff] [blame] | 47 | #define pte_present_nonuma pte_present_nonuma |
| 48 | static inline int pte_present_nonuma(pte_t pte) |
| 49 | { |
| 50 | return pte_val(pte) & (_PAGE_PRESENT); |
| 51 | } |
| 52 | |
Aneesh Kumar K.V | 56eecdb | 2014-02-12 09:13:38 +0530 | [diff] [blame] | 53 | #define ptep_set_numa ptep_set_numa |
| 54 | static inline void ptep_set_numa(struct mm_struct *mm, unsigned long addr, |
| 55 | pte_t *ptep) |
| 56 | { |
| 57 | if ((pte_val(*ptep) & _PAGE_PRESENT) == 0) |
| 58 | VM_BUG_ON(1); |
| 59 | |
| 60 | pte_update(mm, addr, ptep, _PAGE_PRESENT, _PAGE_NUMA, 0); |
| 61 | return; |
| 62 | } |
| 63 | |
Aneesh Kumar K.V | 56eecdb | 2014-02-12 09:13:38 +0530 | [diff] [blame] | 64 | #define pmdp_set_numa pmdp_set_numa |
| 65 | static inline void pmdp_set_numa(struct mm_struct *mm, unsigned long addr, |
| 66 | pmd_t *pmdp) |
| 67 | { |
| 68 | if ((pmd_val(*pmdp) & _PAGE_PRESENT) == 0) |
| 69 | VM_BUG_ON(1); |
| 70 | |
| 71 | pmd_hugepage_update(mm, addr, pmdp, _PAGE_PRESENT, _PAGE_NUMA); |
| 72 | return; |
| 73 | } |
| 74 | |
Mel Gorman | 6a33979 | 2014-10-09 15:26:33 -0700 | [diff] [blame] | 75 | /* |
| 76 | * Generic NUMA pte helpers expect pteval_t and pmdval_t types to exist |
| 77 | * which was inherited from x86. For the purposes of powerpc pte_basic_t and |
| 78 | * pmd_t are equivalent |
| 79 | */ |
| 80 | #define pteval_t pte_basic_t |
| 81 | #define pmdval_t pmd_t |
| 82 | static inline pteval_t ptenuma_flags(pte_t pte) |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 83 | { |
Mel Gorman | 6a33979 | 2014-10-09 15:26:33 -0700 | [diff] [blame] | 84 | return pte_val(pte) & _PAGE_NUMA_MASK; |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 85 | } |
| 86 | |
Mel Gorman | 6a33979 | 2014-10-09 15:26:33 -0700 | [diff] [blame] | 87 | static inline pmdval_t pmdnuma_flags(pmd_t pmd) |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 88 | { |
Mel Gorman | 6a33979 | 2014-10-09 15:26:33 -0700 | [diff] [blame] | 89 | return pmd_val(pmd) & _PAGE_NUMA_MASK; |
Aneesh Kumar K.V | c34a51c | 2013-11-18 14:58:13 +0530 | [diff] [blame] | 90 | } |
| 91 | |
| 92 | # else |
| 93 | |
| 94 | static inline int pte_present(pte_t pte) |
| 95 | { |
| 96 | return pte_val(pte) & _PAGE_PRESENT; |
| 97 | } |
| 98 | #endif /* CONFIG_NUMA_BALANCING */ |
| 99 | |
Benjamin Herrenschmidt | 7108700 | 2009-03-19 19:34:09 +0000 | [diff] [blame] | 100 | /* Conversion functions: convert a page and protection to a page entry, |
| 101 | * and a page entry and page directory to the page they refer to. |
| 102 | * |
| 103 | * Even if PTEs can be unsigned long long, a PFN is always an unsigned |
| 104 | * long for now. |
| 105 | */ |
| 106 | static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { |
| 107 | return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | |
| 108 | pgprot_val(pgprot)); } |
| 109 | static inline unsigned long pte_pfn(pte_t pte) { |
| 110 | return pte_val(pte) >> PTE_RPN_SHIFT; } |
| 111 | |
| 112 | /* Keep these as a macros to avoid include dependency mess */ |
| 113 | #define pte_page(x) pfn_to_page(pte_pfn(x)) |
| 114 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) |
| 115 | |
| 116 | /* Generic modifiers for PTE bits */ |
| 117 | static inline pte_t pte_wrprotect(pte_t pte) { |
| 118 | pte_val(pte) &= ~(_PAGE_RW | _PAGE_HWWRITE); return pte; } |
| 119 | static inline pte_t pte_mkclean(pte_t pte) { |
| 120 | pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_HWWRITE); return pte; } |
| 121 | static inline pte_t pte_mkold(pte_t pte) { |
| 122 | pte_val(pte) &= ~_PAGE_ACCESSED; return pte; } |
| 123 | static inline pte_t pte_mkwrite(pte_t pte) { |
| 124 | pte_val(pte) |= _PAGE_RW; return pte; } |
| 125 | static inline pte_t pte_mkdirty(pte_t pte) { |
| 126 | pte_val(pte) |= _PAGE_DIRTY; return pte; } |
| 127 | static inline pte_t pte_mkyoung(pte_t pte) { |
| 128 | pte_val(pte) |= _PAGE_ACCESSED; return pte; } |
| 129 | static inline pte_t pte_mkspecial(pte_t pte) { |
| 130 | pte_val(pte) |= _PAGE_SPECIAL; return pte; } |
| 131 | static inline pte_t pte_mkhuge(pte_t pte) { |
| 132 | return pte; } |
| 133 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) |
| 134 | { |
| 135 | pte_val(pte) = (pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot); |
| 136 | return pte; |
| 137 | } |
| 138 | |
| 139 | |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 140 | /* Insert a PTE, top-level function is out of line. It uses an inline |
| 141 | * low level function in the respective pgtable-* files |
| 142 | */ |
| 143 | extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, |
| 144 | pte_t pte); |
| 145 | |
| 146 | /* This low level function performs the actual PTE insertion |
| 147 | * Setting the PTE depends on the MMU type and other factors. It's |
| 148 | * an horrible mess that I'm not going to try to clean up now but |
| 149 | * I'm keeping it in one place rather than spread around |
| 150 | */ |
| 151 | static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 152 | pte_t *ptep, pte_t pte, int percpu) |
| 153 | { |
| 154 | #if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT) |
| 155 | /* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the |
| 156 | * helper pte_update() which does an atomic update. We need to do that |
| 157 | * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a |
| 158 | * per-CPU PTE such as a kmap_atomic, we do a simple update preserving |
| 159 | * the hash bits instead (ie, same as the non-SMP case) |
| 160 | */ |
| 161 | if (percpu) |
| 162 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 163 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 164 | else |
| 165 | pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); |
| 166 | |
Paul Mackerras | 1660e9d | 2009-08-17 14:36:32 +1000 | [diff] [blame] | 167 | #elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) |
| 168 | /* Second case is 32-bit with 64-bit PTE. In this case, we |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 169 | * can just store as long as we do the two halves in the right order |
| 170 | * with a barrier in between. This is possible because we take care, |
| 171 | * in the hash code, to pre-invalidate if the PTE was already hashed, |
| 172 | * which synchronizes us with any concurrent invalidation. |
| 173 | * In the percpu case, we also fallback to the simple update preserving |
| 174 | * the hash bits |
| 175 | */ |
| 176 | if (percpu) { |
| 177 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 178 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 179 | return; |
| 180 | } |
| 181 | #if _PAGE_HASHPTE != 0 |
| 182 | if (pte_val(*ptep) & _PAGE_HASHPTE) |
| 183 | flush_hash_entry(mm, ptep, addr); |
| 184 | #endif |
| 185 | __asm__ __volatile__("\ |
| 186 | stw%U0%X0 %2,%0\n\ |
| 187 | eieio\n\ |
| 188 | stw%U0%X0 %L2,%1" |
| 189 | : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) |
| 190 | : "r" (pte) : "memory"); |
| 191 | |
| 192 | #elif defined(CONFIG_PPC_STD_MMU_32) |
| 193 | /* Third case is 32-bit hash table in UP mode, we need to preserve |
| 194 | * the _PAGE_HASHPTE bit since we may not have invalidated the previous |
| 195 | * translation in the hash yet (done in a subsequent flush_tlb_xxx()) |
| 196 | * and see we need to keep track that this PTE needs invalidating |
| 197 | */ |
| 198 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) |
| 199 | | (pte_val(pte) & ~_PAGE_HASHPTE)); |
| 200 | |
| 201 | #else |
| 202 | /* Anything else just stores the PTE normally. That covers all 64-bit |
Paul Mackerras | 1660e9d | 2009-08-17 14:36:32 +1000 | [diff] [blame] | 203 | * cases, and 32-bit non-hash with 32-bit PTEs. |
Benjamin Herrenschmidt | 8d30c14 | 2009-02-10 16:02:37 +0000 | [diff] [blame] | 204 | */ |
| 205 | *ptep = pte; |
| 206 | #endif |
| 207 | } |
| 208 | |
| 209 | |
| 210 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
| 211 | extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, |
| 212 | pte_t *ptep, pte_t entry, int dirty); |
| 213 | |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 214 | /* |
| 215 | * Macro to mark a page protection value as "uncacheable". |
| 216 | */ |
| 217 | |
| 218 | #define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ |
| 219 | _PAGE_WRITETHRU) |
| 220 | |
| 221 | #define pgprot_noncached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 222 | _PAGE_NO_CACHE | _PAGE_GUARDED)) |
| 223 | |
| 224 | #define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 225 | _PAGE_NO_CACHE)) |
| 226 | |
| 227 | #define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 228 | _PAGE_COHERENT)) |
| 229 | |
| 230 | #define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ |
| 231 | _PAGE_COHERENT | _PAGE_WRITETHRU)) |
| 232 | |
Geoff Thorpe | 09c188c | 2011-10-27 02:58:45 +0000 | [diff] [blame] | 233 | #define pgprot_cached_noncoherent(prot) \ |
| 234 | (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) |
| 235 | |
Anton Blanchard | fe3cc0d9 | 2011-02-28 20:00:47 +0000 | [diff] [blame] | 236 | #define pgprot_writecombine pgprot_noncached_wc |
Benjamin Herrenschmidt | 64b3d0e | 2008-12-18 19:13:51 +0000 | [diff] [blame] | 237 | |
| 238 | struct file; |
| 239 | extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
| 240 | unsigned long size, pgprot_t vma_prot); |
| 241 | #define __HAVE_PHYS_MEM_ACCESS_PROT |
| 242 | |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 243 | /* |
| 244 | * ZERO_PAGE is a global shared page that is always zero: used |
| 245 | * for zero-mapped memory areas etc.. |
| 246 | */ |
| 247 | extern unsigned long empty_zero_page[]; |
| 248 | #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) |
| 249 | |
| 250 | extern pgd_t swapper_pg_dir[]; |
| 251 | |
Scott Wood | 1c98025 | 2014-08-08 18:40:42 -0500 | [diff] [blame] | 252 | void limit_zone_pfn(enum zone_type zone, unsigned long max_pfn); |
| 253 | int dma_pfn_limit_to_zone(u64 pfn_limit); |
David Gibson | 9c709f3 | 2007-06-13 14:52:56 +1000 | [diff] [blame] | 254 | extern void paging_init(void); |
| 255 | |
| 256 | /* |
| 257 | * kern_addr_valid is intended to indicate whether an address is a valid |
| 258 | * kernel address. Most 32-bit archs define it as always true (like this) |
| 259 | * but most 64-bit archs actually perform a test. What should we do here? |
| 260 | */ |
| 261 | #define kern_addr_valid(addr) (1) |
| 262 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | #include <asm-generic/pgtable.h> |
Benjamin Herrenschmidt | 1e3519f | 2008-07-25 16:21:11 +1000 | [diff] [blame] | 264 | |
| 265 | |
| 266 | /* |
| 267 | * This gets called at the end of handling a page fault, when |
| 268 | * the kernel has put a new PTE into the page table for the process. |
| 269 | * We use it to ensure coherency between the i-cache and d-cache |
| 270 | * for the page which has just been mapped in. |
| 271 | * On machines which use an MMU hash table, we use this to put a |
| 272 | * corresponding HPTE into the hash table ahead of time, instead of |
| 273 | * waiting for the inevitable extra hash-table miss exception. |
| 274 | */ |
Russell King | 4b3073e | 2009-12-18 16:40:18 +0000 | [diff] [blame] | 275 | extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); |
Benjamin Herrenschmidt | 1e3519f | 2008-07-25 16:21:11 +1000 | [diff] [blame] | 276 | |
Aneesh Kumar K.V | e2b3d20 | 2013-04-28 09:37:30 +0000 | [diff] [blame] | 277 | extern int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, |
Aneesh Kumar K.V | b30e759 | 2014-11-05 21:57:41 +0530 | [diff] [blame^] | 278 | unsigned long end, int write, |
| 279 | struct page **pages, int *nr); |
Aneesh Kumar K.V | 074c2ea | 2013-06-20 14:30:15 +0530 | [diff] [blame] | 280 | #ifndef CONFIG_TRANSPARENT_HUGEPAGE |
| 281 | #define pmd_large(pmd) 0 |
| 282 | #define has_transparent_hugepage() 0 |
| 283 | #endif |
Aneesh Kumar K.V | 2940999 | 2013-06-20 14:30:16 +0530 | [diff] [blame] | 284 | pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, |
| 285 | unsigned *shift); |
Bharat Bhushan | f5e3fe0 | 2013-11-15 11:01:15 +0530 | [diff] [blame] | 286 | |
| 287 | static inline pte_t *lookup_linux_ptep(pgd_t *pgdir, unsigned long hva, |
| 288 | unsigned long *pte_sizep) |
| 289 | { |
| 290 | pte_t *ptep; |
| 291 | unsigned long ps = *pte_sizep; |
| 292 | unsigned int shift; |
| 293 | |
| 294 | ptep = find_linux_pte_or_hugepte(pgdir, hva, &shift); |
| 295 | if (!ptep) |
| 296 | return NULL; |
| 297 | if (shift) |
| 298 | *pte_sizep = 1ul << shift; |
| 299 | else |
| 300 | *pte_sizep = PAGE_SIZE; |
| 301 | |
| 302 | if (ps > *pte_sizep) |
| 303 | return NULL; |
| 304 | |
| 305 | return ptep; |
| 306 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 307 | #endif /* __ASSEMBLY__ */ |
| 308 | |
Arnd Bergmann | 88ced03 | 2005-12-16 22:43:46 +0100 | [diff] [blame] | 309 | #endif /* __KERNEL__ */ |
Paul Mackerras | 047ea78 | 2005-11-19 20:17:32 +1100 | [diff] [blame] | 310 | #endif /* _ASM_POWERPC_PGTABLE_H */ |