Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 2 | #ifndef _ASM_POWERPC_HUGETLB_H |
| 3 | #define _ASM_POWERPC_HUGETLB_H |
| 4 | |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 5 | #ifdef CONFIG_HUGETLB_PAGE |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 6 | #include <asm/page.h> |
Gerald Schaefer | 106c992 | 2013-04-29 15:07:23 -0700 | [diff] [blame] | 7 | #include <asm-generic/hugetlb.h> |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 8 | |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 9 | extern struct kmem_cache *hugepte_cache; |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 10 | |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 11 | #ifdef CONFIG_PPC_BOOK3S_64 |
Aneesh Kumar K.V | 4848376 | 2016-04-29 23:26:25 +1000 | [diff] [blame] | 12 | |
Aneesh Kumar K.V | bee8b3b | 2016-11-28 11:46:58 +0530 | [diff] [blame] | 13 | #include <asm/book3s/64/hugetlb.h> |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 14 | /* |
| 15 | * This should work for other subarchs too. But right now we use the |
| 16 | * new format only for 64bit book3s |
| 17 | */ |
| 18 | static inline pte_t *hugepd_page(hugepd_t hpd) |
| 19 | { |
| 20 | BUG_ON(!hugepd_ok(hpd)); |
| 21 | /* |
| 22 | * We have only four bits to encode, MMU page size |
| 23 | */ |
| 24 | BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf); |
Aneesh Kumar K.V | 20717e1 | 2016-12-14 10:07:53 +0530 | [diff] [blame] | 25 | return __va(hpd_val(hpd) & HUGEPD_ADDR_MASK); |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 26 | } |
| 27 | |
| 28 | static inline unsigned int hugepd_mmu_psize(hugepd_t hpd) |
| 29 | { |
Aneesh Kumar K.V | 20717e1 | 2016-12-14 10:07:53 +0530 | [diff] [blame] | 30 | return (hpd_val(hpd) & HUGEPD_SHIFT_MASK) >> 2; |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 31 | } |
| 32 | |
| 33 | static inline unsigned int hugepd_shift(hugepd_t hpd) |
| 34 | { |
| 35 | return mmu_psize_to_shift(hugepd_mmu_psize(hpd)); |
| 36 | } |
Aneesh Kumar K.V | 4848376 | 2016-04-29 23:26:25 +1000 | [diff] [blame] | 37 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, |
| 38 | unsigned long vmaddr) |
| 39 | { |
| 40 | if (radix_enabled()) |
| 41 | return radix__flush_hugetlb_page(vma, vmaddr); |
| 42 | } |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 43 | |
| 44 | #else |
| 45 | |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 46 | static inline pte_t *hugepd_page(hugepd_t hpd) |
| 47 | { |
| 48 | BUG_ON(!hugepd_ok(hpd)); |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 49 | #ifdef CONFIG_PPC_8xx |
Christophe Leroy | de0f938 | 2018-01-12 13:45:31 +0100 | [diff] [blame] | 50 | return (pte_t *)__va(hpd_val(hpd) & ~HUGEPD_SHIFT_MASK); |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 51 | #else |
Aneesh Kumar K.V | 20717e1 | 2016-12-14 10:07:53 +0530 | [diff] [blame] | 52 | return (pte_t *)((hpd_val(hpd) & |
| 53 | ~HUGEPD_SHIFT_MASK) | PD_HUGE); |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 54 | #endif |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 55 | } |
| 56 | |
| 57 | static inline unsigned int hugepd_shift(hugepd_t hpd) |
| 58 | { |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 59 | #ifdef CONFIG_PPC_8xx |
Aneesh Kumar K.V | 20717e1 | 2016-12-14 10:07:53 +0530 | [diff] [blame] | 60 | return ((hpd_val(hpd) & _PMD_PAGE_MASK) >> 1) + 17; |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 61 | #else |
Aneesh Kumar K.V | 20717e1 | 2016-12-14 10:07:53 +0530 | [diff] [blame] | 62 | return hpd_val(hpd) & HUGEPD_SHIFT_MASK; |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 63 | #endif |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 64 | } |
| 65 | |
Aneesh Kumar K.V | cf9427b | 2013-04-28 09:37:29 +0000 | [diff] [blame] | 66 | #endif /* CONFIG_PPC_BOOK3S_64 */ |
| 67 | |
| 68 | |
Aneesh Kumar K.V | b30e759 | 2014-11-05 21:57:41 +0530 | [diff] [blame] | 69 | static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 70 | unsigned pdshift) |
| 71 | { |
| 72 | /* |
Becky Bruce | 881fde1 | 2011-10-10 10:50:40 +0000 | [diff] [blame] | 73 | * On FSL BookE, we have multiple higher-level table entries that |
| 74 | * point to the same hugepte. Just use the first one since they're all |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 75 | * identical. So for that case, idx=0. |
| 76 | */ |
| 77 | unsigned long idx = 0; |
| 78 | |
Aneesh Kumar K.V | b30e759 | 2014-11-05 21:57:41 +0530 | [diff] [blame] | 79 | pte_t *dir = hugepd_page(hpd); |
Becky Bruce | 881fde1 | 2011-10-10 10:50:40 +0000 | [diff] [blame] | 80 | #ifndef CONFIG_PPC_FSL_BOOK3E |
Aneesh Kumar K.V | b30e759 | 2014-11-05 21:57:41 +0530 | [diff] [blame] | 81 | idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(hpd); |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 82 | #endif |
| 83 | |
| 84 | return dir + idx; |
| 85 | } |
| 86 | |
David Gibson | 0895ecd | 2009-10-26 19:24:31 +0000 | [diff] [blame] | 87 | void flush_dcache_icache_hugepage(struct page *page); |
| 88 | |
Nicholas Piggin | 014a32b | 2018-03-07 11:37:17 +1000 | [diff] [blame] | 89 | int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 90 | unsigned long len); |
Nicholas Piggin | 014a32b | 2018-03-07 11:37:17 +1000 | [diff] [blame] | 91 | |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 92 | static inline int is_hugepage_only_range(struct mm_struct *mm, |
| 93 | unsigned long addr, |
| 94 | unsigned long len) |
| 95 | { |
Nicholas Piggin | 014a32b | 2018-03-07 11:37:17 +1000 | [diff] [blame] | 96 | if (IS_ENABLED(CONFIG_PPC_MM_SLICES) && !radix_enabled()) |
| 97 | return slice_is_hugepage_only_range(mm, addr, len); |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 98 | return 0; |
| 99 | } |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 100 | |
Becky Bruce | d93e4d7 | 2011-11-28 14:43:33 +0000 | [diff] [blame] | 101 | void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea, |
| 102 | pte_t pte); |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 103 | #ifdef CONFIG_PPC_8xx |
| 104 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, |
| 105 | unsigned long vmaddr) |
| 106 | { |
| 107 | flush_tlb_page(vma, vmaddr); |
| 108 | } |
| 109 | #else |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 110 | void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); |
Christophe Leroy | 4b914286 | 2016-12-07 08:47:28 +0100 | [diff] [blame] | 111 | #endif |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 112 | |
Jan Beulich | 42b7772 | 2008-07-23 21:27:10 -0700 | [diff] [blame] | 113 | void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 114 | unsigned long end, unsigned long floor, |
| 115 | unsigned long ceiling); |
| 116 | |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 117 | /* |
| 118 | * If the arch doesn't supply something else, assume that hugepage |
| 119 | * size aligned regions are ok without further preparation. |
| 120 | */ |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 121 | static inline int prepare_hugepage_range(struct file *file, |
| 122 | unsigned long addr, unsigned long len) |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 123 | { |
Jon Tollefson | 0d9ea75 | 2008-07-23 21:27:56 -0700 | [diff] [blame] | 124 | struct hstate *h = hstate_file(file); |
| 125 | if (len & ~huge_page_mask(h)) |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 126 | return -EINVAL; |
Jon Tollefson | 0d9ea75 | 2008-07-23 21:27:56 -0700 | [diff] [blame] | 127 | if (addr & ~huge_page_mask(h)) |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 128 | return -EINVAL; |
| 129 | return 0; |
| 130 | } |
| 131 | |
David Gibson | 0895ecd | 2009-10-26 19:24:31 +0000 | [diff] [blame] | 132 | static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 133 | pte_t *ptep, pte_t pte) |
| 134 | { |
| 135 | set_pte_at(mm, addr, ptep, pte); |
| 136 | } |
| 137 | |
| 138 | static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, |
| 139 | unsigned long addr, pte_t *ptep) |
| 140 | { |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 141 | #ifdef CONFIG_PPC64 |
Aneesh Kumar K.V | 88247e8 | 2014-02-12 09:13:36 +0530 | [diff] [blame] | 142 | return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 143 | #else |
| 144 | return __pte(pte_update(ptep, ~0UL, 0)); |
| 145 | #endif |
David Gibson | 0895ecd | 2009-10-26 19:24:31 +0000 | [diff] [blame] | 146 | } |
| 147 | |
Gerald Schaefer | 8fe627e | 2008-04-28 02:13:28 -0700 | [diff] [blame] | 148 | static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, |
| 149 | unsigned long addr, pte_t *ptep) |
| 150 | { |
David Gibson | 0895ecd | 2009-10-26 19:24:31 +0000 | [diff] [blame] | 151 | pte_t pte; |
| 152 | pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); |
Aneesh Kumar K.V | 13dce03 | 2016-07-13 15:06:38 +0530 | [diff] [blame] | 153 | flush_hugetlb_page(vma, addr); |
Gerald Schaefer | 8fe627e | 2008-04-28 02:13:28 -0700 | [diff] [blame] | 154 | } |
| 155 | |
Gerald Schaefer | 7f2e952 | 2008-04-28 02:13:29 -0700 | [diff] [blame] | 156 | static inline int huge_pte_none(pte_t pte) |
| 157 | { |
| 158 | return pte_none(pte); |
| 159 | } |
| 160 | |
| 161 | static inline pte_t huge_pte_wrprotect(pte_t pte) |
| 162 | { |
| 163 | return pte_wrprotect(pte); |
| 164 | } |
| 165 | |
Aneesh Kumar K.V | f069ff3 | 2018-05-29 19:58:38 +0530 | [diff] [blame] | 166 | extern int huge_ptep_set_access_flags(struct vm_area_struct *vma, |
| 167 | unsigned long addr, pte_t *ptep, |
| 168 | pte_t pte, int dirty); |
Gerald Schaefer | 7f2e952 | 2008-04-28 02:13:29 -0700 | [diff] [blame] | 169 | |
| 170 | static inline pte_t huge_ptep_get(pte_t *ptep) |
| 171 | { |
| 172 | return *ptep; |
| 173 | } |
| 174 | |
Will Deacon | 5d3a551 | 2012-10-08 16:29:32 -0700 | [diff] [blame] | 175 | static inline void arch_clear_hugepage_flags(struct page *page) |
| 176 | { |
| 177 | } |
| 178 | |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 179 | #else /* ! CONFIG_HUGETLB_PAGE */ |
Becky Bruce | 41151e7 | 2011-06-28 09:54:48 +0000 | [diff] [blame] | 180 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, |
| 181 | unsigned long vmaddr) |
| 182 | { |
| 183 | } |
Becky Bruce | a614688 | 2011-10-10 10:50:43 +0000 | [diff] [blame] | 184 | |
Aneesh Kumar K.V | 2940999 | 2013-06-20 14:30:16 +0530 | [diff] [blame] | 185 | #define hugepd_shift(x) 0 |
Aneesh Kumar K.V | b30e759 | 2014-11-05 21:57:41 +0530 | [diff] [blame] | 186 | static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, |
Aneesh Kumar K.V | 2940999 | 2013-06-20 14:30:16 +0530 | [diff] [blame] | 187 | unsigned pdshift) |
| 188 | { |
Mathieu Malaterre | d873152 | 2018-04-13 20:41:43 +0200 | [diff] [blame] | 189 | return NULL; |
Aneesh Kumar K.V | 2940999 | 2013-06-20 14:30:16 +0530 | [diff] [blame] | 190 | } |
| 191 | #endif /* CONFIG_HUGETLB_PAGE */ |
Becky Bruce | a614688 | 2011-10-10 10:50:43 +0000 | [diff] [blame] | 192 | |
Gerald Schaefer | 6d77907 | 2008-04-28 02:13:27 -0700 | [diff] [blame] | 193 | #endif /* _ASM_POWERPC_HUGETLB_H */ |