blob: e2fb408f83983617591e3fe3511f9691b76f0b3d [file] [log] [blame]
Paul Mackerras047ea782005-11-19 20:17:32 +11001#ifndef _ASM_POWERPC_MMU_H_
2#define _ASM_POWERPC_MMU_H_
Arnd Bergmann88ced032005-12-16 22:43:46 +01003#ifdef __KERNEL__
Paul Mackerras047ea782005-11-19 20:17:32 +11004
Benjamin Herrenschmidtcd3db0c2010-07-06 15:39:02 -07005#include <linux/types.h>
6
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +00007#include <asm/asm-compat.h>
8#include <asm/feature-fixups.h>
9
10/*
11 * MMU features bit definitions
12 */
13
14/*
Aneesh Kumar K.V5a25b6f2016-07-27 13:19:01 +100015 * MMU families
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +000016 */
17#define MMU_FTR_HPTE_TABLE ASM_CONST(0x00000001)
18#define MMU_FTR_TYPE_8xx ASM_CONST(0x00000002)
19#define MMU_FTR_TYPE_40x ASM_CONST(0x00000004)
20#define MMU_FTR_TYPE_44x ASM_CONST(0x00000008)
21#define MMU_FTR_TYPE_FSL_E ASM_CONST(0x00000010)
Michael Ellermancd680982014-07-08 17:10:45 +100022#define MMU_FTR_TYPE_47x ASM_CONST(0x00000020)
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +000023
Aneesh Kumar K.V5a25b6f2016-07-27 13:19:01 +100024/* Radix page table supported and enabled */
25#define MMU_FTR_TYPE_RADIX ASM_CONST(0x00000040)
26
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +000027/*
Aneesh Kumar K.V5a25b6f2016-07-27 13:19:01 +100028 * Individual features below.
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +000029 */
Aneesh Kumar K.V5a25b6f2016-07-27 13:19:01 +100030
Aneesh Kumar K.Vaccfad72016-07-13 15:05:24 +053031/*
32 * We need to clear top 16bits of va (from the remaining 64 bits )in
33 * tlbie* instructions
34 */
35#define MMU_FTR_TLBIE_CROP_VA ASM_CONST(0x00008000)
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +000036
37/* Enable use of high BAT registers */
38#define MMU_FTR_USE_HIGH_BATS ASM_CONST(0x00010000)
39
40/* Enable >32-bit physical addresses on 32-bit processor, only used
41 * by CONFIG_6xx currently as BookE supports that from day 1
42 */
43#define MMU_FTR_BIG_PHYS ASM_CONST(0x00020000)
44
Benjamin Herrenschmidtf048aac2008-12-18 19:13:38 +000045/* Enable use of broadcast TLB invalidations. We don't always set it
46 * on processors that support it due to other constraints with the
47 * use of such invalidations
48 */
49#define MMU_FTR_USE_TLBIVAX_BCAST ASM_CONST(0x00040000)
50
Kumar Galac3071952009-02-10 22:26:06 -060051/* Enable use of tlbilx invalidate instructions.
Benjamin Herrenschmidtf048aac2008-12-18 19:13:38 +000052 */
Kumar Galac3071952009-02-10 22:26:06 -060053#define MMU_FTR_USE_TLBILX ASM_CONST(0x00080000)
Benjamin Herrenschmidtf048aac2008-12-18 19:13:38 +000054
55/* This indicates that the processor cannot handle multiple outstanding
56 * broadcast tlbivax or tlbsync. This makes the code use a spinlock
57 * around such invalidate forms.
58 */
59#define MMU_FTR_LOCK_BCAST_INVAL ASM_CONST(0x00100000)
60
Kumar Gala2319f122009-03-19 03:55:41 +000061/* This indicates that the processor doesn't handle way selection
62 * properly and needs SW to track and update the LRU state. This
63 * is specific to an errata on e300c2/c3/c4 class parts
64 */
65#define MMU_FTR_NEED_DTLB_SW_LRU ASM_CONST(0x00200000)
66
Kumar Galadf5d6ec2009-08-24 15:52:48 +000067/* Enable use of TLB reservation. Processor should support tlbsrx.
68 * instruction and MAS0[WQ].
69 */
70#define MMU_FTR_USE_TLBRSRV ASM_CONST(0x00800000)
71
72/* Use paired MAS registers (MAS7||MAS3, etc.)
73 */
74#define MMU_FTR_USE_PAIRED_MAS ASM_CONST(0x01000000)
75
Michael Ellerman13b3d132014-07-10 12:29:20 +100076/* Doesn't support the B bit (1T segment) in SLBIE
Matt Evans44ae3ab2011-04-06 19:48:50 +000077 */
Michael Ellerman13b3d132014-07-10 12:29:20 +100078#define MMU_FTR_NO_SLBIE_B ASM_CONST(0x02000000)
Matt Evans44ae3ab2011-04-06 19:48:50 +000079
80/* Support 16M large pages
81 */
82#define MMU_FTR_16M_PAGE ASM_CONST(0x04000000)
83
84/* Supports TLBIEL variant
85 */
86#define MMU_FTR_TLBIEL ASM_CONST(0x08000000)
87
88/* Supports tlbies w/o locking
89 */
90#define MMU_FTR_LOCKLESS_TLBIE ASM_CONST(0x10000000)
91
92/* Large pages can be marked CI
93 */
94#define MMU_FTR_CI_LARGE_PAGE ASM_CONST(0x20000000)
95
96/* 1T segments available
97 */
98#define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000)
99
Matt Evans44ae3ab2011-04-06 19:48:50 +0000100/* MMU feature bit sets for various CPUs */
101#define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 \
102 MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2
103#define MMU_FTRS_POWER4 MMU_FTRS_DEFAULT_HPTE_ARCH_V2
Aneesh Kumar K.Vaccfad72016-07-13 15:05:24 +0530104#define MMU_FTRS_PPC970 MMU_FTRS_POWER4 | MMU_FTR_TLBIE_CROP_VA
Matt Evans44ae3ab2011-04-06 19:48:50 +0000105#define MMU_FTRS_POWER5 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
106#define MMU_FTRS_POWER6 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
Michael Neulinga32e2522011-04-06 18:23:29 +0000107#define MMU_FTRS_POWER7 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
Michael Neuling71e18492012-10-30 19:34:15 +0000108#define MMU_FTRS_POWER8 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
Michael Neulingc3ab3002016-02-19 11:16:24 +1100109#define MMU_FTRS_POWER9 MMU_FTRS_POWER4 | MMU_FTR_LOCKLESS_TLBIE
Matt Evans44ae3ab2011-04-06 19:48:50 +0000110#define MMU_FTRS_CELL MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
111 MMU_FTR_CI_LARGE_PAGE
112#define MMU_FTRS_PA6T MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
113 MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000114#ifndef __ASSEMBLY__
Kevin Hao4db73272016-07-23 14:42:41 +0530115#include <linux/bug.h>
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000116#include <asm/cputable.h>
117
Becky Bruce3160b092011-06-28 14:54:47 -0500118#ifdef CONFIG_PPC_FSL_BOOK3E
119#include <asm/percpu.h>
120DECLARE_PER_CPU(int, next_tlbcam_idx);
121#endif
122
Michael Ellerman773edea2016-05-11 15:30:47 +1000123enum {
124 MMU_FTRS_POSSIBLE = MMU_FTR_HPTE_TABLE | MMU_FTR_TYPE_8xx |
125 MMU_FTR_TYPE_40x | MMU_FTR_TYPE_44x | MMU_FTR_TYPE_FSL_E |
126 MMU_FTR_TYPE_47x | MMU_FTR_USE_HIGH_BATS | MMU_FTR_BIG_PHYS |
127 MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_USE_TLBILX |
128 MMU_FTR_LOCK_BCAST_INVAL | MMU_FTR_NEED_DTLB_SW_LRU |
129 MMU_FTR_USE_TLBRSRV | MMU_FTR_USE_PAIRED_MAS |
130 MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
131 MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
Aneesh Kumar K.Vaccfad72016-07-13 15:05:24 +0530132 MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
Aneesh Kumar K.Va8ed87c2016-04-29 23:26:06 +1000133#ifdef CONFIG_PPC_RADIX_MMU
Aneesh Kumar K.V5a25b6f2016-07-27 13:19:01 +1000134 MMU_FTR_TYPE_RADIX |
Aneesh Kumar K.Va8ed87c2016-04-29 23:26:06 +1000135#endif
136 0,
Michael Ellerman773edea2016-05-11 15:30:47 +1000137};
138
Michael Ellermana141cca2016-07-27 20:48:36 +1000139static inline bool early_mmu_has_feature(unsigned long feature)
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000140{
Michael Ellermana81dc9d2016-07-27 13:39:42 +1000141 return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000142}
143
Kevin Haoc12e6f22016-07-23 14:42:42 +0530144#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
145#include <linux/jump_label.h>
146
147#define NUM_MMU_FTR_KEYS 32
148
149extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
150
151extern void mmu_feature_keys_init(void);
152
153static __always_inline bool mmu_has_feature(unsigned long feature)
154{
155 int i;
156
157 BUILD_BUG_ON(!__builtin_constant_p(feature));
158
Aneesh Kumar K.Vc812c7d2016-07-23 14:42:43 +0530159#ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
160 if (!static_key_initialized) {
161 printk("Warning! mmu_has_feature() used prior to jump label init!\n");
162 dump_stack();
163 return early_mmu_has_feature(feature);
164 }
165#endif
166
Kevin Haoc12e6f22016-07-23 14:42:42 +0530167 if (!(MMU_FTRS_POSSIBLE & feature))
168 return false;
169
170 i = __builtin_ctzl(feature);
171 return static_branch_likely(&mmu_feature_keys[i]);
172}
173
174static inline void mmu_clear_feature(unsigned long feature)
175{
176 int i;
177
178 i = __builtin_ctzl(feature);
179 cur_cpu_spec->mmu_features &= ~feature;
180 static_branch_disable(&mmu_feature_keys[i]);
181}
182#else
183
184static inline void mmu_feature_keys_init(void)
185{
186
187}
188
Michael Ellermana141cca2016-07-27 20:48:36 +1000189static inline bool mmu_has_feature(unsigned long feature)
190{
191 return early_mmu_has_feature(feature);
192}
193
Dave Kleikamp91b191c2011-07-04 18:38:03 +0000194static inline void mmu_clear_feature(unsigned long feature)
195{
196 cur_cpu_spec->mmu_features &= ~feature;
197}
Kevin Haoc12e6f22016-07-23 14:42:42 +0530198#endif /* CONFIG_JUMP_LABEL */
Dave Kleikamp91b191c2011-07-04 18:38:03 +0000199
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000200extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
201
Benjamin Herrenschmidtcd3db0c2010-07-06 15:39:02 -0700202#ifdef CONFIG_PPC64
203/* This is our real memory area size on ppc64 server, on embedded, we
204 * make it match the size our of bolted TLB area
205 */
206extern u64 ppc64_rma_size;
207#endif /* CONFIG_PPC64 */
208
Aneesh Kumar K.V78f1dbd2012-09-10 02:52:57 +0000209struct mm_struct;
210#ifdef CONFIG_DEBUG_VM
211extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
212#else /* CONFIG_DEBUG_VM */
213static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
214{
215}
216#endif /* !CONFIG_DEBUG_VM */
217
Michael Ellermanbab4c8d2016-07-27 13:37:58 +1000218#ifdef CONFIG_PPC_RADIX_MMU
219static inline bool radix_enabled(void)
220{
221 return mmu_has_feature(MMU_FTR_TYPE_RADIX);
222}
Michael Ellermana141cca2016-07-27 20:48:36 +1000223
224static inline bool early_radix_enabled(void)
225{
226 return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
227}
Michael Ellermanbab4c8d2016-07-27 13:37:58 +1000228#else
229static inline bool radix_enabled(void)
230{
231 return false;
232}
Michael Ellermana141cca2016-07-27 20:48:36 +1000233
234static inline bool early_radix_enabled(void)
235{
236 return false;
237}
Michael Ellermanbab4c8d2016-07-27 13:37:58 +1000238#endif
239
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000240#endif /* !__ASSEMBLY__ */
241
Benjamin Herrenschmidt57e2a992009-07-28 11:59:34 +1000242/* The kernel use the constants below to index in the page sizes array.
243 * The use of fixed constants for this purpose is better for performances
244 * of the low level hash refill handlers.
245 *
246 * A non supported page size has a "shift" field set to 0
247 *
248 * Any new page size being implemented can get a new entry in here. Whether
249 * the kernel will use it or not is a different matter though. The actual page
250 * size used by hugetlbfs is not defined here and may be made variable
251 *
252 * Note: This array ended up being a false good idea as it's growing to the
253 * point where I wonder if we should replace it with something different,
254 * to think about, feedback welcome. --BenH.
255 */
256
Scott Wooda8b91e42012-06-14 13:40:55 +0000257/* These are #defines as they have to be used in assembly */
Benjamin Herrenschmidt57e2a992009-07-28 11:59:34 +1000258#define MMU_PAGE_4K 0
259#define MMU_PAGE_16K 1
260#define MMU_PAGE_64K 2
261#define MMU_PAGE_64K_AP 3 /* "Admixed pages" (hash64 only) */
262#define MMU_PAGE_256K 4
263#define MMU_PAGE_1M 5
Scott Wood28efc352013-10-11 19:22:38 -0500264#define MMU_PAGE_2M 6
265#define MMU_PAGE_4M 7
266#define MMU_PAGE_8M 8
267#define MMU_PAGE_16M 9
268#define MMU_PAGE_64M 10
269#define MMU_PAGE_256M 11
270#define MMU_PAGE_1G 12
271#define MMU_PAGE_16G 13
272#define MMU_PAGE_64G 14
Benjamin Herrenschmidt57e2a992009-07-28 11:59:34 +1000273
Scott Wood28efc352013-10-11 19:22:38 -0500274#define MMU_PAGE_COUNT 15
Benjamin Herrenschmidt7c03d652008-12-18 19:13:32 +0000275
Aneesh Kumar K.V11a6f6a2016-04-29 23:25:41 +1000276#ifdef CONFIG_PPC_BOOK3S_64
277#include <asm/book3s/64/mmu.h>
278#else /* CONFIG_PPC_BOOK3S_64 */
279
Aneesh Kumar K.V756d08d2016-04-29 23:25:57 +1000280#ifndef __ASSEMBLY__
281/* MMU initialization */
282extern void early_init_mmu(void);
283extern void early_init_mmu_secondary(void);
284extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
285 phys_addr_t first_memblock_size);
Michael Ellerman1a01dc82016-07-26 20:09:30 +1000286static inline void mmu_early_init_devtree(void) { }
Aneesh Kumar K.V756d08d2016-04-29 23:25:57 +1000287#endif /* __ASSEMBLY__ */
Aneesh Kumar K.V11a6f6a2016-04-29 23:25:41 +1000288#endif
289
290#if defined(CONFIG_PPC_STD_MMU_32)
David Gibson4db68bf2007-06-13 14:52:54 +1000291/* 32-bit classic hash table MMU */
Aneesh Kumar K.Vf64e8082016-03-01 12:59:20 +0530292#include <asm/book3s/32/mmu-hash.h>
Josh Boyer4d922c82007-08-20 07:28:48 -0500293#elif defined(CONFIG_40x)
294/* 40x-style software loaded TLB */
295# include <asm/mmu-40x.h>
David Gibson57d79092007-04-30 14:06:25 +1000296#elif defined(CONFIG_44x)
297/* 44x-style software loaded TLB */
298# include <asm/mmu-44x.h>
Kumar Gala70fe3af2009-02-12 16:12:40 -0600299#elif defined(CONFIG_PPC_BOOK3E_MMU)
300/* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */
301# include <asm/mmu-book3e.h>
David Gibson31202342007-06-22 14:58:55 +1000302#elif defined (CONFIG_PPC_8xx)
303/* Motorola/Freescale 8xx software loaded TLB */
304# include <asm/mmu-8xx.h>
David Gibson1f8d4192005-05-05 16:15:13 -0700305#endif
David Gibson1f8d4192005-05-05 16:15:13 -0700306
Arnd Bergmann88ced032005-12-16 22:43:46 +0100307#endif /* __KERNEL__ */
Paul Mackerras047ea782005-11-19 20:17:32 +1100308#endif /* _ASM_POWERPC_MMU_H_ */