blob: 352c73b6a59ea3817c9f2c79a867e704b7c1009e [file] [log] [blame]
Catalin Marinas9cce7a42012-03-05 11:49:28 +00001/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
Catalin Marinas9cce7a42012-03-05 11:49:28 +000026#include <asm/pgtable.h>
James Morsecabe1c82016-04-27 17:47:07 +010027#include <asm/pgtable-hwdef.h>
Andrew Pinski104a0c02016-02-24 17:44:57 -080028#include <asm/cpufeature.h>
29#include <asm/alternative.h>
Catalin Marinas9cce7a42012-03-05 11:49:28 +000030
Catalin Marinas35a86972014-04-02 17:55:40 +010031#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
Suzuki K. Poulose44eaacf2015-10-19 14:19:37 +010033#elif defined(CONFIG_ARM64_16K_PAGES)
34#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
35#else /* CONFIG_ARM64_4K_PAGES */
Catalin Marinas35a86972014-04-02 17:55:40 +010036#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
Catalin Marinas9cce7a42012-03-05 11:49:28 +000037#endif
38
Catalin Marinas35a86972014-04-02 17:55:40 +010039#define TCR_SMP_FLAGS TCR_SHARED
Catalin Marinas35a86972014-04-02 17:55:40 +010040
41/* PTWs cacheable, inner/outer WBWA */
42#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
43
Catalin Marinas9cce7a42012-03-05 11:49:28 +000044#define MAIR(attr, mt) ((attr) << ((mt) * 8))
45
46/*
Catalin Marinas9cce7a42012-03-05 11:49:28 +000047 * cpu_do_idle()
48 *
49 * Idle the processor (wait for interrupt).
50 */
51ENTRY(cpu_do_idle)
52 dsb sy // WFI may enter a low-power mode
53 wfi
54 ret
55ENDPROC(cpu_do_idle)
56
Lorenzo Pieralisiaf3cfdb2015-01-26 18:33:44 +000057#ifdef CONFIG_CPU_PM
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010058/**
59 * cpu_do_suspend - save CPU registers context
60 *
61 * x0: virtual address of context pointer
62 */
63ENTRY(cpu_do_suspend)
64 mrs x2, tpidr_el0
65 mrs x3, tpidrro_el0
66 mrs x4, contextidr_el1
James Morsecabe1c82016-04-27 17:47:07 +010067 mrs x5, cpacr_el1
68 mrs x6, tcr_el1
69 mrs x7, vbar_el1
70 mrs x8, mdscr_el1
71 mrs x9, oslsr_el1
72 mrs x10, sctlr_el1
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010073 stp x2, x3, [x0]
James Morsecabe1c82016-04-27 17:47:07 +010074 stp x4, xzr, [x0, #16]
75 stp x5, x6, [x0, #32]
76 stp x7, x8, [x0, #48]
77 stp x9, x10, [x0, #64]
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010078 ret
79ENDPROC(cpu_do_suspend)
80
81/**
82 * cpu_do_resume - restore CPU register context
83 *
James Morsecabe1c82016-04-27 17:47:07 +010084 * x0: Address of context pointer
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010085 */
James Morseb6113032016-08-24 18:27:29 +010086 .pushsection ".idmap.text", "ax"
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010087ENTRY(cpu_do_resume)
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010088 ldp x2, x3, [x0]
89 ldp x4, x5, [x0, #16]
James Morsecabe1c82016-04-27 17:47:07 +010090 ldp x6, x8, [x0, #32]
91 ldp x9, x10, [x0, #48]
92 ldp x11, x12, [x0, #64]
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010093 msr tpidr_el0, x2
94 msr tpidrro_el0, x3
95 msr contextidr_el1, x4
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +010096 msr cpacr_el1, x6
James Morsecabe1c82016-04-27 17:47:07 +010097
98 /* Don't change t0sz here, mask those bits when restoring */
99 mrs x5, tcr_el1
100 bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
101
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +0100102 msr tcr_el1, x8
103 msr vbar_el1, x9
James Morse744c6c32016-08-26 16:03:42 +0100104
105 /*
106 * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking
107 * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug
108 * exception. Mask them until local_dbg_restore() in cpu_suspend()
109 * resets them.
110 */
111 disable_dbg
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +0100112 msr mdscr_el1, x10
James Morse744c6c32016-08-26 16:03:42 +0100113
James Morsecabe1c82016-04-27 17:47:07 +0100114 msr sctlr_el1, x12
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +0100115 /*
116 * Restore oslsr_el1 by writing oslar_el1
117 */
118 ubfx x11, x11, #1, #1
119 msr oslar_el1, x11
Lorenzo Pieralisif436b2a2016-01-13 14:50:03 +0000120 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +0100121 isb
122 ret
123ENDPROC(cpu_do_resume)
James Morseb6113032016-08-24 18:27:29 +0100124 .popsection
Lorenzo Pieralisi6732bc62013-07-17 10:14:45 +0100125#endif
126
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000127/*
Jingoo Han812944e2014-01-27 07:19:32 +0000128 * cpu_do_switch_mm(pgd_phys, tsk)
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000129 *
130 * Set the translation table base pointer to be pgd_phys.
131 *
132 * - pgd_phys - physical address of new TTB
133 */
134ENTRY(cpu_do_switch_mm)
Will Deacon5aec7152015-10-06 18:46:24 +0100135 mmid x1, x1 // get mm->context.id
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000136 bfi x0, x1, #48, #16 // set the ASID
137 msr ttbr0_el1, x0 // set TTBR0
138 isb
Mark Rutland6ba3b552016-09-07 11:07:09 +0100139alternative_if ARM64_WORKAROUND_CAVIUM_27456
Andrew Pinski104a0c02016-02-24 17:44:57 -0800140 ic iallu
141 dsb nsh
142 isb
Mark Rutland6ba3b552016-09-07 11:07:09 +0100143alternative_else_nop_endif
Andrew Pinski104a0c02016-02-24 17:44:57 -0800144 ret
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000145ENDPROC(cpu_do_switch_mm)
146
Mark Rutland50e18812016-01-25 11:45:01 +0000147 .pushsection ".idmap.text", "ax"
148/*
149 * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
150 *
151 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
152 * called by anything else. It can only be executed from a TTBR0 mapping.
153 */
154ENTRY(idmap_cpu_replace_ttbr1)
155 mrs x2, daif
156 msr daifset, #0xf
157
158 adrp x1, empty_zero_page
159 msr ttbr1_el1, x1
160 isb
161
162 tlbi vmalle1
163 dsb nsh
164 isb
165
166 msr ttbr1_el1, x0
167 isb
168
169 msr daif, x2
170
171 ret
172ENDPROC(idmap_cpu_replace_ttbr1)
173 .popsection
174
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000175/*
176 * __cpu_setup
177 *
178 * Initialise the processor for turning the MMU on. Return in x0 the
179 * value of the SCTLR_EL1 register.
180 */
James Morseb6113032016-08-24 18:27:29 +0100181 .pushsection ".idmap.text", "ax"
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000182ENTRY(__cpu_setup)
Will Deaconfa7aae82015-10-06 18:46:22 +0100183 tlbi vmalle1 // Invalidate local TLB
184 dsb nsh
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000185
186 mov x0, #3 << 20
187 msr cpacr_el1, x0 // Enable FP/ASIMD
Will Deacond8d23fa2015-08-20 11:47:13 +0100188 mov x0, #1 << 12 // Reset mdscr_el1 and disable
189 msr mdscr_el1, x0 // access to the DCC from EL0
Will Deacon2ce39ad2016-07-19 15:07:37 +0100190 isb // Unmask debug exceptions now,
191 enable_dbg // since this is per-cpu
Lorenzo Pieralisif436b2a2016-01-13 14:50:03 +0000192 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000193 /*
194 * Memory region attributes for LPAE:
195 *
196 * n = AttrIndx[2:0]
197 * n MAIR
198 * DEVICE_nGnRnE 000 00000000
199 * DEVICE_nGnRE 001 00000100
200 * DEVICE_GRE 010 00001100
201 * NORMAL_NC 011 01000100
202 * NORMAL 100 11111111
Jonathan (Zhixiong) Zhang8d446c82015-08-07 09:36:59 +0100203 * NORMAL_WT 101 10111011
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000204 */
205 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
206 MAIR(0x04, MT_DEVICE_nGnRE) | \
207 MAIR(0x0c, MT_DEVICE_GRE) | \
208 MAIR(0x44, MT_NORMAL_NC) | \
Jonathan (Zhixiong) Zhang8d446c82015-08-07 09:36:59 +0100209 MAIR(0xff, MT_NORMAL) | \
210 MAIR(0xbb, MT_NORMAL_WT)
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000211 msr mair_el1, x5
212 /*
213 * Prepare SCTLR
214 */
215 adr x5, crval
216 ldp w5, w6, [x5]
217 mrs x0, sctlr_el1
218 bic x0, x0, x5 // clear bits
219 orr x0, x0, x6 // set bits
220 /*
221 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
222 * both user and kernel.
223 */
Catalin Marinas35a86972014-04-02 17:55:40 +0100224 ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
225 TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
Ard Biesheuveldd006da2015-03-19 16:42:27 +0000226 tcr_set_idmap_t0sz x10, x9
227
Radha Mohan Chintakuntla87366d82014-03-07 08:49:25 +0000228 /*
229 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
230 * TCR_EL1.
231 */
232 mrs x9, ID_AA64MMFR0_EL1
233 bfi x10, x9, #32, #3
Catalin Marinas2f4b8292015-07-10 17:24:28 +0100234#ifdef CONFIG_ARM64_HW_AFDBM
235 /*
236 * Hardware update of the Access and Dirty bits.
237 */
238 mrs x9, ID_AA64MMFR1_EL1
239 and x9, x9, #0xf
240 cbz x9, 2f
241 cmp x9, #2
242 b.lt 1f
243 orr x10, x10, #TCR_HD // hardware Dirty flag update
2441: orr x10, x10, #TCR_HA // hardware Access flag update
2452:
246#endif /* CONFIG_ARM64_HW_AFDBM */
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000247 msr tcr_el1, x10
248 ret // return to head.S
249ENDPROC(__cpu_setup)
250
251 /*
Suzuki K. Poulose9f71ac92014-12-17 15:50:21 +0000252 * We set the desired value explicitly, including those of the
253 * reserved bits. The values of bits EE & E0E were set early in
254 * el2_setup, which are left untouched below.
255 *
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000256 * n n T
257 * U E WT T UD US IHBS
258 * CE0 XWHW CZ ME TEEA S
259 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
Suzuki K. Poulose9f71ac92014-12-17 15:50:21 +0000260 * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
261 * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
Catalin Marinas9cce7a42012-03-05 11:49:28 +0000262 */
263 .type crval, #object
264crval:
Suzuki K. Poulose9f71ac92014-12-17 15:50:21 +0000265 .word 0xfcffffff // clear
266 .word 0x34d5d91d // set
James Morseb6113032016-08-24 18:27:29 +0100267 .popsection