Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 1 | /* |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 2 | * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 1996-2000 Russell King |
| 5 | * Copyright (C) 2012 ARM Ltd. |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 18 | */ |
| 19 | #ifndef __ASSEMBLY__ |
| 20 | #error "Only include this from assembly code" |
| 21 | #endif |
| 22 | |
Marc Zyngier | f3e3927 | 2015-02-20 13:53:13 +0000 | [diff] [blame] | 23 | #ifndef __ASM_ASSEMBLER_H |
| 24 | #define __ASM_ASSEMBLER_H |
| 25 | |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 26 | #include <asm/asm-offsets.h> |
Andre Przywara | 823066d | 2016-06-28 18:07:29 +0100 | [diff] [blame] | 27 | #include <asm/cpufeature.h> |
Suzuki K Poulose | b8c3208 | 2018-03-26 15:12:49 +0100 | [diff] [blame] | 28 | #include <asm/cputype.h> |
Geoff Levand | 5003dbd | 2016-04-27 17:47:10 +0100 | [diff] [blame] | 29 | #include <asm/page.h> |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 30 | #include <asm/pgtable-hwdef.h> |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 31 | #include <asm/ptrace.h> |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 32 | #include <asm/thread_info.h> |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 33 | |
| 34 | /* |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 35 | * Enable and disable interrupts. |
| 36 | */ |
| 37 | .macro disable_irq |
| 38 | msr daifset, #2 |
| 39 | .endm |
| 40 | |
| 41 | .macro enable_irq |
| 42 | msr daifclr, #2 |
| 43 | .endm |
| 44 | |
| 45 | /* |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 46 | * Enable and disable debug exceptions. |
| 47 | */ |
| 48 | .macro disable_dbg |
| 49 | msr daifset, #8 |
| 50 | .endm |
| 51 | |
| 52 | .macro enable_dbg |
| 53 | msr daifclr, #8 |
| 54 | .endm |
| 55 | |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 56 | .macro disable_step_tsk, flgs, tmp |
| 57 | tbz \flgs, #TIF_SINGLESTEP, 9990f |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 58 | mrs \tmp, mdscr_el1 |
| 59 | bic \tmp, \tmp, #1 |
| 60 | msr mdscr_el1, \tmp |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 61 | isb // Synchronise with enable_dbg |
| 62 | 9990: |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 63 | .endm |
| 64 | |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 65 | .macro enable_step_tsk, flgs, tmp |
| 66 | tbz \flgs, #TIF_SINGLESTEP, 9990f |
| 67 | disable_dbg |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 68 | mrs \tmp, mdscr_el1 |
| 69 | orr \tmp, \tmp, #1 |
| 70 | msr mdscr_el1, \tmp |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 71 | 9990: |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 72 | .endm |
| 73 | |
Will Deacon | 2a28307 | 2014-04-29 19:04:06 +0100 | [diff] [blame] | 74 | /* |
| 75 | * Enable both debug exceptions and interrupts. This is likely to be |
| 76 | * faster than two daifclr operations, since writes to this register |
| 77 | * are self-synchronising. |
| 78 | */ |
| 79 | .macro enable_dbg_and_irq |
| 80 | msr daifclr, #(8 | 2) |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 81 | .endm |
| 82 | |
| 83 | /* |
| 84 | * SMP data memory barrier |
| 85 | */ |
| 86 | .macro smp_dmb, opt |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 87 | dmb \opt |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 88 | .endm |
| 89 | |
Ard Biesheuvel | 6c94f27 | 2016-01-01 15:02:12 +0100 | [diff] [blame] | 90 | /* |
Mark Rutland | afc0954 | 2018-04-12 12:10:57 +0100 | [diff] [blame] | 91 | * Value prediction barrier |
| 92 | */ |
| 93 | .macro csdb |
| 94 | hint #20 |
| 95 | .endm |
| 96 | |
| 97 | /* |
Mark Rutland | f3ed64a | 2018-04-12 12:11:02 +0100 | [diff] [blame] | 98 | * Sanitise a 64-bit bounded index wrt speculation, returning zero if out |
| 99 | * of bounds. |
| 100 | */ |
| 101 | .macro mask_nospec64, idx, limit, tmp |
| 102 | sub \tmp, \idx, \limit |
| 103 | bic \tmp, \tmp, \idx |
| 104 | and \idx, \idx, \tmp, asr #63 |
| 105 | csdb |
| 106 | .endm |
| 107 | |
| 108 | /* |
Will Deacon | f99a250 | 2016-09-06 16:40:23 +0100 | [diff] [blame] | 109 | * NOP sequence |
| 110 | */ |
| 111 | .macro nops, num |
| 112 | .rept \num |
| 113 | nop |
| 114 | .endr |
| 115 | .endm |
| 116 | |
| 117 | /* |
Ard Biesheuvel | 6c94f27 | 2016-01-01 15:02:12 +0100 | [diff] [blame] | 118 | * Emit an entry into the exception table |
| 119 | */ |
| 120 | .macro _asm_extable, from, to |
| 121 | .pushsection __ex_table, "a" |
| 122 | .align 3 |
| 123 | .long (\from - .), (\to - .) |
| 124 | .popsection |
| 125 | .endm |
| 126 | |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 127 | #define USER(l, x...) \ |
| 128 | 9999: x; \ |
Ard Biesheuvel | 6c94f27 | 2016-01-01 15:02:12 +0100 | [diff] [blame] | 129 | _asm_extable 9999b, l |
Catalin Marinas | 0be7320 | 2012-03-05 11:49:26 +0000 | [diff] [blame] | 130 | |
| 131 | /* |
| 132 | * Register aliases. |
| 133 | */ |
| 134 | lr .req x30 // link register |
Marc Zyngier | dc637f1 | 2012-10-19 17:37:35 +0100 | [diff] [blame] | 135 | |
| 136 | /* |
| 137 | * Vector entry |
| 138 | */ |
| 139 | .macro ventry label |
| 140 | .align 7 |
| 141 | b \label |
| 142 | .endm |
Matthew Leach | e68beda | 2013-10-11 14:52:15 +0100 | [diff] [blame] | 143 | |
| 144 | /* |
| 145 | * Select code when configured for BE. |
| 146 | */ |
| 147 | #ifdef CONFIG_CPU_BIG_ENDIAN |
| 148 | #define CPU_BE(code...) code |
| 149 | #else |
| 150 | #define CPU_BE(code...) |
| 151 | #endif |
| 152 | |
| 153 | /* |
| 154 | * Select code when configured for LE. |
| 155 | */ |
| 156 | #ifdef CONFIG_CPU_BIG_ENDIAN |
| 157 | #define CPU_LE(code...) |
| 158 | #else |
| 159 | #define CPU_LE(code...) code |
| 160 | #endif |
| 161 | |
Matthew Leach | 55b8954 | 2013-10-11 14:52:13 +0100 | [diff] [blame] | 162 | /* |
| 163 | * Define a macro that constructs a 64-bit value by concatenating two |
| 164 | * 32-bit registers. Note that on big endian systems the order of the |
| 165 | * registers is swapped. |
| 166 | */ |
| 167 | #ifndef CONFIG_CPU_BIG_ENDIAN |
| 168 | .macro regs_to_64, rd, lbits, hbits |
| 169 | #else |
| 170 | .macro regs_to_64, rd, hbits, lbits |
| 171 | #endif |
| 172 | orr \rd, \lbits, \hbits, lsl #32 |
| 173 | .endm |
Marc Zyngier | f3e3927 | 2015-02-20 13:53:13 +0000 | [diff] [blame] | 174 | |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 175 | /* |
| 176 | * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 177 | * <symbol> is within the range +/- 4 GB of the PC when running |
| 178 | * in core kernel context. In module context, a movz/movk sequence |
| 179 | * is used, since modules may be loaded far away from the kernel |
| 180 | * when KASLR is in effect. |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 181 | */ |
| 182 | /* |
| 183 | * @dst: destination register (64 bit wide) |
| 184 | * @sym: name of the symbol |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 185 | */ |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 186 | .macro adr_l, dst, sym |
| 187 | #ifndef MODULE |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 188 | adrp \dst, \sym |
| 189 | add \dst, \dst, :lo12:\sym |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 190 | #else |
| 191 | movz \dst, #:abs_g3:\sym |
| 192 | movk \dst, #:abs_g2_nc:\sym |
| 193 | movk \dst, #:abs_g1_nc:\sym |
| 194 | movk \dst, #:abs_g0_nc:\sym |
| 195 | #endif |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 196 | .endm |
| 197 | |
| 198 | /* |
| 199 | * @dst: destination register (32 or 64 bit wide) |
| 200 | * @sym: name of the symbol |
| 201 | * @tmp: optional 64-bit scratch register to be used if <dst> is a |
| 202 | * 32-bit wide register, in which case it cannot be used to hold |
| 203 | * the address |
| 204 | */ |
| 205 | .macro ldr_l, dst, sym, tmp= |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 206 | #ifndef MODULE |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 207 | .ifb \tmp |
| 208 | adrp \dst, \sym |
| 209 | ldr \dst, [\dst, :lo12:\sym] |
| 210 | .else |
| 211 | adrp \tmp, \sym |
| 212 | ldr \dst, [\tmp, :lo12:\sym] |
| 213 | .endif |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 214 | #else |
| 215 | .ifb \tmp |
| 216 | adr_l \dst, \sym |
| 217 | ldr \dst, [\dst] |
| 218 | .else |
| 219 | adr_l \tmp, \sym |
| 220 | ldr \dst, [\tmp] |
| 221 | .endif |
| 222 | #endif |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 223 | .endm |
| 224 | |
| 225 | /* |
| 226 | * @src: source register (32 or 64 bit wide) |
| 227 | * @sym: name of the symbol |
| 228 | * @tmp: mandatory 64-bit scratch register to calculate the address |
| 229 | * while <src> needs to be preserved. |
| 230 | */ |
| 231 | .macro str_l, src, sym, tmp |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 232 | #ifndef MODULE |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 233 | adrp \tmp, \sym |
| 234 | str \src, [\tmp, :lo12:\sym] |
Ard Biesheuvel | f88f06e | 2017-01-11 14:54:53 +0000 | [diff] [blame] | 235 | #else |
| 236 | adr_l \tmp, \sym |
| 237 | str \src, [\tmp] |
| 238 | #endif |
Ard Biesheuvel | b784a5d | 2015-03-04 19:45:38 +0100 | [diff] [blame] | 239 | .endm |
| 240 | |
James Morse | aa4d5d3 | 2015-12-10 10:22:39 +0000 | [diff] [blame] | 241 | /* |
Mark Rutland | c488ae4 | 2018-07-20 10:56:12 +0100 | [diff] [blame] | 242 | * @dst: Result of per_cpu(sym, smp_processor_id()) |
James Morse | aa4d5d3 | 2015-12-10 10:22:39 +0000 | [diff] [blame] | 243 | * @sym: The name of the per-cpu variable |
James Morse | aa4d5d3 | 2015-12-10 10:22:39 +0000 | [diff] [blame] | 244 | * @tmp: scratch register |
| 245 | */ |
Mark Rutland | c488ae4 | 2018-07-20 10:56:12 +0100 | [diff] [blame] | 246 | .macro adr_this_cpu, dst, sym, tmp |
| 247 | adr_l \dst, \sym |
James Morse | eea5902 | 2018-07-20 10:56:16 +0100 | [diff] [blame] | 248 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
James Morse | aa4d5d3 | 2015-12-10 10:22:39 +0000 | [diff] [blame] | 249 | mrs \tmp, tpidr_el1 |
James Morse | eea5902 | 2018-07-20 10:56:16 +0100 | [diff] [blame] | 250 | alternative_else |
| 251 | mrs \tmp, tpidr_el2 |
| 252 | alternative_endif |
Mark Rutland | c488ae4 | 2018-07-20 10:56:12 +0100 | [diff] [blame] | 253 | add \dst, \dst, \tmp |
| 254 | .endm |
| 255 | |
| 256 | /* |
| 257 | * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) |
| 258 | * @sym: The name of the per-cpu variable |
| 259 | * @tmp: scratch register |
| 260 | */ |
| 261 | .macro ldr_this_cpu dst, sym, tmp |
| 262 | adr_l \dst, \sym |
James Morse | eea5902 | 2018-07-20 10:56:16 +0100 | [diff] [blame] | 263 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
Mark Rutland | c488ae4 | 2018-07-20 10:56:12 +0100 | [diff] [blame] | 264 | mrs \tmp, tpidr_el1 |
James Morse | eea5902 | 2018-07-20 10:56:16 +0100 | [diff] [blame] | 265 | alternative_else |
| 266 | mrs \tmp, tpidr_el2 |
| 267 | alternative_endif |
Mark Rutland | c488ae4 | 2018-07-20 10:56:12 +0100 | [diff] [blame] | 268 | ldr \dst, [\dst, \tmp] |
James Morse | aa4d5d3 | 2015-12-10 10:22:39 +0000 | [diff] [blame] | 269 | .endm |
| 270 | |
Ard Biesheuvel | 2079184 | 2015-10-08 20:02:03 +0100 | [diff] [blame] | 271 | /* |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 272 | * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) |
| 273 | */ |
| 274 | .macro vma_vm_mm, rd, rn |
| 275 | ldr \rd, [\rn, #VMA_VM_MM] |
| 276 | .endm |
| 277 | |
| 278 | /* |
| 279 | * mmid - get context id from mm pointer (mm->context.id) |
| 280 | */ |
| 281 | .macro mmid, rd, rn |
| 282 | ldr \rd, [\rn, #MM_CONTEXT_ID] |
| 283 | .endm |
Suzuki K Poulose | 116c81f | 2016-09-09 14:07:16 +0100 | [diff] [blame] | 284 | /* |
| 285 | * read_ctr - read CTR_EL0. If the system has mismatched |
| 286 | * cache line sizes, provide the system wide safe value |
| 287 | * from arm64_ftr_reg_ctrel0.sys_val |
| 288 | */ |
| 289 | .macro read_ctr, reg |
| 290 | alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE |
| 291 | mrs \reg, ctr_el0 // read CTR |
| 292 | nop |
| 293 | alternative_else |
| 294 | ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL |
| 295 | alternative_endif |
| 296 | .endm |
| 297 | |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 298 | |
| 299 | /* |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 300 | * raw_dcache_line_size - get the minimum D-cache line size on this CPU |
| 301 | * from the CTR register. |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 302 | */ |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 303 | .macro raw_dcache_line_size, reg, tmp |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 304 | mrs \tmp, ctr_el0 // read CTR |
| 305 | ubfm \tmp, \tmp, #16, #19 // cache line size encoding |
| 306 | mov \reg, #4 // bytes per word |
| 307 | lsl \reg, \reg, \tmp // actual cache line size |
| 308 | .endm |
| 309 | |
| 310 | /* |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 311 | * dcache_line_size - get the safe D-cache line size across all CPUs |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 312 | */ |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 313 | .macro dcache_line_size, reg, tmp |
Suzuki K Poulose | 116c81f | 2016-09-09 14:07:16 +0100 | [diff] [blame] | 314 | read_ctr \tmp |
| 315 | ubfm \tmp, \tmp, #16, #19 // cache line size encoding |
| 316 | mov \reg, #4 // bytes per word |
| 317 | lsl \reg, \reg, \tmp // actual cache line size |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 318 | .endm |
| 319 | |
| 320 | /* |
| 321 | * raw_icache_line_size - get the minimum I-cache line size on this CPU |
| 322 | * from the CTR register. |
| 323 | */ |
| 324 | .macro raw_icache_line_size, reg, tmp |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 325 | mrs \tmp, ctr_el0 // read CTR |
| 326 | and \tmp, \tmp, #0xf // cache line size encoding |
| 327 | mov \reg, #4 // bytes per word |
| 328 | lsl \reg, \reg, \tmp // actual cache line size |
| 329 | .endm |
| 330 | |
| 331 | /* |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 332 | * icache_line_size - get the safe I-cache line size across all CPUs |
| 333 | */ |
| 334 | .macro icache_line_size, reg, tmp |
Suzuki K Poulose | 116c81f | 2016-09-09 14:07:16 +0100 | [diff] [blame] | 335 | read_ctr \tmp |
| 336 | and \tmp, \tmp, #0xf // cache line size encoding |
| 337 | mov \reg, #4 // bytes per word |
| 338 | lsl \reg, \reg, \tmp // actual cache line size |
Suzuki K Poulose | 072f0a6 | 2016-09-09 14:07:14 +0100 | [diff] [blame] | 339 | .endm |
| 340 | |
| 341 | /* |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 342 | * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map |
| 343 | */ |
| 344 | .macro tcr_set_idmap_t0sz, valreg, tmpreg |
| 345 | #ifndef CONFIG_ARM64_VA_BITS_48 |
| 346 | ldr_l \tmpreg, idmap_t0sz |
| 347 | bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH |
| 348 | #endif |
| 349 | .endm |
| 350 | |
| 351 | /* |
| 352 | * Macro to perform a data cache maintenance for the interval |
| 353 | * [kaddr, kaddr + size) |
| 354 | * |
| 355 | * op: operation passed to dc instruction |
| 356 | * domain: domain used in dsb instruciton |
| 357 | * kaddr: starting virtual address of the region |
| 358 | * size: size of the region |
| 359 | * Corrupts: kaddr, size, tmp1, tmp2 |
| 360 | */ |
| 361 | .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 |
| 362 | dcache_line_size \tmp1, \tmp2 |
| 363 | add \size, \kaddr, \size |
| 364 | sub \tmp2, \tmp1, #1 |
| 365 | bic \kaddr, \kaddr, \tmp2 |
Andre Przywara | 823066d | 2016-06-28 18:07:29 +0100 | [diff] [blame] | 366 | 9998: |
| 367 | .if (\op == cvau || \op == cvac) |
| 368 | alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE |
| 369 | dc \op, \kaddr |
| 370 | alternative_else |
| 371 | dc civac, \kaddr |
| 372 | alternative_endif |
| 373 | .else |
| 374 | dc \op, \kaddr |
| 375 | .endif |
Geoff Levand | 7b7293a | 2016-04-27 17:47:00 +0100 | [diff] [blame] | 376 | add \kaddr, \kaddr, \tmp1 |
| 377 | cmp \kaddr, \size |
| 378 | b.lo 9998b |
| 379 | dsb \domain |
| 380 | .endm |
| 381 | |
| 382 | /* |
| 383 | * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present |
| 384 | */ |
| 385 | .macro reset_pmuserenr_el0, tmpreg |
| 386 | mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer |
| 387 | sbfx \tmpreg, \tmpreg, #8, #4 |
| 388 | cmp \tmpreg, #1 // Skip if no PMU present |
| 389 | b.lt 9000f |
| 390 | msr pmuserenr_el0, xzr // Disable PMU access from EL0 |
| 391 | 9000: |
| 392 | .endm |
| 393 | |
| 394 | /* |
Geoff Levand | 5003dbd | 2016-04-27 17:47:10 +0100 | [diff] [blame] | 395 | * copy_page - copy src to dest using temp registers t1-t8 |
| 396 | */ |
| 397 | .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req |
| 398 | 9998: ldp \t1, \t2, [\src] |
| 399 | ldp \t3, \t4, [\src, #16] |
| 400 | ldp \t5, \t6, [\src, #32] |
| 401 | ldp \t7, \t8, [\src, #48] |
| 402 | add \src, \src, #64 |
| 403 | stnp \t1, \t2, [\dest] |
| 404 | stnp \t3, \t4, [\dest, #16] |
| 405 | stnp \t5, \t6, [\dest, #32] |
| 406 | stnp \t7, \t8, [\dest, #48] |
| 407 | add \dest, \dest, #64 |
| 408 | tst \src, #(PAGE_SIZE - 1) |
| 409 | b.ne 9998b |
| 410 | .endm |
| 411 | |
| 412 | /* |
Ard Biesheuvel | 2079184 | 2015-10-08 20:02:03 +0100 | [diff] [blame] | 413 | * Annotate a function as position independent, i.e., safe to be called before |
| 414 | * the kernel virtual mapping is activated. |
| 415 | */ |
| 416 | #define ENDPIPROC(x) \ |
| 417 | .globl __pi_##x; \ |
| 418 | .type __pi_##x, %function; \ |
| 419 | .set __pi_##x, x; \ |
| 420 | .size __pi_##x, . - x; \ |
| 421 | ENDPROC(x) |
| 422 | |
Ard Biesheuvel | 6ad1fe5 | 2015-12-26 13:48:02 +0100 | [diff] [blame] | 423 | /* |
| 424 | * Emit a 64-bit absolute little endian symbol reference in a way that |
| 425 | * ensures that it will be resolved at build time, even when building a |
| 426 | * PIE binary. This requires cooperation from the linker script, which |
| 427 | * must emit the lo32/hi32 halves individually. |
| 428 | */ |
| 429 | .macro le64sym, sym |
| 430 | .long \sym\()_lo32 |
| 431 | .long \sym\()_hi32 |
| 432 | .endm |
| 433 | |
Ard Biesheuvel | 30b5ba5 | 2016-04-18 17:09:44 +0200 | [diff] [blame] | 434 | /* |
| 435 | * mov_q - move an immediate constant into a 64-bit register using |
| 436 | * between 2 and 4 movz/movk instructions (depending on the |
| 437 | * magnitude and sign of the operand) |
| 438 | */ |
| 439 | .macro mov_q, reg, val |
| 440 | .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) |
| 441 | movz \reg, :abs_g1_s:\val |
| 442 | .else |
| 443 | .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) |
| 444 | movz \reg, :abs_g2_s:\val |
| 445 | .else |
| 446 | movz \reg, :abs_g3:\val |
| 447 | movk \reg, :abs_g2_nc:\val |
| 448 | .endif |
| 449 | movk \reg, :abs_g1_nc:\val |
| 450 | .endif |
| 451 | movk \reg, :abs_g0_nc:\val |
| 452 | .endm |
| 453 | |
Will Deacon | 4025fe1 | 2018-04-03 12:09:20 +0100 | [diff] [blame] | 454 | .macro pte_to_phys, phys, pte |
| 455 | and \phys, \pte, #(((1 << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT) |
| 456 | .endm |
Mark Rutland | 965924e | 2018-04-12 12:11:11 +0100 | [diff] [blame] | 457 | |
Suzuki K Poulose | b8c3208 | 2018-03-26 15:12:49 +0100 | [diff] [blame] | 458 | /* |
| 459 | * Check the MIDR_EL1 of the current CPU for a given model and a range of |
| 460 | * variant/revision. See asm/cputype.h for the macros used below. |
| 461 | * |
| 462 | * model: MIDR_CPU_MODEL of CPU |
| 463 | * rv_min: Minimum of MIDR_CPU_VAR_REV() |
| 464 | * rv_max: Maximum of MIDR_CPU_VAR_REV() |
| 465 | * res: Result register. |
| 466 | * tmp1, tmp2, tmp3: Temporary registers |
| 467 | * |
| 468 | * Corrupts: res, tmp1, tmp2, tmp3 |
| 469 | * Returns: 0, if the CPU id doesn't match. Non-zero otherwise |
| 470 | */ |
| 471 | .macro cpu_midr_match model, rv_min, rv_max, res, tmp1, tmp2, tmp3 |
| 472 | mrs \res, midr_el1 |
| 473 | mov_q \tmp1, (MIDR_REVISION_MASK | MIDR_VARIANT_MASK) |
| 474 | mov_q \tmp2, MIDR_CPU_MODEL_MASK |
| 475 | and \tmp3, \res, \tmp2 // Extract model |
| 476 | and \tmp1, \res, \tmp1 // rev & variant |
| 477 | mov_q \tmp2, \model |
| 478 | cmp \tmp3, \tmp2 |
| 479 | cset \res, eq |
| 480 | cbz \res, .Ldone\@ // Model matches ? |
| 481 | |
| 482 | .if (\rv_min != 0) // Skip min check if rv_min == 0 |
| 483 | mov_q \tmp3, \rv_min |
| 484 | cmp \tmp1, \tmp3 |
| 485 | cset \res, ge |
| 486 | .endif // \rv_min != 0 |
| 487 | /* Skip rv_max check if rv_min == rv_max && rv_min != 0 */ |
| 488 | .if ((\rv_min != \rv_max) || \rv_min == 0) |
| 489 | mov_q \tmp2, \rv_max |
| 490 | cmp \tmp1, \tmp2 |
| 491 | cset \tmp2, le |
| 492 | and \res, \res, \tmp2 |
| 493 | .endif |
| 494 | .Ldone\@: |
| 495 | .endm |
| 496 | |
Marc Zyngier | f3e3927 | 2015-02-20 13:53:13 +0000 | [diff] [blame] | 497 | #endif /* __ASM_ASSEMBLER_H */ |