blob: 3626655175a2e96af5ede5ca064a1bc651f73642 [file] [log] [blame]
Andre Przywarae039ee42014-11-14 15:54:08 +00001#ifndef __ASM_ALTERNATIVE_H
2#define __ASM_ALTERNATIVE_H
3
Catalin Marinas272d01b2016-11-03 18:34:34 +00004#include <asm/cpucaps.h>
Mark Rutland792d4732016-09-07 11:07:08 +01005#include <asm/insn.h>
James Morse57f49592016-02-05 14:58:48 +00006
Marc Zyngier3e75f252018-07-20 10:56:18 +01007#define ARM64_CB_PATCH ARM64_NCAPS
8
Marc Zyngier8d883b22015-06-01 10:47:41 +01009#ifndef __ASSEMBLY__
10
Will Deaconef5e7242015-07-28 19:07:28 +010011#include <linux/init.h>
Andre Przywarae039ee42014-11-14 15:54:08 +000012#include <linux/types.h>
13#include <linux/stddef.h>
14#include <linux/stringify.h>
15
James Morseeea59022018-07-20 10:56:16 +010016extern int alternatives_applied;
17
Andre Przywarae039ee42014-11-14 15:54:08 +000018struct alt_instr {
19 s32 orig_offset; /* offset to original instruction */
20 s32 alt_offset; /* offset to replacement instruction */
21 u16 cpufeature; /* cpufeature bit set for replacement */
22 u8 orig_len; /* size of original instruction(s) */
23 u8 alt_len; /* size of new instruction(s), <= orig_len */
24};
25
Marc Zyngier3e75f252018-07-20 10:56:18 +010026typedef void (*alternative_cb_t)(struct alt_instr *alt,
27 __le32 *origptr, __le32 *updptr, int nr_inst);
28
Will Deaconef5e7242015-07-28 19:07:28 +010029void __init apply_alternatives_all(void);
Andre Przywara932ded42014-11-28 13:40:45 +000030void apply_alternatives(void *start, size_t length);
Andre Przywarae039ee42014-11-14 15:54:08 +000031
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070032#define ALTINSTR_ENTRY(feature) \
Andre Przywarae039ee42014-11-14 15:54:08 +000033 " .word 661b - .\n" /* label */ \
34 " .word 663f - .\n" /* new instruction */ \
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070035 " .hword " __stringify(feature) "\n" /* feature bit */ \
36 " .byte 662b-661b\n" /* source len */ \
37 " .byte 664f-663f\n" /* replacement len */
38
39#define ALTINSTR_ENTRY_CB(feature, cb) \
40 " .word 661b - .\n" /* label */ \
Marc Zyngier3e75f252018-07-20 10:56:18 +010041 " .word " __stringify(cb) "- .\n" /* callback */ \
Andre Przywarae039ee42014-11-14 15:54:08 +000042 " .hword " __stringify(feature) "\n" /* feature bit */ \
43 " .byte 662b-661b\n" /* source len */ \
44 " .byte 664f-663f\n" /* replacement len */
45
Marc Zyngiereb7c11e2015-06-01 10:47:42 +010046/*
47 * alternative assembly primitive:
48 *
49 * If any of these .org directive fail, it means that insn1 and insn2
50 * don't have the same length. This used to be written as
51 *
52 * .if ((664b-663b) != (662b-661b))
53 * .error "Alternatives instruction length mismatch"
54 * .endif
55 *
56 * but most assemblers die if insn1 or insn2 have a .inst. This should
57 * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
58 * containing commit 4e4d08cf7399b606 or c1baaddf8861).
Marc Zyngier3e75f252018-07-20 10:56:18 +010059 *
60 * Alternatives with callbacks do not generate replacement instructions.
Marc Zyngiereb7c11e2015-06-01 10:47:42 +010061 */
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070062#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
James Morse91a5cef2015-07-21 13:23:30 +010063 ".if "__stringify(cfg_enabled)" == 1\n" \
Andre Przywarae039ee42014-11-14 15:54:08 +000064 "661:\n\t" \
65 oldinstr "\n" \
66 "662:\n" \
67 ".pushsection .altinstructions,\"a\"\n" \
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070068 ALTINSTR_ENTRY(feature) \
Andre Przywarae039ee42014-11-14 15:54:08 +000069 ".popsection\n" \
70 ".pushsection .altinstr_replacement, \"a\"\n" \
71 "663:\n\t" \
72 newinstr "\n" \
73 "664:\n\t" \
74 ".popsection\n\t" \
Marc Zyngiereb7c11e2015-06-01 10:47:42 +010075 ".org . - (664b-663b) + (662b-661b)\n\t" \
James Morse91a5cef2015-07-21 13:23:30 +010076 ".org . - (662b-661b) + (664b-663b)\n" \
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070077 ".endif\n"
78
79#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \
80 ".if "__stringify(cfg_enabled)" == 1\n" \
81 "661:\n\t" \
82 oldinstr "\n" \
83 "662:\n" \
84 ".pushsection .altinstructions,\"a\"\n" \
85 ALTINSTR_ENTRY_CB(feature, cb) \
86 ".popsection\n" \
Marc Zyngier3e75f252018-07-20 10:56:18 +010087 "663:\n\t" \
88 "664:\n\t" \
James Morse91a5cef2015-07-21 13:23:30 +010089 ".endif\n"
90
91#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070092 __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
Andre Przywarae039ee42014-11-14 15:54:08 +000093
Marc Zyngier3e75f252018-07-20 10:56:18 +010094#define ALTERNATIVE_CB(oldinstr, cb) \
Sami Tolvanen3d0850a2019-10-31 12:46:52 -070095 __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
Marc Zyngier8d883b22015-06-01 10:47:41 +010096#else
97
James Morse57f49592016-02-05 14:58:48 +000098#include <asm/assembler.h>
99
Marc Zyngier8d883b22015-06-01 10:47:41 +0100100.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
101 .word \orig_offset - .
102 .word \alt_offset - .
103 .hword \feature
104 .byte \orig_len
105 .byte \alt_len
106.endm
107
James Morse91a5cef2015-07-21 13:23:30 +0100108.macro alternative_insn insn1, insn2, cap, enable = 1
109 .if \enable
Marc Zyngier8d883b22015-06-01 10:47:41 +0100110661: \insn1
111662: .pushsection .altinstructions, "a"
112 altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
113 .popsection
114 .pushsection .altinstr_replacement, "ax"
115663: \insn2
116664: .popsection
Marc Zyngiereb7c11e2015-06-01 10:47:42 +0100117 .org . - (664b-663b) + (662b-661b)
118 .org . - (662b-661b) + (664b-663b)
James Morse91a5cef2015-07-21 13:23:30 +0100119 .endif
Marc Zyngier8d883b22015-06-01 10:47:41 +0100120.endm
121
Daniel Thompson63e40812015-07-22 12:21:01 +0100122/*
Mark Rutland792d4732016-09-07 11:07:08 +0100123 * Alternative sequences
Daniel Thompson63e40812015-07-22 12:21:01 +0100124 *
Mark Rutland792d4732016-09-07 11:07:08 +0100125 * The code for the case where the capability is not present will be
126 * assembled and linked as normal. There are no restrictions on this
127 * code.
Daniel Thompson63e40812015-07-22 12:21:01 +0100128 *
Mark Rutland792d4732016-09-07 11:07:08 +0100129 * The code for the case where the capability is present will be
130 * assembled into a special section to be used for dynamic patching.
131 * Code for that case must:
Daniel Thompson63e40812015-07-22 12:21:01 +0100132 *
133 * 1. Be exactly the same length (in bytes) as the default code
134 * sequence.
135 *
136 * 2. Not contain a branch target that is used outside of the
137 * alternative sequence it is defined in (branches into an
138 * alternative sequence are not fixed up).
139 */
Mark Rutland792d4732016-09-07 11:07:08 +0100140
141/*
142 * Begin an alternative code sequence.
143 */
144.macro alternative_if_not cap
145 .set .Lasm_alt_mode, 0
146 .pushsection .altinstructions, "a"
147 altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
148 .popsection
149661:
150.endm
151
152.macro alternative_if cap
153 .set .Lasm_alt_mode, 1
154 .pushsection .altinstructions, "a"
155 altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
156 .popsection
157 .pushsection .altinstr_replacement, "ax"
158 .align 2 /* So GAS knows label 661 is suitably aligned */
159661:
160.endm
161
Marc Zyngier3e75f252018-07-20 10:56:18 +0100162.macro alternative_cb cb
163 .set .Lasm_alt_mode, 0
164 .pushsection .altinstructions, "a"
165 altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
166 .popsection
167661:
168.endm
169
Mark Rutland792d4732016-09-07 11:07:08 +0100170/*
171 * Provide the other half of the alternative code sequence.
172 */
Andre Przywarab82bfa42016-06-28 18:07:27 +0100173.macro alternative_else
Mark Rutland792d4732016-09-07 11:07:08 +0100174662:
175 .if .Lasm_alt_mode==0
176 .pushsection .altinstr_replacement, "ax"
177 .else
178 .popsection
179 .endif
Daniel Thompson63e40812015-07-22 12:21:01 +0100180663:
181.endm
182
183/*
184 * Complete an alternative code sequence.
185 */
Andre Przywarab82bfa42016-06-28 18:07:27 +0100186.macro alternative_endif
Mark Rutland792d4732016-09-07 11:07:08 +0100187664:
188 .if .Lasm_alt_mode==0
189 .popsection
190 .endif
Daniel Thompson63e40812015-07-22 12:21:01 +0100191 .org . - (664b-663b) + (662b-661b)
192 .org . - (662b-661b) + (664b-663b)
193.endm
194
Mark Rutland792d4732016-09-07 11:07:08 +0100195/*
Marc Zyngier3e75f252018-07-20 10:56:18 +0100196 * Callback-based alternative epilogue
197 */
198.macro alternative_cb_end
199662:
200.endm
201
202/*
Mark Rutland792d4732016-09-07 11:07:08 +0100203 * Provides a trivial alternative or default sequence consisting solely
204 * of NOPs. The number of NOPs is chosen automatically to match the
205 * previous case.
206 */
207.macro alternative_else_nop_endif
208alternative_else
209 nops (662b-661b) / AARCH64_INSN_SIZE
210alternative_endif
211.endm
212
James Morse91a5cef2015-07-21 13:23:30 +0100213#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
214 alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
215
Andre Przywara290622e2016-06-28 18:07:28 +0100216.macro user_alt, label, oldinstr, newinstr, cond
2179999: alternative_insn "\oldinstr", "\newinstr", \cond
218 _ASM_EXTABLE 9999b, \label
219.endm
James Morse91a5cef2015-07-21 13:23:30 +0100220
James Morse57f49592016-02-05 14:58:48 +0000221/*
222 * Generate the assembly for UAO alternatives with exception table entries.
223 * This is complicated as there is no post-increment or pair versions of the
224 * unprivileged instructions, and USER() only works for single instructions.
225 */
226#ifdef CONFIG_ARM64_UAO
227 .macro uao_ldp l, reg1, reg2, addr, post_inc
228 alternative_if_not ARM64_HAS_UAO
2298888: ldp \reg1, \reg2, [\addr], \post_inc;
2308889: nop;
231 nop;
232 alternative_else
233 ldtr \reg1, [\addr];
234 ldtr \reg2, [\addr, #8];
235 add \addr, \addr, \post_inc;
236 alternative_endif
237
Ard Biesheuvel6c94f272016-01-01 15:02:12 +0100238 _asm_extable 8888b,\l;
239 _asm_extable 8889b,\l;
James Morse57f49592016-02-05 14:58:48 +0000240 .endm
241
242 .macro uao_stp l, reg1, reg2, addr, post_inc
243 alternative_if_not ARM64_HAS_UAO
2448888: stp \reg1, \reg2, [\addr], \post_inc;
2458889: nop;
246 nop;
247 alternative_else
248 sttr \reg1, [\addr];
249 sttr \reg2, [\addr, #8];
250 add \addr, \addr, \post_inc;
251 alternative_endif
252
Ard Biesheuvel6c94f272016-01-01 15:02:12 +0100253 _asm_extable 8888b,\l;
254 _asm_extable 8889b,\l;
James Morse57f49592016-02-05 14:58:48 +0000255 .endm
256
257 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
258 alternative_if_not ARM64_HAS_UAO
2598888: \inst \reg, [\addr], \post_inc;
260 nop;
261 alternative_else
262 \alt_inst \reg, [\addr];
263 add \addr, \addr, \post_inc;
264 alternative_endif
265
Ard Biesheuvel6c94f272016-01-01 15:02:12 +0100266 _asm_extable 8888b,\l;
James Morse57f49592016-02-05 14:58:48 +0000267 .endm
268#else
269 .macro uao_ldp l, reg1, reg2, addr, post_inc
270 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
271 .endm
272 .macro uao_stp l, reg1, reg2, addr, post_inc
273 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
274 .endm
275 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
276 USER(\l, \inst \reg, [\addr], \post_inc)
277 .endm
278#endif
279
Marc Zyngier8d883b22015-06-01 10:47:41 +0100280#endif /* __ASSEMBLY__ */
281
James Morse91a5cef2015-07-21 13:23:30 +0100282/*
283 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
284 *
285 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
286 * N.B. If CONFIG_FOO is specified, but not selected, the whole block
287 * will be omitted, including oldinstr.
288 */
289#define ALTERNATIVE(oldinstr, newinstr, ...) \
290 _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
291
Andre Przywarae039ee42014-11-14 15:54:08 +0000292#endif /* __ASM_ALTERNATIVE_H */