David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | |
Borislav Petkov | 7a32fc5 | 2018-01-26 13:11:37 +0100 | [diff] [blame] | 3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
| 4 | #define _ASM_X86_NOSPEC_BRANCH_H_ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 5 | |
| 6 | #include <asm/alternative.h> |
| 7 | #include <asm/alternative-asm.h> |
| 8 | #include <asm/cpufeatures.h> |
Peter Zijlstra | ea00f30 | 2018-02-13 14:28:19 +0100 | [diff] [blame] | 9 | #include <asm/msr-index.h> |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 10 | |
David Woodhouse | d1c9910 | 2018-02-19 10:50:56 +0000 | [diff] [blame] | 11 | /* |
| 12 | * Fill the CPU return stack buffer. |
| 13 | * |
| 14 | * Each entry in the RSB, if used for a speculative 'ret', contains an |
| 15 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. |
| 16 | * |
| 17 | * This is required in various cases for retpoline and IBRS-based |
| 18 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to |
| 19 | * eliminate potentially bogus entries from the RSB, and sometimes |
| 20 | * purely to ensure that it doesn't get empty, which on some CPUs would |
| 21 | * allow predictions from other (unwanted!) sources to be used. |
| 22 | * |
| 23 | * We define a CPP macro such that it can be used from both .S files and |
| 24 | * inline assembly. It's possible to do a .macro and then include that |
| 25 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. |
| 26 | */ |
| 27 | |
| 28 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ |
| 29 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ |
| 30 | |
| 31 | /* |
| 32 | * Google experimented with loop-unrolling and this turned out to be |
| 33 | * the optimal version — two calls, each with their own speculation |
| 34 | * trap should their return address end up getting used, in a loop. |
| 35 | */ |
| 36 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ |
| 37 | mov $(nr/2), reg; \ |
| 38 | 771: \ |
| 39 | call 772f; \ |
| 40 | 773: /* speculation trap */ \ |
| 41 | pause; \ |
| 42 | lfence; \ |
| 43 | jmp 773b; \ |
| 44 | 772: \ |
| 45 | call 774f; \ |
| 46 | 775: /* speculation trap */ \ |
| 47 | pause; \ |
| 48 | lfence; \ |
| 49 | jmp 775b; \ |
| 50 | 774: \ |
| 51 | dec reg; \ |
| 52 | jnz 771b; \ |
| 53 | add $(BITS_PER_LONG/8) * nr, sp; |
| 54 | |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 55 | #ifdef __ASSEMBLY__ |
| 56 | |
| 57 | /* |
| 58 | * This should be used immediately before a retpoline alternative. It tells |
| 59 | * objtool where the retpolines are so that it can make sense of the control |
| 60 | * flow by just reading the original instruction(s) and ignoring the |
| 61 | * alternatives. |
| 62 | */ |
| 63 | .macro ANNOTATE_NOSPEC_ALTERNATIVE |
| 64 | .Lannotate_\@: |
| 65 | .pushsection .discard.nospec |
| 66 | .long .Lannotate_\@ - . |
| 67 | .popsection |
| 68 | .endm |
| 69 | |
| 70 | /* |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 71 | * This should be used immediately before an indirect jump/call. It tells |
| 72 | * objtool the subsequent indirect jump/call is vouched safe for retpoline |
| 73 | * builds. |
| 74 | */ |
| 75 | .macro ANNOTATE_RETPOLINE_SAFE |
| 76 | .Lannotate_\@: |
| 77 | .pushsection .discard.retpoline_safe |
| 78 | _ASM_PTR .Lannotate_\@ |
| 79 | .popsection |
| 80 | .endm |
| 81 | |
| 82 | /* |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 83 | * These are the bare retpoline primitives for indirect jmp and call. |
| 84 | * Do not use these directly; they only exist to make the ALTERNATIVE |
| 85 | * invocation below less ugly. |
| 86 | */ |
| 87 | .macro RETPOLINE_JMP reg:req |
| 88 | call .Ldo_rop_\@ |
| 89 | .Lspec_trap_\@: |
| 90 | pause |
Tom Lendacky | 28d437d | 2018-01-13 17:27:30 -0600 | [diff] [blame] | 91 | lfence |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 92 | jmp .Lspec_trap_\@ |
| 93 | .Ldo_rop_\@: |
| 94 | mov \reg, (%_ASM_SP) |
| 95 | ret |
| 96 | .endm |
| 97 | |
| 98 | /* |
| 99 | * This is a wrapper around RETPOLINE_JMP so the called function in reg |
| 100 | * returns to the instruction after the macro. |
| 101 | */ |
| 102 | .macro RETPOLINE_CALL reg:req |
| 103 | jmp .Ldo_call_\@ |
| 104 | .Ldo_retpoline_jmp_\@: |
| 105 | RETPOLINE_JMP \reg |
| 106 | .Ldo_call_\@: |
| 107 | call .Ldo_retpoline_jmp_\@ |
| 108 | .endm |
| 109 | |
| 110 | /* |
| 111 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple |
| 112 | * indirect jmp/call which may be susceptible to the Spectre variant 2 |
| 113 | * attack. |
| 114 | */ |
| 115 | .macro JMP_NOSPEC reg:req |
| 116 | #ifdef CONFIG_RETPOLINE |
| 117 | ANNOTATE_NOSPEC_ALTERNATIVE |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 118 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 119 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 120 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 121 | #else |
| 122 | jmp *\reg |
| 123 | #endif |
| 124 | .endm |
| 125 | |
| 126 | .macro CALL_NOSPEC reg:req |
| 127 | #ifdef CONFIG_RETPOLINE |
| 128 | ANNOTATE_NOSPEC_ALTERNATIVE |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 129 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 130 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 131 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 132 | #else |
| 133 | call *\reg |
| 134 | #endif |
| 135 | .endm |
| 136 | |
David Woodhouse | d1c9910 | 2018-02-19 10:50:56 +0000 | [diff] [blame] | 137 | /* |
| 138 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP |
| 139 | * monstrosity above, manually. |
| 140 | */ |
| 141 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 142 | #ifdef CONFIG_RETPOLINE |
David Woodhouse | d1c9910 | 2018-02-19 10:50:56 +0000 | [diff] [blame] | 143 | ANNOTATE_NOSPEC_ALTERNATIVE |
| 144 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ |
| 145 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ |
| 146 | \ftr |
| 147 | .Lskip_rsb_\@: |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 148 | #endif |
| 149 | .endm |
| 150 | |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 151 | #else /* __ASSEMBLY__ */ |
| 152 | |
| 153 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ |
| 154 | "999:\n\t" \ |
| 155 | ".pushsection .discard.nospec\n\t" \ |
| 156 | ".long 999b - .\n\t" \ |
| 157 | ".popsection\n\t" |
| 158 | |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 159 | #define ANNOTATE_RETPOLINE_SAFE \ |
| 160 | "999:\n\t" \ |
| 161 | ".pushsection .discard.retpoline_safe\n\t" \ |
| 162 | _ASM_PTR " 999b\n\t" \ |
| 163 | ".popsection\n\t" |
| 164 | |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 165 | #if defined(CONFIG_X86_64) && defined(RETPOLINE) |
| 166 | |
| 167 | /* |
| 168 | * Since the inline asm uses the %V modifier which is only in newer GCC, |
| 169 | * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. |
| 170 | */ |
| 171 | # define CALL_NOSPEC \ |
| 172 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
| 173 | ALTERNATIVE( \ |
Peter Zijlstra | 9e0e3c5 | 2018-01-17 22:34:34 +0100 | [diff] [blame] | 174 | ANNOTATE_RETPOLINE_SAFE \ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 175 | "call *%[thunk_target]\n", \ |
| 176 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ |
| 177 | X86_FEATURE_RETPOLINE) |
| 178 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
| 179 | |
| 180 | #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) |
| 181 | /* |
| 182 | * For i386 we use the original ret-equivalent retpoline, because |
| 183 | * otherwise we'll run out of registers. We don't care about CET |
| 184 | * here, anyway. |
| 185 | */ |
Andy Whitcroft | a14bff1 | 2018-03-14 11:24:27 +0000 | [diff] [blame] | 186 | # define CALL_NOSPEC \ |
| 187 | ALTERNATIVE( \ |
| 188 | ANNOTATE_RETPOLINE_SAFE \ |
| 189 | "call *%[thunk_target]\n", \ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 190 | " jmp 904f;\n" \ |
| 191 | " .align 16\n" \ |
| 192 | "901: call 903f;\n" \ |
| 193 | "902: pause;\n" \ |
Tom Lendacky | 28d437d | 2018-01-13 17:27:30 -0600 | [diff] [blame] | 194 | " lfence;\n" \ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 195 | " jmp 902b;\n" \ |
| 196 | " .align 16\n" \ |
| 197 | "903: addl $4, %%esp;\n" \ |
| 198 | " pushl %[thunk_target];\n" \ |
| 199 | " ret;\n" \ |
| 200 | " .align 16\n" \ |
| 201 | "904: call 901b;\n", \ |
| 202 | X86_FEATURE_RETPOLINE) |
| 203 | |
| 204 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 205 | #else /* No retpoline for C / inline asm */ |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 206 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
| 207 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
| 208 | #endif |
| 209 | |
David Woodhouse | da28512 | 2018-01-11 21:46:26 +0000 | [diff] [blame] | 210 | /* The Spectre V2 mitigation variants */ |
| 211 | enum spectre_v2_mitigation { |
| 212 | SPECTRE_V2_NONE, |
| 213 | SPECTRE_V2_RETPOLINE_MINIMAL, |
| 214 | SPECTRE_V2_RETPOLINE_MINIMAL_AMD, |
| 215 | SPECTRE_V2_RETPOLINE_GENERIC, |
| 216 | SPECTRE_V2_RETPOLINE_AMD, |
| 217 | SPECTRE_V2_IBRS, |
| 218 | }; |
| 219 | |
Konrad Rzeszutek Wilk | 1b86883 | 2018-04-25 22:04:18 -0400 | [diff] [blame] | 220 | /* |
| 221 | * The Intel specification for the SPEC_CTRL MSR requires that we |
| 222 | * preserve any already set reserved bits at boot time (e.g. for |
| 223 | * future additions that this kernel is not currently aware of). |
| 224 | * We then set any additional mitigation bits that we want |
| 225 | * ourselves and always use this as the base for SPEC_CTRL. |
| 226 | * We also use this when handling guest entry/exit as below. |
| 227 | */ |
| 228 | extern void x86_spec_ctrl_set(u64); |
| 229 | extern u64 x86_spec_ctrl_get_default(void); |
| 230 | |
Konrad Rzeszutek Wilk | 24f7fc8 | 2018-04-25 22:04:21 -0400 | [diff] [blame] | 231 | /* The Speculative Store Bypass disable variants */ |
| 232 | enum ssb_mitigation { |
| 233 | SPEC_STORE_BYPASS_NONE, |
| 234 | SPEC_STORE_BYPASS_DISABLE, |
Thomas Gleixner | a73ec77 | 2018-04-29 15:26:40 +0200 | [diff] [blame] | 235 | SPEC_STORE_BYPASS_PRCTL, |
Kees Cook | f21b53b | 2018-05-03 14:37:54 -0700 | [diff] [blame^] | 236 | SPEC_STORE_BYPASS_SECCOMP, |
Konrad Rzeszutek Wilk | 24f7fc8 | 2018-04-25 22:04:21 -0400 | [diff] [blame] | 237 | }; |
| 238 | |
Masami Hiramatsu | 736e80a | 2018-01-19 01:14:21 +0900 | [diff] [blame] | 239 | extern char __indirect_thunk_start[]; |
| 240 | extern char __indirect_thunk_end[]; |
| 241 | |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 242 | /* |
| 243 | * On VMEXIT we must ensure that no RSB predictions learned in the guest |
| 244 | * can be followed in the host, by overwriting the RSB completely. Both |
| 245 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future |
Darren Kenny | af189c9 | 2018-02-02 19:12:20 +0000 | [diff] [blame] | 246 | * CPUs with IBRS_ALL *might* it be avoided. |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 247 | */ |
| 248 | static inline void vmexit_fill_RSB(void) |
| 249 | { |
| 250 | #ifdef CONFIG_RETPOLINE |
David Woodhouse | d1c9910 | 2018-02-19 10:50:56 +0000 | [diff] [blame] | 251 | unsigned long loops; |
| 252 | |
| 253 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE |
| 254 | ALTERNATIVE("jmp 910f", |
| 255 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), |
| 256 | X86_FEATURE_RETPOLINE) |
| 257 | "910:" |
| 258 | : "=r" (loops), ASM_CALL_CONSTRAINT |
| 259 | : : "memory" ); |
David Woodhouse | 117cc7a | 2018-01-12 11:11:27 +0000 | [diff] [blame] | 260 | #endif |
| 261 | } |
Andi Kleen | 3f7d875 | 2018-01-17 14:53:28 -0800 | [diff] [blame] | 262 | |
Linus Torvalds | 1aa7a573 | 2018-05-01 15:55:51 +0200 | [diff] [blame] | 263 | static __always_inline |
| 264 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) |
| 265 | { |
| 266 | asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) |
| 267 | : : "c" (msr), |
| 268 | "a" (val), |
| 269 | "d" (val >> 32), |
| 270 | [feature] "i" (feature) |
| 271 | : "memory"); |
| 272 | } |
David Woodhouse | dd84441 | 2018-02-19 10:50:54 +0000 | [diff] [blame] | 273 | |
David Woodhouse | 20ffa1c | 2018-01-25 16:14:15 +0000 | [diff] [blame] | 274 | static inline void indirect_branch_prediction_barrier(void) |
| 275 | { |
Konrad Rzeszutek Wilk | 1b86883 | 2018-04-25 22:04:18 -0400 | [diff] [blame] | 276 | u64 val = PRED_CMD_IBPB; |
| 277 | |
| 278 | alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); |
David Woodhouse | 20ffa1c | 2018-01-25 16:14:15 +0000 | [diff] [blame] | 279 | } |
| 280 | |
David Woodhouse | dd84441 | 2018-02-19 10:50:54 +0000 | [diff] [blame] | 281 | /* |
| 282 | * With retpoline, we must use IBRS to restrict branch prediction |
| 283 | * before calling into firmware. |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 284 | * |
| 285 | * (Implemented as CPP macros due to header hell.) |
David Woodhouse | dd84441 | 2018-02-19 10:50:54 +0000 | [diff] [blame] | 286 | */ |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 287 | #define firmware_restrict_branch_speculation_start() \ |
| 288 | do { \ |
Konrad Rzeszutek Wilk | 1b86883 | 2018-04-25 22:04:18 -0400 | [diff] [blame] | 289 | u64 val = x86_spec_ctrl_get_default() | SPEC_CTRL_IBRS; \ |
| 290 | \ |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 291 | preempt_disable(); \ |
Konrad Rzeszutek Wilk | 1b86883 | 2018-04-25 22:04:18 -0400 | [diff] [blame] | 292 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 293 | X86_FEATURE_USE_IBRS_FW); \ |
| 294 | } while (0) |
David Woodhouse | dd84441 | 2018-02-19 10:50:54 +0000 | [diff] [blame] | 295 | |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 296 | #define firmware_restrict_branch_speculation_end() \ |
| 297 | do { \ |
Konrad Rzeszutek Wilk | 1b86883 | 2018-04-25 22:04:18 -0400 | [diff] [blame] | 298 | u64 val = x86_spec_ctrl_get_default(); \ |
| 299 | \ |
| 300 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
Ingo Molnar | d72f4e2 | 2018-02-21 09:20:37 +0100 | [diff] [blame] | 301 | X86_FEATURE_USE_IBRS_FW); \ |
| 302 | preempt_enable(); \ |
| 303 | } while (0) |
David Woodhouse | 76b0438 | 2018-01-11 21:46:25 +0000 | [diff] [blame] | 304 | |
| 305 | #endif /* __ASSEMBLY__ */ |
Daniel Borkmann | a493a87 | 2018-02-22 15:12:53 +0100 | [diff] [blame] | 306 | |
| 307 | /* |
| 308 | * Below is used in the eBPF JIT compiler and emits the byte sequence |
| 309 | * for the following assembly: |
| 310 | * |
| 311 | * With retpolines configured: |
| 312 | * |
| 313 | * callq do_rop |
| 314 | * spec_trap: |
| 315 | * pause |
| 316 | * lfence |
| 317 | * jmp spec_trap |
| 318 | * do_rop: |
| 319 | * mov %rax,(%rsp) |
| 320 | * retq |
| 321 | * |
| 322 | * Without retpolines configured: |
| 323 | * |
| 324 | * jmp *%rax |
| 325 | */ |
| 326 | #ifdef CONFIG_RETPOLINE |
| 327 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 |
| 328 | # define RETPOLINE_RAX_BPF_JIT() \ |
| 329 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ |
| 330 | /* spec_trap: */ \ |
| 331 | EMIT2(0xF3, 0x90); /* pause */ \ |
| 332 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ |
| 333 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ |
| 334 | /* do_rop: */ \ |
| 335 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ |
| 336 | EMIT1(0xC3); /* retq */ |
| 337 | #else |
| 338 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 |
| 339 | # define RETPOLINE_RAX_BPF_JIT() \ |
| 340 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ |
| 341 | #endif |
| 342 | |
Borislav Petkov | 7a32fc5 | 2018-01-26 13:11:37 +0100 | [diff] [blame] | 343 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |