blob: 402e34a21559e3a6227a6e76b57029c8db74236d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Ingo Molnar54ad7262015-06-05 13:02:28 +02002 * Compatibility mode system call entry point for x86-64.
3 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
Ingo Molnar54ad7262015-06-05 13:02:28 +02005 */
Ingo Molnard36f9472015-06-03 18:29:26 +02006#include "calling.h"
Sam Ravnborge2d5df92005-09-09 21:28:48 +02007#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <asm/current.h>
9#include <asm/errno.h>
Ingo Molnar54ad7262015-06-05 13:02:28 +020010#include <asm/ia32_unistd.h>
11#include <asm/thread_info.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <asm/segment.h>
Ingo Molnar2601e642006-07-03 00:24:45 -070013#include <asm/irqflags.h>
H. Peter Anvin1ce6f862012-04-20 12:19:50 -070014#include <asm/asm.h>
H. Peter Anvin63bcff22012-09-21 12:43:12 -070015#include <asm/smap.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/linkage.h>
Eric Parisd7e75282012-01-03 14:23:06 -050017#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
Jiri Olsaea714542011-03-07 19:10:39 +010019 .section .entry.text, "ax"
20
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040021#ifdef CONFIG_PARAVIRT
22ENTRY(native_usergs_sysret32)
23 swapgs
24 sysretl
25ENDPROC(native_usergs_sysret32)
Jeremy Fitzhardinge2be29982008-06-25 00:19:28 -040026#endif
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028/*
Ingo Molnar54ad7262015-06-05 13:02:28 +020029 * 32-bit SYSENTER instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -080031 * SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
32 * IF and VM in rflags are cleared (IOW: interrupts are off).
33 * SYSENTER does not save anything on the stack,
34 * and does not save old rip (!!!) and rflags.
35 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -080037 * eax system call number
38 * ebx arg1
39 * ecx arg2
40 * edx arg3
41 * esi arg4
42 * edi arg5
43 * ebp user stack
44 * 0(%ebp) arg6
45 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * This is purely a fast path. For anything complicated we use the int 0x80
Denys Vlasenkob87cf632015-02-26 14:40:32 -080047 * path below. We set up a complete hardware stack frame to share code
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * with the int 0x80 path.
Denys Vlasenkob87cf632015-02-26 14:40:32 -080049 */
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +020050ENTRY(entry_SYSENTER_compat)
Andy Lutomirskib611acf2015-10-05 17:47:55 -070051 /* Interrupts are off on entry. */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -070052 SWAPGS_UNSAFE_STACK
Denys Vlasenko3a232082015-04-24 17:31:35 +020053 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +010054
Andy Lutomirskia474e672015-10-05 17:48:11 -070055 /*
56 * User tracing code (ptrace or signal handlers) might assume that
57 * the saved RAX contains a 32-bit number when we're invoking a 32-bit
58 * syscall. Just in case the high bits are nonzero, zero-extend
59 * the syscall number. (This could almost certainly be deleted
60 * with no ill effects.)
61 */
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +010062 movl %eax, %eax
63
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020064 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +020065 pushq $__USER32_DS /* pt_regs->ss */
Andy Lutomirskia474e672015-10-05 17:48:11 -070066 pushq %rcx /* pt_regs->sp */
Andy Lutomirskib611acf2015-10-05 17:47:55 -070067
68 /*
69 * Push flags. This is nasty. First, interrupts are currently
70 * off, but we need pt_regs->flags to have IF set. Second, even
71 * if TF was set when SYSENTER started, it's clear by now. We fix
72 * that later using TIF_SINGLESTEP.
73 */
74 pushfq /* pt_regs->flags (except IF = 0) */
75 orl $X86_EFLAGS_IF, (%rsp) /* Fix saved flags */
Andy Lutomirskia474e672015-10-05 17:48:11 -070076 ASM_CLAC /* Clear AC after saving FLAGS */
Andy Lutomirskib611acf2015-10-05 17:47:55 -070077
Ingo Molnar131484c2015-05-28 12:21:47 +020078 pushq $__USER32_CS /* pt_regs->cs */
Andy Lutomirskia474e672015-10-05 17:48:11 -070079 xorq %r8,%r8
80 pushq %r8 /* pt_regs->ip = 0 (placeholder) */
Ingo Molnar131484c2015-05-28 12:21:47 +020081 pushq %rax /* pt_regs->orig_ax */
82 pushq %rdi /* pt_regs->di */
83 pushq %rsi /* pt_regs->si */
84 pushq %rdx /* pt_regs->dx */
Andy Lutomirskia474e672015-10-05 17:48:11 -070085 pushq %rcx /* pt_regs->cx (will be overwritten) */
Ingo Molnar131484c2015-05-28 12:21:47 +020086 pushq $-ENOSYS /* pt_regs->ax */
Andy Lutomirskia474e672015-10-05 17:48:11 -070087 pushq %r8 /* pt_regs->r8 = 0 */
88 pushq %r8 /* pt_regs->r9 = 0 */
89 pushq %r8 /* pt_regs->r10 = 0 */
90 pushq %r8 /* pt_regs->r11 = 0 */
91 pushq %rbx /* pt_regs->rbx */
92 pushq %rbp /* pt_regs->rbp */
93 pushq %r8 /* pt_regs->r12 = 0 */
94 pushq %r8 /* pt_regs->r13 = 0 */
95 pushq %r8 /* pt_regs->r14 = 0 */
96 pushq %r8 /* pt_regs->r15 = 0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 cld
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +020098
Denys Vlasenkob87cf632015-02-26 14:40:32 -080099 /*
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700100 * Sysenter doesn't filter flags, so we need to clear NT
101 * ourselves. To save a few cycles, we can check whether
102 * NT was set instead of doing an unconditional popfq.
Andy Lutomirskib611acf2015-10-05 17:47:55 -0700103 * This needs to happen before enabling interrupts so that
104 * we don't get preempted with NT set.
Borislav Petkov374a3a32015-10-09 19:08:59 +0200105 *
106 * NB.: sysenter_fix_flags is a label with the code under it moved
107 * out-of-line as an optimization: NT is unlikely to be set in the
108 * majority of the cases and instead of polluting the I$ unnecessarily,
109 * we're keeping that code behind a branch which will predict as
110 * not-taken and therefore its instructions won't be fetched.
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700111 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200112 testl $X86_EFLAGS_NT, EFLAGS(%rsp)
113 jnz sysenter_fix_flags
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700114sysenter_flags_fixed:
115
Andy Lutomirskia474e672015-10-05 17:48:11 -0700116 /*
117 * User mode is traced as though IRQs are on, and SYSENTER
118 * turned them off.
119 */
120 TRACE_IRQS_OFF
Andy Lutomirskie62a2542015-10-05 17:48:02 -0700121
Andy Lutomirskia474e672015-10-05 17:48:11 -0700122 movq %rsp, %rdi
123 call do_fast_syscall_32
Boris Ostrovsky91e2eea2015-11-19 16:55:45 -0500124 /* XEN PV guests always use IRET path */
125 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
126 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
Andy Lutomirski7841b402015-10-05 17:48:12 -0700127 jmp sysret32_from_system_call
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Andy Lutomirski8c7aa692014-10-01 11:49:04 -0700129sysenter_fix_flags:
Andy Lutomirskib611acf2015-10-05 17:47:55 -0700130 pushq $X86_EFLAGS_FIXED
Ingo Molnar131484c2015-05-28 12:21:47 +0200131 popfq
Ingo Molnar54ad7262015-06-05 13:02:28 +0200132 jmp sysenter_flags_fixed
Ingo Molnar4c8cd0c2015-06-08 08:33:56 +0200133ENDPROC(entry_SYSENTER_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
135/*
Ingo Molnar54ad7262015-06-05 13:02:28 +0200136 * 32-bit SYSCALL instruction entry.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 *
Ingo Molnar54ad7262015-06-05 13:02:28 +0200138 * 32-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800139 * then loads new ss, cs, and rip from previously programmed MSRs.
140 * rflags gets masked by a value from another MSR (so CLD and CLAC
141 * are not needed). SYSCALL does not save anything on the stack
142 * and does not change rsp.
143 *
144 * Note: rflags saving+masking-with-MSR happens only in Long mode
Ingo Molnar54ad7262015-06-05 13:02:28 +0200145 * (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it).
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800146 * Don't get confused: rflags saving+masking depends on Long Mode Active bit
147 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
148 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
149 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 * Arguments:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800151 * eax system call number
152 * ecx return address
153 * ebx arg1
154 * ebp arg2 (note: not saved in the stack frame, should not be touched)
155 * edx arg3
156 * esi arg4
157 * edi arg5
158 * esp user stack
159 * 0(%esp) arg6
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800160 */
Ingo Molnar2cd23552015-06-08 08:28:07 +0200161ENTRY(entry_SYSCALL_compat)
Andy Lutomirskia474e672015-10-05 17:48:11 -0700162 /* Interrupts are off on entry. */
Jeremy Fitzhardinge457da702008-06-26 07:28:51 -0700163 SWAPGS_UNSAFE_STACK
Andy Lutomirskie62a2542015-10-05 17:48:02 -0700164
Andy Lutomirskia474e672015-10-05 17:48:11 -0700165 /* Stash user ESP and switch to the kernel stack. */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200166 movl %esp, %r8d
167 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100168
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100169 /* Zero-extending 32-bit regs, do not remove */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200170 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100171
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200172 /* Construct struct pt_regs on stack */
Ingo Molnar131484c2015-05-28 12:21:47 +0200173 pushq $__USER32_DS /* pt_regs->ss */
174 pushq %r8 /* pt_regs->sp */
175 pushq %r11 /* pt_regs->flags */
176 pushq $__USER32_CS /* pt_regs->cs */
177 pushq %rcx /* pt_regs->ip */
178 pushq %rax /* pt_regs->orig_ax */
179 pushq %rdi /* pt_regs->di */
180 pushq %rsi /* pt_regs->si */
181 pushq %rdx /* pt_regs->dx */
Andy Lutomirskia474e672015-10-05 17:48:11 -0700182 pushq %rcx /* pt_regs->cx (will be overwritten) */
Ingo Molnar131484c2015-05-28 12:21:47 +0200183 pushq $-ENOSYS /* pt_regs->ax */
Andy Lutomirskia474e672015-10-05 17:48:11 -0700184 xorq %r8,%r8
185 pushq %r8 /* pt_regs->r8 = 0 */
186 pushq %r8 /* pt_regs->r9 = 0 */
187 pushq %r8 /* pt_regs->r10 = 0 */
188 pushq %r8 /* pt_regs->r11 = 0 */
189 pushq %rbx /* pt_regs->rbx */
190 pushq %rbp /* pt_regs->rbp */
191 pushq %r8 /* pt_regs->r12 = 0 */
192 pushq %r8 /* pt_regs->r13 = 0 */
193 pushq %r8 /* pt_regs->r14 = 0 */
194 pushq %r8 /* pt_regs->r15 = 0 */
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200195
Andy Lutomirskia474e672015-10-05 17:48:11 -0700196 /*
197 * User mode is traced as though IRQs are on, and SYSENTER
198 * turned them off.
199 */
200 TRACE_IRQS_OFF
201
202 movq %rsp, %rdi
203 call do_fast_syscall_32
Boris Ostrovsky91e2eea2015-11-19 16:55:45 -0500204 /* XEN PV guests always use IRET path */
205 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
206 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
Andy Lutomirski7841b402015-10-05 17:48:12 -0700207
208 /* Opportunistic SYSRET */
209sysret32_from_system_call:
210 TRACE_IRQS_ON /* User mode traces as IRQs on. */
211 movq RBX(%rsp), %rbx /* pt_regs->rbx */
212 movq RBP(%rsp), %rbp /* pt_regs->rbp */
213 movq EFLAGS(%rsp), %r11 /* pt_regs->flags (in r11) */
214 movq RIP(%rsp), %rcx /* pt_regs->ip (in rcx) */
215 addq $RAX, %rsp /* Skip r8-r15 */
216 popq %rax /* pt_regs->rax */
217 popq %rdx /* Skip pt_regs->cx */
218 popq %rdx /* pt_regs->dx */
219 popq %rsi /* pt_regs->si */
220 popq %rdi /* pt_regs->di */
221
222 /*
223 * USERGS_SYSRET32 does:
224 * GSBASE = user's GS base
225 * EIP = ECX
226 * RFLAGS = R11
227 * CS = __USER32_CS
228 * SS = __USER_DS
229 *
230 * ECX will not match pt_regs->cx, but we're returning to a vDSO
231 * trampoline that will fix up RCX, so this is okay.
232 *
233 * R12-R15 are callee-saved, so they contain whatever was in them
234 * when the system call started, which is already known to user
235 * code. We zero R8-R10 to avoid info leaks.
236 */
237 xorq %r8, %r8
238 xorq %r9, %r9
239 xorq %r10, %r10
240 movq RSP-ORIG_RAX(%rsp), %rsp
241 USERGS_SYSRET32
Ingo Molnar2cd23552015-06-08 08:28:07 +0200242END(entry_SYSCALL_compat)
Ingo Molnar54ad7262015-06-05 13:02:28 +0200243
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800244/*
245 * Emulated IA32 system calls via int 0x80.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 *
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800247 * Arguments:
248 * eax system call number
249 * ebx arg1
250 * ecx arg2
251 * edx arg3
252 * esi arg4
253 * edi arg5
254 * ebp arg6 (note: not saved in the stack frame, should not be touched)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 *
256 * Notes:
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800257 * Uses the same stack frame as the x86-64 version.
258 * All registers except eax must be saved (but ptrace may violate that).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 * Arguments are zero extended. For system calls that want sign extension and
260 * take long arguments a wrapper is needed. Most calls can just be called
261 * directly.
Denys Vlasenkob87cf632015-02-26 14:40:32 -0800262 * Assumes it is only called from user space and entered with interrupts off.
263 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264
Ingo Molnar2cd23552015-06-08 08:28:07 +0200265ENTRY(entry_INT80_compat)
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100266 /*
267 * Interrupts are off on entry.
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100268 */
Jeremy Fitzhardinge360c0442008-07-08 15:06:28 -0700269 PARAVIRT_ADJUST_EXCEPTION_FRAME
Jeremy Fitzhardinge66804152008-06-25 00:19:29 -0400270 SWAPGS
Denys Vlasenkoa232e3d2015-03-27 11:36:20 +0100271
Andy Lutomirskiee08c6b2015-10-05 17:48:09 -0700272 /*
273 * User tracing code (ptrace or signal handlers) might assume that
274 * the saved RAX contains a 32-bit number when we're invoking a 32-bit
275 * syscall. Just in case the high bits are nonzero, zero-extend
276 * the syscall number. (This could almost certainly be deleted
277 * with no ill effects.)
278 */
Ingo Molnar54ad7262015-06-05 13:02:28 +0200279 movl %eax, %eax
Denys Vlasenko4ee8ec12015-03-27 11:36:21 +0100280
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200281 /* Construct struct pt_regs on stack (iret frame is already on stack) */
Ingo Molnar131484c2015-05-28 12:21:47 +0200282 pushq %rax /* pt_regs->orig_ax */
283 pushq %rdi /* pt_regs->di */
284 pushq %rsi /* pt_regs->si */
285 pushq %rdx /* pt_regs->dx */
286 pushq %rcx /* pt_regs->cx */
287 pushq $-ENOSYS /* pt_regs->ax */
Andy Lutomirski8169aff2015-10-05 17:48:05 -0700288 xorq %r8,%r8
289 pushq %r8 /* pt_regs->r8 = 0 */
290 pushq %r8 /* pt_regs->r9 = 0 */
291 pushq %r8 /* pt_regs->r10 = 0 */
292 pushq %r8 /* pt_regs->r11 = 0 */
293 pushq %rbx /* pt_regs->rbx */
294 pushq %rbp /* pt_regs->rbp */
295 pushq %r12 /* pt_regs->r12 */
296 pushq %r13 /* pt_regs->r13 */
297 pushq %r14 /* pt_regs->r14 */
298 pushq %r15 /* pt_regs->r15 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 cld
Denys Vlasenko4c9c0e92015-03-31 19:00:04 +0200300
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200301 /*
Andy Lutomirskiee08c6b2015-10-05 17:48:09 -0700302 * User mode is traced as though IRQs are on, and the interrupt
303 * gate turned them off.
Denys Vlasenko73cbf682015-06-02 21:04:02 +0200304 */
Andy Lutomirskiee08c6b2015-10-05 17:48:09 -0700305 TRACE_IRQS_OFF
306
307 movq %rsp, %rdi
Andy Lutomirski657c1ee2015-10-16 15:42:54 -0700308 call do_syscall_32_irqs_off
Andy Lutomirskia474e672015-10-05 17:48:11 -0700309.Lsyscall_32_done:
Andy Lutomirskiee08c6b2015-10-05 17:48:09 -0700310
311 /* Go back to user mode. */
312 TRACE_IRQS_ON
313 SWAPGS
314 jmp restore_regs_and_iret
Ingo Molnar2cd23552015-06-08 08:28:07 +0200315END(entry_INT80_compat)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
Jan Beulichf6b2bc82011-11-29 11:24:10 +0000317 ALIGN
Al Viro1d4b4b22012-10-22 22:34:11 -0400318GLOBAL(stub32_clone)
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200319 /*
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200320 * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr).
321 * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val).
322 *
323 * The native 64-bit kernel's sys_clone() implements the latter,
324 * so we need to swap arguments here before calling it:
Denys Vlasenko5cdc6832015-06-03 15:58:49 +0200325 */
Denys Vlasenko7a5a9822015-06-03 15:58:50 +0200326 xchg %r8, %rcx
Andy Lutomirski8169aff2015-10-05 17:48:05 -0700327 jmp sys_clone