Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 2 | * Compatibility mode system call entry point for x86-64. |
| 3 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * Copyright 2000-2002 Andi Kleen, SuSE Labs. |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 5 | */ |
Ingo Molnar | d36f947 | 2015-06-03 18:29:26 +0200 | [diff] [blame] | 6 | #include "calling.h" |
Sam Ravnborg | e2d5df9 | 2005-09-09 21:28:48 +0200 | [diff] [blame] | 7 | #include <asm/asm-offsets.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <asm/current.h> |
| 9 | #include <asm/errno.h> |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 10 | #include <asm/ia32_unistd.h> |
| 11 | #include <asm/thread_info.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <asm/segment.h> |
Ingo Molnar | 2601e64 | 2006-07-03 00:24:45 -0700 | [diff] [blame] | 13 | #include <asm/irqflags.h> |
H. Peter Anvin | 1ce6f86 | 2012-04-20 12:19:50 -0700 | [diff] [blame] | 14 | #include <asm/asm.h> |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 15 | #include <asm/smap.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/linkage.h> |
Eric Paris | d7e7528 | 2012-01-03 14:23:06 -0500 | [diff] [blame] | 17 | #include <linux/err.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
Jiri Olsa | ea71454 | 2011-03-07 19:10:39 +0100 | [diff] [blame] | 19 | .section .entry.text, "ax" |
| 20 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | /* |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 22 | * 32-bit SYSENTER entry. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | * |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 24 | * 32-bit system calls through the vDSO's __kernel_vsyscall enter here |
| 25 | * on 64-bit kernels running on Intel CPUs. |
| 26 | * |
| 27 | * The SYSENTER instruction, in principle, should *only* occur in the |
| 28 | * vDSO. In practice, a small number of Android devices were shipped |
| 29 | * with a copy of Bionic that inlined a SYSENTER instruction. This |
| 30 | * never happened in any of Google's Bionic versions -- it only happened |
| 31 | * in a narrow range of Intel-provided versions. |
| 32 | * |
| 33 | * SYSENTER loads SS, RSP, CS, and RIP from previously programmed MSRs. |
| 34 | * IF and VM in RFLAGS are cleared (IOW: interrupts are off). |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 35 | * SYSENTER does not save anything on the stack, |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 36 | * and does not save old RIP (!!!), RSP, or RFLAGS. |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 37 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | * Arguments: |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 39 | * eax system call number |
| 40 | * ebx arg1 |
| 41 | * ecx arg2 |
| 42 | * edx arg3 |
| 43 | * esi arg4 |
| 44 | * edi arg5 |
| 45 | * ebp user stack |
| 46 | * 0(%ebp) arg6 |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 47 | */ |
Ingo Molnar | 4c8cd0c | 2015-06-08 08:33:56 +0200 | [diff] [blame] | 48 | ENTRY(entry_SYSENTER_compat) |
Andy Lutomirski | b611acf | 2015-10-05 17:47:55 -0700 | [diff] [blame] | 49 | /* Interrupts are off on entry. */ |
Jeremy Fitzhardinge | 457da70 | 2008-06-26 07:28:51 -0700 | [diff] [blame] | 50 | SWAPGS_UNSAFE_STACK |
Denys Vlasenko | 3a23208 | 2015-04-24 17:31:35 +0200 | [diff] [blame] | 51 | movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp |
Denys Vlasenko | a232e3d | 2015-03-27 11:36:20 +0100 | [diff] [blame] | 52 | |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 53 | /* |
| 54 | * User tracing code (ptrace or signal handlers) might assume that |
| 55 | * the saved RAX contains a 32-bit number when we're invoking a 32-bit |
| 56 | * syscall. Just in case the high bits are nonzero, zero-extend |
| 57 | * the syscall number. (This could almost certainly be deleted |
| 58 | * with no ill effects.) |
| 59 | */ |
Denys Vlasenko | 4ee8ec1 | 2015-03-27 11:36:21 +0100 | [diff] [blame] | 60 | movl %eax, %eax |
| 61 | |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 62 | /* Construct struct pt_regs on stack */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 63 | pushq $__USER32_DS /* pt_regs->ss */ |
Andy Lutomirski | 30bfa7b | 2015-12-16 23:18:48 -0800 | [diff] [blame] | 64 | pushq %rbp /* pt_regs->sp (stashed in bp) */ |
Andy Lutomirski | b611acf | 2015-10-05 17:47:55 -0700 | [diff] [blame] | 65 | |
| 66 | /* |
| 67 | * Push flags. This is nasty. First, interrupts are currently |
| 68 | * off, but we need pt_regs->flags to have IF set. Second, even |
| 69 | * if TF was set when SYSENTER started, it's clear by now. We fix |
| 70 | * that later using TIF_SINGLESTEP. |
| 71 | */ |
| 72 | pushfq /* pt_regs->flags (except IF = 0) */ |
| 73 | orl $X86_EFLAGS_IF, (%rsp) /* Fix saved flags */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 74 | pushq $__USER32_CS /* pt_regs->cs */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 75 | pushq $0 /* pt_regs->ip = 0 (placeholder) */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 76 | pushq %rax /* pt_regs->orig_ax */ |
| 77 | pushq %rdi /* pt_regs->di */ |
| 78 | pushq %rsi /* pt_regs->si */ |
| 79 | pushq %rdx /* pt_regs->dx */ |
Andy Lutomirski | 30bfa7b | 2015-12-16 23:18:48 -0800 | [diff] [blame] | 80 | pushq %rcx /* pt_regs->cx */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 81 | pushq $-ENOSYS /* pt_regs->ax */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 82 | pushq $0 /* pt_regs->r8 = 0 */ |
| 83 | pushq $0 /* pt_regs->r9 = 0 */ |
| 84 | pushq $0 /* pt_regs->r10 = 0 */ |
| 85 | pushq $0 /* pt_regs->r11 = 0 */ |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 86 | pushq %rbx /* pt_regs->rbx */ |
Andy Lutomirski | 30bfa7b | 2015-12-16 23:18:48 -0800 | [diff] [blame] | 87 | pushq %rbp /* pt_regs->rbp (will be overwritten) */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 88 | pushq $0 /* pt_regs->r12 = 0 */ |
| 89 | pushq $0 /* pt_regs->r13 = 0 */ |
| 90 | pushq $0 /* pt_regs->r14 = 0 */ |
| 91 | pushq $0 /* pt_regs->r15 = 0 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | cld |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 93 | |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 94 | /* |
Andy Lutomirski | e786041 | 2016-03-09 19:00:25 -0800 | [diff] [blame] | 95 | * SYSENTER doesn't filter flags, so we need to clear NT and AC |
Andy Lutomirski | 8c7aa69 | 2014-10-01 11:49:04 -0700 | [diff] [blame] | 96 | * ourselves. To save a few cycles, we can check whether |
Andy Lutomirski | e786041 | 2016-03-09 19:00:25 -0800 | [diff] [blame] | 97 | * either was set instead of doing an unconditional popfq. |
Andy Lutomirski | b611acf | 2015-10-05 17:47:55 -0700 | [diff] [blame] | 98 | * This needs to happen before enabling interrupts so that |
| 99 | * we don't get preempted with NT set. |
Borislav Petkov | 374a3a3 | 2015-10-09 19:08:59 +0200 | [diff] [blame] | 100 | * |
Andy Lutomirski | f2b3757 | 2016-03-09 19:00:30 -0800 | [diff] [blame] | 101 | * If TF is set, we will single-step all the way to here -- do_debug |
| 102 | * will ignore all the traps. (Yes, this is slow, but so is |
| 103 | * single-stepping in general. This allows us to avoid having |
| 104 | * a more complicated code to handle the case where a user program |
| 105 | * forces us to single-step through the SYSENTER entry code.) |
| 106 | * |
Borislav Petkov | f74acf0 | 2015-12-12 11:27:57 +0100 | [diff] [blame] | 107 | * NB.: .Lsysenter_fix_flags is a label with the code under it moved |
Borislav Petkov | 374a3a3 | 2015-10-09 19:08:59 +0200 | [diff] [blame] | 108 | * out-of-line as an optimization: NT is unlikely to be set in the |
| 109 | * majority of the cases and instead of polluting the I$ unnecessarily, |
| 110 | * we're keeping that code behind a branch which will predict as |
| 111 | * not-taken and therefore its instructions won't be fetched. |
Andy Lutomirski | 8c7aa69 | 2014-10-01 11:49:04 -0700 | [diff] [blame] | 112 | */ |
Andy Lutomirski | f2b3757 | 2016-03-09 19:00:30 -0800 | [diff] [blame] | 113 | testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, EFLAGS(%rsp) |
Borislav Petkov | f74acf0 | 2015-12-12 11:27:57 +0100 | [diff] [blame] | 114 | jnz .Lsysenter_fix_flags |
| 115 | .Lsysenter_flags_fixed: |
Andy Lutomirski | 8c7aa69 | 2014-10-01 11:49:04 -0700 | [diff] [blame] | 116 | |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 117 | /* |
| 118 | * User mode is traced as though IRQs are on, and SYSENTER |
| 119 | * turned them off. |
| 120 | */ |
| 121 | TRACE_IRQS_OFF |
Andy Lutomirski | e62a254 | 2015-10-05 17:48:02 -0700 | [diff] [blame] | 122 | |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 123 | movq %rsp, %rdi |
| 124 | call do_fast_syscall_32 |
Boris Ostrovsky | 91e2eea | 2015-11-19 16:55:45 -0500 | [diff] [blame] | 125 | /* XEN PV guests always use IRET path */ |
| 126 | ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ |
| 127 | "jmp .Lsyscall_32_done", X86_FEATURE_XENPV |
Andy Lutomirski | 7841b40 | 2015-10-05 17:48:12 -0700 | [diff] [blame] | 128 | jmp sysret32_from_system_call |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | |
Borislav Petkov | f74acf0 | 2015-12-12 11:27:57 +0100 | [diff] [blame] | 130 | .Lsysenter_fix_flags: |
Andy Lutomirski | b611acf | 2015-10-05 17:47:55 -0700 | [diff] [blame] | 131 | pushq $X86_EFLAGS_FIXED |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 132 | popfq |
Borislav Petkov | f74acf0 | 2015-12-12 11:27:57 +0100 | [diff] [blame] | 133 | jmp .Lsysenter_flags_fixed |
Andy Lutomirski | f2b3757 | 2016-03-09 19:00:30 -0800 | [diff] [blame] | 134 | GLOBAL(__end_entry_SYSENTER_compat) |
Ingo Molnar | 4c8cd0c | 2015-06-08 08:33:56 +0200 | [diff] [blame] | 135 | ENDPROC(entry_SYSENTER_compat) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | |
| 137 | /* |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 138 | * 32-bit SYSCALL entry. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | * |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 140 | * 32-bit system calls through the vDSO's __kernel_vsyscall enter here |
| 141 | * on 64-bit kernels running on AMD CPUs. |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 142 | * |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 143 | * The SYSCALL instruction, in principle, should *only* occur in the |
| 144 | * vDSO. In practice, it appears that this really is the case. |
| 145 | * As evidence: |
| 146 | * |
| 147 | * - The calling convention for SYSCALL has changed several times without |
| 148 | * anyone noticing. |
| 149 | * |
| 150 | * - Prior to the in-kernel X86_BUG_SYSRET_SS_ATTRS fixup, anything |
| 151 | * user task that did SYSCALL without immediately reloading SS |
| 152 | * would randomly crash. |
| 153 | * |
| 154 | * - Most programmers do not directly target AMD CPUs, and the 32-bit |
| 155 | * SYSCALL instruction does not exist on Intel CPUs. Even on AMD |
| 156 | * CPUs, Linux disables the SYSCALL instruction on 32-bit kernels |
| 157 | * because the SYSCALL instruction in legacy/native 32-bit mode (as |
| 158 | * opposed to compat mode) is sufficiently poorly designed as to be |
| 159 | * essentially unusable. |
| 160 | * |
| 161 | * 32-bit SYSCALL saves RIP to RCX, clears RFLAGS.RF, then saves |
| 162 | * RFLAGS to R11, then loads new SS, CS, and RIP from previously |
| 163 | * programmed MSRs. RFLAGS gets masked by a value from another MSR |
| 164 | * (so CLD and CLAC are not needed). SYSCALL does not save anything on |
| 165 | * the stack and does not change RSP. |
| 166 | * |
| 167 | * Note: RFLAGS saving+masking-with-MSR happens only in Long mode |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 168 | * (in legacy 32-bit mode, IF, RF and VM bits are cleared and that's it). |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 169 | * Don't get confused: RFLAGS saving+masking depends on Long Mode Active bit |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 170 | * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes |
| 171 | * or target CS descriptor's L bit (SYSCALL does not read segment descriptors). |
| 172 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | * Arguments: |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 174 | * eax system call number |
| 175 | * ecx return address |
| 176 | * ebx arg1 |
| 177 | * ebp arg2 (note: not saved in the stack frame, should not be touched) |
| 178 | * edx arg3 |
| 179 | * esi arg4 |
| 180 | * edi arg5 |
| 181 | * esp user stack |
| 182 | * 0(%esp) arg6 |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 183 | */ |
Ingo Molnar | 2cd2355 | 2015-06-08 08:28:07 +0200 | [diff] [blame] | 184 | ENTRY(entry_SYSCALL_compat) |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 185 | /* Interrupts are off on entry. */ |
Jeremy Fitzhardinge | 457da70 | 2008-06-26 07:28:51 -0700 | [diff] [blame] | 186 | SWAPGS_UNSAFE_STACK |
Andy Lutomirski | e62a254 | 2015-10-05 17:48:02 -0700 | [diff] [blame] | 187 | |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 188 | /* Stash user ESP and switch to the kernel stack. */ |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 189 | movl %esp, %r8d |
| 190 | movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp |
Denys Vlasenko | a232e3d | 2015-03-27 11:36:20 +0100 | [diff] [blame] | 191 | |
Denys Vlasenko | 4ee8ec1 | 2015-03-27 11:36:21 +0100 | [diff] [blame] | 192 | /* Zero-extending 32-bit regs, do not remove */ |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 193 | movl %eax, %eax |
Denys Vlasenko | 4ee8ec1 | 2015-03-27 11:36:21 +0100 | [diff] [blame] | 194 | |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 195 | /* Construct struct pt_regs on stack */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 196 | pushq $__USER32_DS /* pt_regs->ss */ |
| 197 | pushq %r8 /* pt_regs->sp */ |
| 198 | pushq %r11 /* pt_regs->flags */ |
| 199 | pushq $__USER32_CS /* pt_regs->cs */ |
| 200 | pushq %rcx /* pt_regs->ip */ |
| 201 | pushq %rax /* pt_regs->orig_ax */ |
| 202 | pushq %rdi /* pt_regs->di */ |
| 203 | pushq %rsi /* pt_regs->si */ |
| 204 | pushq %rdx /* pt_regs->dx */ |
Andy Lutomirski | 30bfa7b | 2015-12-16 23:18:48 -0800 | [diff] [blame] | 205 | pushq %rbp /* pt_regs->cx (stashed in bp) */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 206 | pushq $-ENOSYS /* pt_regs->ax */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 207 | pushq $0 /* pt_regs->r8 = 0 */ |
| 208 | pushq $0 /* pt_regs->r9 = 0 */ |
| 209 | pushq $0 /* pt_regs->r10 = 0 */ |
| 210 | pushq $0 /* pt_regs->r11 = 0 */ |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 211 | pushq %rbx /* pt_regs->rbx */ |
Andy Lutomirski | 30bfa7b | 2015-12-16 23:18:48 -0800 | [diff] [blame] | 212 | pushq %rbp /* pt_regs->rbp (will be overwritten) */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 213 | pushq $0 /* pt_regs->r12 = 0 */ |
| 214 | pushq $0 /* pt_regs->r13 = 0 */ |
| 215 | pushq $0 /* pt_regs->r14 = 0 */ |
| 216 | pushq $0 /* pt_regs->r15 = 0 */ |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 217 | |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 218 | /* |
| 219 | * User mode is traced as though IRQs are on, and SYSENTER |
| 220 | * turned them off. |
| 221 | */ |
| 222 | TRACE_IRQS_OFF |
| 223 | |
| 224 | movq %rsp, %rdi |
| 225 | call do_fast_syscall_32 |
Boris Ostrovsky | 91e2eea | 2015-11-19 16:55:45 -0500 | [diff] [blame] | 226 | /* XEN PV guests always use IRET path */ |
| 227 | ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ |
| 228 | "jmp .Lsyscall_32_done", X86_FEATURE_XENPV |
Andy Lutomirski | 7841b40 | 2015-10-05 17:48:12 -0700 | [diff] [blame] | 229 | |
| 230 | /* Opportunistic SYSRET */ |
| 231 | sysret32_from_system_call: |
| 232 | TRACE_IRQS_ON /* User mode traces as IRQs on. */ |
| 233 | movq RBX(%rsp), %rbx /* pt_regs->rbx */ |
| 234 | movq RBP(%rsp), %rbp /* pt_regs->rbp */ |
| 235 | movq EFLAGS(%rsp), %r11 /* pt_regs->flags (in r11) */ |
| 236 | movq RIP(%rsp), %rcx /* pt_regs->ip (in rcx) */ |
| 237 | addq $RAX, %rsp /* Skip r8-r15 */ |
| 238 | popq %rax /* pt_regs->rax */ |
| 239 | popq %rdx /* Skip pt_regs->cx */ |
| 240 | popq %rdx /* pt_regs->dx */ |
| 241 | popq %rsi /* pt_regs->si */ |
| 242 | popq %rdi /* pt_regs->di */ |
| 243 | |
| 244 | /* |
| 245 | * USERGS_SYSRET32 does: |
| 246 | * GSBASE = user's GS base |
| 247 | * EIP = ECX |
| 248 | * RFLAGS = R11 |
| 249 | * CS = __USER32_CS |
| 250 | * SS = __USER_DS |
| 251 | * |
| 252 | * ECX will not match pt_regs->cx, but we're returning to a vDSO |
| 253 | * trampoline that will fix up RCX, so this is okay. |
| 254 | * |
| 255 | * R12-R15 are callee-saved, so they contain whatever was in them |
| 256 | * when the system call started, which is already known to user |
| 257 | * code. We zero R8-R10 to avoid info leaks. |
| 258 | */ |
| 259 | xorq %r8, %r8 |
| 260 | xorq %r9, %r9 |
| 261 | xorq %r10, %r10 |
| 262 | movq RSP-ORIG_RAX(%rsp), %rsp |
Boris Ostrovsky | 75ef821 | 2015-11-19 16:55:47 -0500 | [diff] [blame] | 263 | swapgs |
| 264 | sysretl |
Ingo Molnar | 2cd2355 | 2015-06-08 08:28:07 +0200 | [diff] [blame] | 265 | END(entry_SYSCALL_compat) |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 266 | |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 267 | /* |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 268 | * 32-bit legacy system call entry. |
| 269 | * |
| 270 | * 32-bit x86 Linux system calls traditionally used the INT $0x80 |
| 271 | * instruction. INT $0x80 lands here. |
| 272 | * |
| 273 | * This entry point can be used by 32-bit and 64-bit programs to perform |
| 274 | * 32-bit system calls. Instances of INT $0x80 can be found inline in |
| 275 | * various programs and libraries. It is also used by the vDSO's |
| 276 | * __kernel_vsyscall fallback for hardware that doesn't support a faster |
| 277 | * entry method. Restarted 32-bit system calls also fall back to INT |
| 278 | * $0x80 regardless of what instruction was originally used to do the |
| 279 | * system call. |
| 280 | * |
| 281 | * This is considered a slow path. It is not used by most libc |
| 282 | * implementations on modern hardware except during process startup. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 283 | * |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 284 | * Arguments: |
| 285 | * eax system call number |
| 286 | * ebx arg1 |
| 287 | * ecx arg2 |
| 288 | * edx arg3 |
| 289 | * esi arg4 |
| 290 | * edi arg5 |
Andy Lutomirski | fda57b2 | 2016-03-09 19:00:35 -0800 | [diff] [blame] | 291 | * ebp arg6 |
Denys Vlasenko | b87cf63 | 2015-02-26 14:40:32 -0800 | [diff] [blame] | 292 | */ |
Ingo Molnar | 2cd2355 | 2015-06-08 08:28:07 +0200 | [diff] [blame] | 293 | ENTRY(entry_INT80_compat) |
Denys Vlasenko | a232e3d | 2015-03-27 11:36:20 +0100 | [diff] [blame] | 294 | /* |
| 295 | * Interrupts are off on entry. |
Denys Vlasenko | a232e3d | 2015-03-27 11:36:20 +0100 | [diff] [blame] | 296 | */ |
Jeremy Fitzhardinge | 360c044 | 2008-07-08 15:06:28 -0700 | [diff] [blame] | 297 | PARAVIRT_ADJUST_EXCEPTION_FRAME |
Andy Lutomirski | 3d44d51 | 2016-02-24 12:18:49 -0800 | [diff] [blame] | 298 | ASM_CLAC /* Do this early to minimize exposure */ |
Jeremy Fitzhardinge | 6680415 | 2008-06-25 00:19:29 -0400 | [diff] [blame] | 299 | SWAPGS |
Denys Vlasenko | a232e3d | 2015-03-27 11:36:20 +0100 | [diff] [blame] | 300 | |
Andy Lutomirski | ee08c6b | 2015-10-05 17:48:09 -0700 | [diff] [blame] | 301 | /* |
| 302 | * User tracing code (ptrace or signal handlers) might assume that |
| 303 | * the saved RAX contains a 32-bit number when we're invoking a 32-bit |
| 304 | * syscall. Just in case the high bits are nonzero, zero-extend |
| 305 | * the syscall number. (This could almost certainly be deleted |
| 306 | * with no ill effects.) |
| 307 | */ |
Ingo Molnar | 54ad726 | 2015-06-05 13:02:28 +0200 | [diff] [blame] | 308 | movl %eax, %eax |
Denys Vlasenko | 4ee8ec1 | 2015-03-27 11:36:21 +0100 | [diff] [blame] | 309 | |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 310 | /* Construct struct pt_regs on stack (iret frame is already on stack) */ |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 311 | pushq %rax /* pt_regs->orig_ax */ |
| 312 | pushq %rdi /* pt_regs->di */ |
| 313 | pushq %rsi /* pt_regs->si */ |
| 314 | pushq %rdx /* pt_regs->dx */ |
| 315 | pushq %rcx /* pt_regs->cx */ |
| 316 | pushq $-ENOSYS /* pt_regs->ax */ |
Denys Vlasenko | 778843f | 2016-05-02 16:56:50 +0200 | [diff] [blame] | 317 | pushq $0 /* pt_regs->r8 = 0 */ |
| 318 | pushq $0 /* pt_regs->r9 = 0 */ |
| 319 | pushq $0 /* pt_regs->r10 = 0 */ |
| 320 | pushq $0 /* pt_regs->r11 = 0 */ |
Andy Lutomirski | 8169aff | 2015-10-05 17:48:05 -0700 | [diff] [blame] | 321 | pushq %rbx /* pt_regs->rbx */ |
| 322 | pushq %rbp /* pt_regs->rbp */ |
| 323 | pushq %r12 /* pt_regs->r12 */ |
| 324 | pushq %r13 /* pt_regs->r13 */ |
| 325 | pushq %r14 /* pt_regs->r14 */ |
| 326 | pushq %r15 /* pt_regs->r15 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 327 | cld |
Denys Vlasenko | 4c9c0e9 | 2015-03-31 19:00:04 +0200 | [diff] [blame] | 328 | |
Denys Vlasenko | 73cbf68 | 2015-06-02 21:04:02 +0200 | [diff] [blame] | 329 | /* |
Andy Lutomirski | ee08c6b | 2015-10-05 17:48:09 -0700 | [diff] [blame] | 330 | * User mode is traced as though IRQs are on, and the interrupt |
| 331 | * gate turned them off. |
Denys Vlasenko | 73cbf68 | 2015-06-02 21:04:02 +0200 | [diff] [blame] | 332 | */ |
Andy Lutomirski | ee08c6b | 2015-10-05 17:48:09 -0700 | [diff] [blame] | 333 | TRACE_IRQS_OFF |
| 334 | |
| 335 | movq %rsp, %rdi |
Andy Lutomirski | a798f09 | 2016-03-09 13:24:32 -0800 | [diff] [blame] | 336 | call do_int80_syscall_32 |
Andy Lutomirski | a474e67 | 2015-10-05 17:48:11 -0700 | [diff] [blame] | 337 | .Lsyscall_32_done: |
Andy Lutomirski | ee08c6b | 2015-10-05 17:48:09 -0700 | [diff] [blame] | 338 | |
| 339 | /* Go back to user mode. */ |
| 340 | TRACE_IRQS_ON |
| 341 | SWAPGS |
| 342 | jmp restore_regs_and_iret |
Ingo Molnar | 2cd2355 | 2015-06-08 08:28:07 +0200 | [diff] [blame] | 343 | END(entry_INT80_compat) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 344 | |
Jan Beulich | f6b2bc8 | 2011-11-29 11:24:10 +0000 | [diff] [blame] | 345 | ALIGN |
Al Viro | 1d4b4b2 | 2012-10-22 22:34:11 -0400 | [diff] [blame] | 346 | GLOBAL(stub32_clone) |
Denys Vlasenko | 5cdc683 | 2015-06-03 15:58:49 +0200 | [diff] [blame] | 347 | /* |
Denys Vlasenko | 7a5a982 | 2015-06-03 15:58:50 +0200 | [diff] [blame] | 348 | * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr). |
| 349 | * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val). |
| 350 | * |
| 351 | * The native 64-bit kernel's sys_clone() implements the latter, |
| 352 | * so we need to swap arguments here before calling it: |
Denys Vlasenko | 5cdc683 | 2015-06-03 15:58:49 +0200 | [diff] [blame] | 353 | */ |
Denys Vlasenko | 7a5a982 | 2015-06-03 15:58:50 +0200 | [diff] [blame] | 354 | xchg %r8, %rcx |
Andy Lutomirski | 8169aff | 2015-10-05 17:48:05 -0700 | [diff] [blame] | 355 | jmp sys_clone |