Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 2 | * Copyright (C) 1991,1992 Linus Torvalds |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 4 | * entry_32.S contains the system-call and low-level fault and trap handling routines. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * |
Andy Lutomirski | 39e8701 | 2015-10-05 17:48:13 -0700 | [diff] [blame] | 6 | * Stack layout while running C code: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 7 | * ptrace needs to have all registers on the stack. |
| 8 | * If the order here is changed, it needs to be |
| 9 | * updated in fork.c:copy_process(), signal.c:do_signal(), |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | * ptrace.c and ptrace.h |
| 11 | * |
| 12 | * 0(%esp) - %ebx |
| 13 | * 4(%esp) - %ecx |
| 14 | * 8(%esp) - %edx |
Denys Vlasenko | 9b47feb | 2015-06-08 22:35:33 +0200 | [diff] [blame] | 15 | * C(%esp) - %esi |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | * 10(%esp) - %edi |
| 17 | * 14(%esp) - %ebp |
| 18 | * 18(%esp) - %eax |
| 19 | * 1C(%esp) - %ds |
| 20 | * 20(%esp) - %es |
Jeremy Fitzhardinge | 464d1a7 | 2007-02-13 13:26:20 +0100 | [diff] [blame] | 21 | * 24(%esp) - %fs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 22 | * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS |
| 23 | * 2C(%esp) - orig_eax |
| 24 | * 30(%esp) - %eip |
| 25 | * 34(%esp) - %cs |
| 26 | * 38(%esp) - %eflags |
| 27 | * 3C(%esp) - %oldesp |
| 28 | * 40(%esp) - %oldss |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | */ |
| 30 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #include <linux/linkage.h> |
Eric Paris | d7e7528 | 2012-01-03 14:23:06 -0500 | [diff] [blame] | 32 | #include <linux/err.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 33 | #include <asm/thread_info.h> |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 34 | #include <asm/irqflags.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | #include <asm/errno.h> |
| 36 | #include <asm/segment.h> |
| 37 | #include <asm/smp.h> |
Jeremy Fitzhardinge | 0341c14 | 2009-02-13 11:14:01 -0800 | [diff] [blame] | 38 | #include <asm/page_types.h> |
Stas Sergeev | be44d2a | 2006-12-07 02:14:01 +0100 | [diff] [blame] | 39 | #include <asm/percpu.h> |
Cyrill Gorcunov | ab68ed9 | 2008-03-25 22:16:32 +0300 | [diff] [blame] | 40 | #include <asm/processor-flags.h> |
Abhishek Sagar | 395a59d | 2008-06-21 23:47:27 +0530 | [diff] [blame] | 41 | #include <asm/ftrace.h> |
Thomas Gleixner | 9b7dc56 | 2008-05-02 20:10:09 +0200 | [diff] [blame] | 42 | #include <asm/irq_vectors.h> |
Brian Gerst | 40d2e76 | 2010-03-21 09:00:43 -0400 | [diff] [blame] | 43 | #include <asm/cpufeature.h> |
Andy Lutomirski | b4ca46e | 2011-08-25 16:10:33 -0400 | [diff] [blame] | 44 | #include <asm/alternative-asm.h> |
H. Peter Anvin | 6837a54 | 2012-04-20 12:19:50 -0700 | [diff] [blame] | 45 | #include <asm/asm.h> |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 46 | #include <asm/smap.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | |
Jiri Olsa | ea71454 | 2011-03-07 19:10:39 +0100 | [diff] [blame] | 48 | .section .entry.text, "ax" |
| 49 | |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 50 | /* |
| 51 | * We use macros for low-level operations which need to be overridden |
| 52 | * for paravirtualization. The following will never clobber any registers: |
| 53 | * INTERRUPT_RETURN (aka. "iret") |
| 54 | * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 55 | * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 56 | * |
| 57 | * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must |
| 58 | * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). |
| 59 | * Allowing a register to be clobbered can shrink the paravirt replacement |
| 60 | * enough to patch inline, increasing performance. |
| 61 | */ |
| 62 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | #ifdef CONFIG_PREEMPT |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 64 | # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | #else |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 66 | # define preempt_stop(clobbers) |
| 67 | # define resume_kernel restore_all |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | #endif |
| 69 | |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 70 | .macro TRACE_IRQS_IRET |
| 71 | #ifdef CONFIG_TRACE_IRQFLAGS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 72 | testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? |
| 73 | jz 1f |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 74 | TRACE_IRQS_ON |
| 75 | 1: |
| 76 | #endif |
| 77 | .endm |
| 78 | |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 79 | /* |
| 80 | * User gs save/restore |
| 81 | * |
| 82 | * %gs is used for userland TLS and kernel only uses it for stack |
| 83 | * canary which is required to be at %gs:20 by gcc. Read the comment |
| 84 | * at the top of stackprotector.h for more info. |
| 85 | * |
| 86 | * Local labels 98 and 99 are used. |
| 87 | */ |
| 88 | #ifdef CONFIG_X86_32_LAZY_GS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 89 | |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 90 | /* unfortunately push/pop can't be no-op */ |
| 91 | .macro PUSH_GS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 92 | pushl $0 |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 93 | .endm |
| 94 | .macro POP_GS pop=0 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 95 | addl $(4 + \pop), %esp |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 96 | .endm |
| 97 | .macro POP_GS_EX |
| 98 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 100 | /* all the rest are no-op */ |
| 101 | .macro PTGS_TO_GS |
| 102 | .endm |
| 103 | .macro PTGS_TO_GS_EX |
| 104 | .endm |
| 105 | .macro GS_TO_REG reg |
| 106 | .endm |
| 107 | .macro REG_TO_PTGS reg |
| 108 | .endm |
| 109 | .macro SET_KERNEL_GS reg |
| 110 | .endm |
| 111 | |
| 112 | #else /* CONFIG_X86_32_LAZY_GS */ |
| 113 | |
| 114 | .macro PUSH_GS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 115 | pushl %gs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 116 | .endm |
| 117 | |
| 118 | .macro POP_GS pop=0 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 119 | 98: popl %gs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 120 | .if \pop <> 0 |
Denys Vlasenko | 9b47feb | 2015-06-08 22:35:33 +0200 | [diff] [blame] | 121 | add $\pop, %esp |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 122 | .endif |
| 123 | .endm |
| 124 | .macro POP_GS_EX |
| 125 | .pushsection .fixup, "ax" |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 126 | 99: movl $0, (%esp) |
| 127 | jmp 98b |
Jeremy Fitzhardinge | f95d47c | 2006-12-07 02:14:02 +0100 | [diff] [blame] | 128 | .popsection |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 129 | _ASM_EXTABLE(98b, 99b) |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 130 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 132 | .macro PTGS_TO_GS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 133 | 98: mov PT_GS(%esp), %gs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 134 | .endm |
| 135 | .macro PTGS_TO_GS_EX |
| 136 | .pushsection .fixup, "ax" |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 137 | 99: movl $0, PT_GS(%esp) |
| 138 | jmp 98b |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 139 | .popsection |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 140 | _ASM_EXTABLE(98b, 99b) |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 141 | .endm |
| 142 | |
| 143 | .macro GS_TO_REG reg |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 144 | movl %gs, \reg |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 145 | .endm |
| 146 | .macro REG_TO_PTGS reg |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 147 | movl \reg, PT_GS(%esp) |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 148 | .endm |
| 149 | .macro SET_KERNEL_GS reg |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 150 | movl $(__KERNEL_STACK_CANARY), \reg |
| 151 | movl \reg, %gs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 152 | .endm |
| 153 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 154 | #endif /* CONFIG_X86_32_LAZY_GS */ |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 155 | |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 156 | .macro SAVE_ALL pt_regs_ax=%eax |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 157 | cld |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 158 | PUSH_GS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 159 | pushl %fs |
| 160 | pushl %es |
| 161 | pushl %ds |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 162 | pushl \pt_regs_ax |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 163 | pushl %ebp |
| 164 | pushl %edi |
| 165 | pushl %esi |
| 166 | pushl %edx |
| 167 | pushl %ecx |
| 168 | pushl %ebx |
| 169 | movl $(__USER_DS), %edx |
| 170 | movl %edx, %ds |
| 171 | movl %edx, %es |
| 172 | movl $(__KERNEL_PERCPU), %edx |
| 173 | movl %edx, %fs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 174 | SET_KERNEL_GS %edx |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 175 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 177 | .macro RESTORE_INT_REGS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 178 | popl %ebx |
| 179 | popl %ecx |
| 180 | popl %edx |
| 181 | popl %esi |
| 182 | popl %edi |
| 183 | popl %ebp |
| 184 | popl %eax |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 185 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 187 | .macro RESTORE_REGS pop=0 |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 188 | RESTORE_INT_REGS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 189 | 1: popl %ds |
| 190 | 2: popl %es |
| 191 | 3: popl %fs |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 192 | POP_GS \pop |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 193 | .pushsection .fixup, "ax" |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 194 | 4: movl $0, (%esp) |
| 195 | jmp 1b |
| 196 | 5: movl $0, (%esp) |
| 197 | jmp 2b |
| 198 | 6: movl $0, (%esp) |
| 199 | jmp 3b |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | .popsection |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 201 | _ASM_EXTABLE(1b, 4b) |
| 202 | _ASM_EXTABLE(2b, 5b) |
| 203 | _ASM_EXTABLE(3b, 6b) |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 204 | POP_GS_EX |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 205 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | ENTRY(ret_from_fork) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 208 | pushl %eax |
| 209 | call schedule_tail |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | GET_THREAD_INFO(%ebp) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 211 | popl %eax |
| 212 | pushl $0x0202 # Reset kernel eflags |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 213 | popfl |
Andy Lutomirski | 39e8701 | 2015-10-05 17:48:13 -0700 | [diff] [blame] | 214 | |
| 215 | /* When we fork, we trace the syscall return in the child, too. */ |
| 216 | movl %esp, %eax |
| 217 | call syscall_return_slowpath |
| 218 | jmp restore_all |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 219 | END(ret_from_fork) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | |
Al Viro | 22e2430d | 2012-10-10 21:35:42 -0400 | [diff] [blame] | 221 | ENTRY(ret_from_kernel_thread) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 222 | pushl %eax |
| 223 | call schedule_tail |
Al Viro | 6783eaa | 2012-08-02 23:05:11 +0400 | [diff] [blame] | 224 | GET_THREAD_INFO(%ebp) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 225 | popl %eax |
| 226 | pushl $0x0202 # Reset kernel eflags |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 227 | popfl |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 228 | movl PT_EBP(%esp), %eax |
| 229 | call *PT_EBX(%esp) |
| 230 | movl $0, PT_EAX(%esp) |
Andy Lutomirski | 39e8701 | 2015-10-05 17:48:13 -0700 | [diff] [blame] | 231 | |
| 232 | /* |
| 233 | * Kernel threads return to userspace as if returning from a syscall. |
| 234 | * We should check whether anything actually uses this path and, if so, |
| 235 | * consider switching it over to ret_from_fork. |
| 236 | */ |
| 237 | movl %esp, %eax |
| 238 | call syscall_return_slowpath |
| 239 | jmp restore_all |
Al Viro | 22e2430d | 2012-10-10 21:35:42 -0400 | [diff] [blame] | 240 | ENDPROC(ret_from_kernel_thread) |
Al Viro | 6783eaa | 2012-08-02 23:05:11 +0400 | [diff] [blame] | 241 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 242 | /* |
| 243 | * Return to user mode is not as complex as all this looks, |
| 244 | * but we want the default path for a system call return to |
| 245 | * go as quickly as possible which is why some of this is |
| 246 | * less clear than it otherwise should be. |
| 247 | */ |
| 248 | |
| 249 | # userspace resumption stub bypassing syscall exit tracing |
| 250 | ALIGN |
| 251 | ret_from_exception: |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 252 | preempt_stop(CLBR_ANY) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 253 | ret_from_intr: |
| 254 | GET_THREAD_INFO(%ebp) |
Dmitry Adamushko | 29a2e28 | 2012-03-22 21:39:25 +0100 | [diff] [blame] | 255 | #ifdef CONFIG_VM86 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 256 | movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS |
| 257 | movb PT_CS(%esp), %al |
| 258 | andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax |
Dmitry Adamushko | 29a2e28 | 2012-03-22 21:39:25 +0100 | [diff] [blame] | 259 | #else |
| 260 | /* |
Al Viro | 6783eaa | 2012-08-02 23:05:11 +0400 | [diff] [blame] | 261 | * We can be coming here from child spawned by kernel_thread(). |
Dmitry Adamushko | 29a2e28 | 2012-03-22 21:39:25 +0100 | [diff] [blame] | 262 | */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 263 | movl PT_CS(%esp), %eax |
| 264 | andl $SEGMENT_RPL_MASK, %eax |
Dmitry Adamushko | 29a2e28 | 2012-03-22 21:39:25 +0100 | [diff] [blame] | 265 | #endif |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 266 | cmpl $USER_RPL, %eax |
| 267 | jb resume_kernel # not returning to v8086 or userspace |
Jeremy Fitzhardinge | f95d47c | 2006-12-07 02:14:02 +0100 | [diff] [blame] | 268 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | ENTRY(resume_userspace) |
Andy Lutomirski | 5d73fc7 | 2015-07-31 14:41:09 -0700 | [diff] [blame] | 270 | DISABLE_INTERRUPTS(CLBR_ANY) |
Peter Zijlstra | e32e58a | 2008-06-06 10:14:08 +0200 | [diff] [blame] | 271 | TRACE_IRQS_OFF |
Andy Lutomirski | 5d73fc7 | 2015-07-31 14:41:09 -0700 | [diff] [blame] | 272 | movl %esp, %eax |
| 273 | call prepare_exit_to_usermode |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 274 | jmp restore_all |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 275 | END(ret_from_exception) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | |
| 277 | #ifdef CONFIG_PREEMPT |
| 278 | ENTRY(resume_kernel) |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 279 | DISABLE_INTERRUPTS(CLBR_ANY) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | need_resched: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 281 | cmpl $0, PER_CPU_VAR(__preempt_count) |
| 282 | jnz restore_all |
| 283 | testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? |
| 284 | jz restore_all |
| 285 | call preempt_schedule_irq |
| 286 | jmp need_resched |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 287 | END(resume_kernel) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | #endif |
| 289 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 290 | # SYSENTER call handler stub |
Ingo Molnar | 4c8cd0c | 2015-06-08 08:33:56 +0200 | [diff] [blame] | 291 | ENTRY(entry_SYSENTER_32) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 292 | movl TSS_sysenter_sp0(%esp), %esp |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 293 | sysenter_past_esp: |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 294 | pushl $__USER_DS /* pt_regs->ss */ |
| 295 | pushl %ecx /* pt_regs->cx */ |
| 296 | pushfl /* pt_regs->flags (except IF = 0) */ |
| 297 | orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ |
| 298 | pushl $__USER_CS /* pt_regs->cs */ |
| 299 | pushl $0 /* pt_regs->ip = 0 (placeholder) */ |
| 300 | pushl %eax /* pt_regs->orig_ax */ |
| 301 | SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ |
| 302 | |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 303 | /* |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 304 | * User mode is traced as though IRQs are on, and SYSENTER |
| 305 | * turned them off. |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 306 | */ |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 307 | TRACE_IRQS_OFF |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 308 | |
| 309 | movl %esp, %eax |
| 310 | call do_fast_syscall_32 |
| 311 | testl %eax, %eax |
| 312 | jz .Lsyscall_32_done |
| 313 | |
| 314 | /* Opportunistic SYSEXIT */ |
| 315 | TRACE_IRQS_ON /* User mode traces as IRQs on. */ |
| 316 | movl PT_EIP(%esp), %edx /* pt_regs->ip */ |
| 317 | movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ |
Andy Lutomirski | 3bd2951 | 2015-10-16 15:42:55 -0700 | [diff] [blame^] | 318 | 1: mov PT_FS(%esp), %fs |
| 319 | PTGS_TO_GS |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 320 | popl %ebx /* pt_regs->bx */ |
| 321 | addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ |
| 322 | popl %esi /* pt_regs->si */ |
| 323 | popl %edi /* pt_regs->di */ |
| 324 | popl %ebp /* pt_regs->bp */ |
| 325 | popl %eax /* pt_regs->ax */ |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 326 | |
| 327 | /* |
| 328 | * Return back to the vDSO, which will pop ecx and edx. |
| 329 | * Don't bother with DS and ES (they already contain __USER_DS). |
| 330 | */ |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 331 | ENABLE_INTERRUPTS_SYSEXIT |
Roland McGrath | af0575b | 2008-06-24 04:16:52 -0700 | [diff] [blame] | 332 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 333 | .pushsection .fixup, "ax" |
| 334 | 2: movl $0, PT_FS(%esp) |
| 335 | jmp 1b |
Jeremy Fitzhardinge | f95d47c | 2006-12-07 02:14:02 +0100 | [diff] [blame] | 336 | .popsection |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 337 | _ASM_EXTABLE(1b, 2b) |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 338 | PTGS_TO_GS_EX |
Ingo Molnar | 4c8cd0c | 2015-06-08 08:33:56 +0200 | [diff] [blame] | 339 | ENDPROC(entry_SYSENTER_32) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | |
| 341 | # system call handler stub |
Ingo Molnar | b2502b4 | 2015-06-08 08:42:03 +0200 | [diff] [blame] | 342 | ENTRY(entry_INT80_32) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 343 | ASM_CLAC |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 344 | pushl %eax /* pt_regs->orig_ax */ |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 345 | SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 346 | |
| 347 | /* |
Andy Lutomirski | 657c1ee | 2015-10-16 15:42:54 -0700 | [diff] [blame] | 348 | * User mode is traced as though IRQs are on. Unlike the 64-bit |
| 349 | * case, INT80 is a trap gate on 32-bit kernels, so interrupts |
| 350 | * are already on (unless user code is messing around with iopl). |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 351 | */ |
Andy Lutomirski | 150ac78 | 2015-10-05 17:48:14 -0700 | [diff] [blame] | 352 | |
| 353 | movl %esp, %eax |
Andy Lutomirski | 657c1ee | 2015-10-16 15:42:54 -0700 | [diff] [blame] | 354 | call do_syscall_32_irqs_on |
Andy Lutomirski | 5f310f7 | 2015-10-05 17:48:15 -0700 | [diff] [blame] | 355 | .Lsyscall_32_done: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 356 | |
| 357 | restore_all: |
Alexander van Heukelum | 2e04bc7 | 2009-06-18 00:35:57 +0200 | [diff] [blame] | 358 | TRACE_IRQS_IRET |
| 359 | restore_all_notrace: |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 360 | #ifdef CONFIG_X86_ESPFIX32 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 361 | movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS |
| 362 | /* |
| 363 | * Warning: PT_OLDSS(%esp) contains the wrong/random values if we |
| 364 | * are returning to the kernel. |
| 365 | * See comments in process.c:copy_thread() for details. |
| 366 | */ |
| 367 | movb PT_OLDSS(%esp), %ah |
| 368 | movb PT_CS(%esp), %al |
| 369 | andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax |
| 370 | cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax |
| 371 | je ldt_ss # returning to user-space with LDT SS |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 372 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 373 | restore_nocheck: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 374 | RESTORE_REGS 4 # skip orig_eax/error_code |
Adrian Bunk | f7f3d79 | 2008-02-13 23:29:53 +0200 | [diff] [blame] | 375 | irq_return: |
Ingo Molnar | 3701d863 | 2008-02-09 23:24:08 +0100 | [diff] [blame] | 376 | INTERRUPT_RETURN |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 377 | .section .fixup, "ax" |
| 378 | ENTRY(iret_exc ) |
| 379 | pushl $0 # no error code |
| 380 | pushl $do_iret_error |
| 381 | jmp error_code |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 382 | .previous |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 383 | _ASM_EXTABLE(irq_return, iret_exc) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 385 | #ifdef CONFIG_X86_ESPFIX32 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | ldt_ss: |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 387 | #ifdef CONFIG_PARAVIRT |
| 388 | /* |
| 389 | * The kernel can't run on a non-flat stack if paravirt mode |
| 390 | * is active. Rather than try to fixup the high bits of |
| 391 | * ESP, bypass this code entirely. This may break DOSemu |
| 392 | * and/or Wine support in a paravirt VM, although the option |
| 393 | * is still available to implement the setting of the high |
| 394 | * 16-bits in the INTERRUPT_RETURN paravirt-op. |
| 395 | */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 396 | cmpl $0, pv_info+PARAVIRT_enabled |
| 397 | jne restore_nocheck |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 398 | #endif |
| 399 | |
Alexander van Heukelum | dc4c2a0 | 2009-06-18 00:35:58 +0200 | [diff] [blame] | 400 | /* |
| 401 | * Setup and switch to ESPFIX stack |
| 402 | * |
| 403 | * We're returning to userspace with a 16 bit stack. The CPU will not |
| 404 | * restore the high word of ESP for us on executing iret... This is an |
| 405 | * "official" bug of all the x86-compatible CPUs, which we can work |
| 406 | * around to make dosemu and wine happy. We do this by preloading the |
| 407 | * high word of ESP with the high word of the userspace ESP while |
| 408 | * compensating for the offset by changing to the ESPFIX segment with |
| 409 | * a base address that matches for the difference. |
| 410 | */ |
Brian Gerst | 72c511d | 2010-07-31 12:48:23 -0400 | [diff] [blame] | 411 | #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 412 | mov %esp, %edx /* load kernel esp */ |
| 413 | mov PT_OLDESP(%esp), %eax /* load userspace esp */ |
| 414 | mov %dx, %ax /* eax: new kernel esp */ |
Denys Vlasenko | 9b47feb | 2015-06-08 22:35:33 +0200 | [diff] [blame] | 415 | sub %eax, %edx /* offset (low word is 0) */ |
| 416 | shr $16, %edx |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 417 | mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ |
| 418 | mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ |
| 419 | pushl $__ESPFIX_SS |
| 420 | pushl %eax /* new kernel esp */ |
| 421 | /* |
| 422 | * Disable interrupts, but do not irqtrace this section: we |
Alexander van Heukelum | 2e04bc7 | 2009-06-18 00:35:57 +0200 | [diff] [blame] | 423 | * will soon execute iret and the tracer was already set to |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 424 | * the irqstate after the IRET: |
| 425 | */ |
Rusty Russell | 139ec7c | 2006-12-07 02:14:08 +0100 | [diff] [blame] | 426 | DISABLE_INTERRUPTS(CLBR_EAX) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 427 | lss (%esp), %esp /* switch to espfix segment */ |
| 428 | jmp restore_nocheck |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 429 | #endif |
Ingo Molnar | b2502b4 | 2015-06-08 08:42:03 +0200 | [diff] [blame] | 430 | ENDPROC(entry_INT80_32) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 431 | |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 432 | .macro FIXUP_ESPFIX_STACK |
Alexander van Heukelum | dc4c2a0 | 2009-06-18 00:35:58 +0200 | [diff] [blame] | 433 | /* |
| 434 | * Switch back for ESPFIX stack to the normal zerobased stack |
| 435 | * |
| 436 | * We can't call C functions using the ESPFIX stack. This code reads |
| 437 | * the high word of the segment base from the GDT and swiches to the |
| 438 | * normal stack and adjusts ESP with the matching offset. |
| 439 | */ |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 440 | #ifdef CONFIG_X86_ESPFIX32 |
Alexander van Heukelum | dc4c2a0 | 2009-06-18 00:35:58 +0200 | [diff] [blame] | 441 | /* fixup the stack */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 442 | mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ |
| 443 | mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ |
Denys Vlasenko | 9b47feb | 2015-06-08 22:35:33 +0200 | [diff] [blame] | 444 | shl $16, %eax |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 445 | addl %esp, %eax /* the adjusted stack pointer */ |
| 446 | pushl $__KERNEL_DS |
| 447 | pushl %eax |
| 448 | lss (%esp), %esp /* switch to the normal stack segment */ |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 449 | #endif |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 450 | .endm |
| 451 | .macro UNWIND_ESPFIX_STACK |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 452 | #ifdef CONFIG_X86_ESPFIX32 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 453 | movl %ss, %eax |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 454 | /* see if on espfix stack */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 455 | cmpw $__ESPFIX_SS, %ax |
| 456 | jne 27f |
| 457 | movl $__KERNEL_DS, %eax |
| 458 | movl %eax, %ds |
| 459 | movl %eax, %es |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 460 | /* switch to normal stack */ |
| 461 | FIXUP_ESPFIX_STACK |
| 462 | 27: |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 463 | #endif |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 464 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | |
| 466 | /* |
Denys Vlasenko | 3304c9c | 2015-04-03 21:49:13 +0200 | [diff] [blame] | 467 | * Build the entry stubs with some assembler magic. |
| 468 | * We pack 1 stub into every 8-byte block. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | */ |
Denys Vlasenko | 3304c9c | 2015-04-03 21:49:13 +0200 | [diff] [blame] | 470 | .align 8 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | ENTRY(irq_entries_start) |
Denys Vlasenko | 3304c9c | 2015-04-03 21:49:13 +0200 | [diff] [blame] | 472 | vector=FIRST_EXTERNAL_VECTOR |
| 473 | .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 474 | pushl $(~vector+0x80) /* Note: always in signed byte range */ |
Denys Vlasenko | 3304c9c | 2015-04-03 21:49:13 +0200 | [diff] [blame] | 475 | vector=vector+1 |
| 476 | jmp common_interrupt |
Denys Vlasenko | 3304c9c | 2015-04-03 21:49:13 +0200 | [diff] [blame] | 477 | .align 8 |
| 478 | .endr |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 479 | END(irq_entries_start) |
| 480 | |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 481 | /* |
| 482 | * the CPU automatically disables interrupts when executing an IRQ vector, |
| 483 | * so IRQ-flags tracing has to follow that: |
| 484 | */ |
H. Peter Anvin | b7c6244 | 2008-11-11 13:24:58 -0800 | [diff] [blame] | 485 | .p2align CONFIG_X86_L1_CACHE_SHIFT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 486 | common_interrupt: |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 487 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 488 | addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 489 | SAVE_ALL |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 490 | TRACE_IRQS_OFF |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 491 | movl %esp, %eax |
| 492 | call do_IRQ |
| 493 | jmp ret_from_intr |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 494 | ENDPROC(common_interrupt) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 495 | |
Tejun Heo | 02cf94c | 2009-01-21 17:26:06 +0900 | [diff] [blame] | 496 | #define BUILD_INTERRUPT3(name, nr, fn) \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 497 | ENTRY(name) \ |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 498 | ASM_CLAC; \ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 499 | pushl $~(nr); \ |
Jan Beulich | fe7cacc | 2006-06-26 13:57:44 +0200 | [diff] [blame] | 500 | SAVE_ALL; \ |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 501 | TRACE_IRQS_OFF \ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 502 | movl %esp, %eax; \ |
| 503 | call fn; \ |
| 504 | jmp ret_from_intr; \ |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 505 | ENDPROC(name) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 506 | |
Seiji Aguchi | cf910e8 | 2013-06-20 11:46:53 -0400 | [diff] [blame] | 507 | |
| 508 | #ifdef CONFIG_TRACING |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 509 | # define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) |
Seiji Aguchi | cf910e8 | 2013-06-20 11:46:53 -0400 | [diff] [blame] | 510 | #else |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 511 | # define TRACE_BUILD_INTERRUPT(name, nr) |
Seiji Aguchi | cf910e8 | 2013-06-20 11:46:53 -0400 | [diff] [blame] | 512 | #endif |
| 513 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 514 | #define BUILD_INTERRUPT(name, nr) \ |
| 515 | BUILD_INTERRUPT3(name, nr, smp_##name); \ |
Seiji Aguchi | cf910e8 | 2013-06-20 11:46:53 -0400 | [diff] [blame] | 516 | TRACE_BUILD_INTERRUPT(name, nr) |
Tejun Heo | 02cf94c | 2009-01-21 17:26:06 +0900 | [diff] [blame] | 517 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 518 | /* The include is where all of the SMP etc. interrupts come from */ |
Ingo Molnar | 1164dd0 | 2009-01-28 19:34:09 +0100 | [diff] [blame] | 519 | #include <asm/entry_arch.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 520 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 521 | ENTRY(coprocessor_error) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 522 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 523 | pushl $0 |
| 524 | pushl $do_coprocessor_error |
| 525 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 526 | END(coprocessor_error) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 527 | |
| 528 | ENTRY(simd_coprocessor_error) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 529 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 530 | pushl $0 |
Brian Gerst | 40d2e76 | 2010-03-21 09:00:43 -0400 | [diff] [blame] | 531 | #ifdef CONFIG_X86_INVD_BUG |
| 532 | /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 533 | ALTERNATIVE "pushl $do_general_protection", \ |
| 534 | "pushl $do_simd_coprocessor_error", \ |
Borislav Petkov | 8e65f6e | 2015-01-18 12:35:55 +0100 | [diff] [blame] | 535 | X86_FEATURE_XMM |
Brian Gerst | 40d2e76 | 2010-03-21 09:00:43 -0400 | [diff] [blame] | 536 | #else |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 537 | pushl $do_simd_coprocessor_error |
Brian Gerst | 40d2e76 | 2010-03-21 09:00:43 -0400 | [diff] [blame] | 538 | #endif |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 539 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 540 | END(simd_coprocessor_error) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 541 | |
| 542 | ENTRY(device_not_available) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 543 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 544 | pushl $-1 # mark this as an int |
| 545 | pushl $do_device_not_available |
| 546 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 547 | END(device_not_available) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 549 | #ifdef CONFIG_PARAVIRT |
| 550 | ENTRY(native_iret) |
Ingo Molnar | 3701d863 | 2008-02-09 23:24:08 +0100 | [diff] [blame] | 551 | iret |
H. Peter Anvin | 6837a54 | 2012-04-20 12:19:50 -0700 | [diff] [blame] | 552 | _ASM_EXTABLE(native_iret, iret_exc) |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 553 | END(native_iret) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 554 | |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 555 | ENTRY(native_irq_enable_sysexit) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 556 | sti |
| 557 | sysexit |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 558 | END(native_irq_enable_sysexit) |
Rusty Russell | d3561b7 | 2006-12-07 02:14:07 +0100 | [diff] [blame] | 559 | #endif |
| 560 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 561 | ENTRY(overflow) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 562 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 563 | pushl $0 |
| 564 | pushl $do_overflow |
| 565 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 566 | END(overflow) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 567 | |
| 568 | ENTRY(bounds) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 569 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 570 | pushl $0 |
| 571 | pushl $do_bounds |
| 572 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 573 | END(bounds) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 574 | |
| 575 | ENTRY(invalid_op) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 576 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 577 | pushl $0 |
| 578 | pushl $do_invalid_op |
| 579 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 580 | END(invalid_op) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 581 | |
| 582 | ENTRY(coprocessor_segment_overrun) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 583 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 584 | pushl $0 |
| 585 | pushl $do_coprocessor_segment_overrun |
| 586 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 587 | END(coprocessor_segment_overrun) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 588 | |
| 589 | ENTRY(invalid_TSS) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 590 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 591 | pushl $do_invalid_TSS |
| 592 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 593 | END(invalid_TSS) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 594 | |
| 595 | ENTRY(segment_not_present) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 596 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 597 | pushl $do_segment_not_present |
| 598 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 599 | END(segment_not_present) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 600 | |
| 601 | ENTRY(stack_segment) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 602 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 603 | pushl $do_stack_segment |
| 604 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 605 | END(stack_segment) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 606 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 607 | ENTRY(alignment_check) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 608 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 609 | pushl $do_alignment_check |
| 610 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 611 | END(alignment_check) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 612 | |
Prasanna S.P | d28c439 | 2006-09-26 10:52:34 +0200 | [diff] [blame] | 613 | ENTRY(divide_error) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 614 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 615 | pushl $0 # no error code |
| 616 | pushl $do_divide_error |
| 617 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 618 | END(divide_error) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 619 | |
| 620 | #ifdef CONFIG_X86_MCE |
| 621 | ENTRY(machine_check) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 622 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 623 | pushl $0 |
| 624 | pushl machine_check_vector |
| 625 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 626 | END(machine_check) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 627 | #endif |
| 628 | |
| 629 | ENTRY(spurious_interrupt_bug) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 630 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 631 | pushl $0 |
| 632 | pushl $do_spurious_interrupt_bug |
| 633 | jmp error_code |
Jan Beulich | 47a55cd | 2007-02-13 13:26:24 +0100 | [diff] [blame] | 634 | END(spurious_interrupt_bug) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 635 | |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 636 | #ifdef CONFIG_XEN |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 637 | /* |
| 638 | * Xen doesn't set %esp to be precisely what the normal SYSENTER |
| 639 | * entry point expects, so fix it up before using the normal path. |
| 640 | */ |
Jeremy Fitzhardinge | e2a81ba | 2008-03-17 16:37:17 -0700 | [diff] [blame] | 641 | ENTRY(xen_sysenter_target) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 642 | addl $5*4, %esp /* remove xen-provided frame */ |
| 643 | jmp sysenter_past_esp |
Jeremy Fitzhardinge | e2a81ba | 2008-03-17 16:37:17 -0700 | [diff] [blame] | 644 | |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 645 | ENTRY(xen_hypervisor_callback) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 646 | pushl $-1 /* orig_ax = -1 => not a system call */ |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 647 | SAVE_ALL |
| 648 | TRACE_IRQS_OFF |
Jeremy Fitzhardinge | 9ec2b80 | 2007-07-17 18:37:07 -0700 | [diff] [blame] | 649 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 650 | /* |
| 651 | * Check to see if we got the event in the critical |
| 652 | * region in xen_iret_direct, after we've reenabled |
| 653 | * events and checked for pending events. This simulates |
| 654 | * iret instruction's behaviour where it delivers a |
| 655 | * pending interrupt when enabling interrupts: |
| 656 | */ |
| 657 | movl PT_EIP(%esp), %eax |
| 658 | cmpl $xen_iret_start_crit, %eax |
| 659 | jb 1f |
| 660 | cmpl $xen_iret_end_crit, %eax |
| 661 | jae 1f |
Jeremy Fitzhardinge | 9ec2b80 | 2007-07-17 18:37:07 -0700 | [diff] [blame] | 662 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 663 | jmp xen_iret_crit_fixup |
Jeremy Fitzhardinge | 9ec2b80 | 2007-07-17 18:37:07 -0700 | [diff] [blame] | 664 | |
Jeremy Fitzhardinge | e2a81ba | 2008-03-17 16:37:17 -0700 | [diff] [blame] | 665 | ENTRY(xen_do_upcall) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 666 | 1: mov %esp, %eax |
| 667 | call xen_evtchn_do_upcall |
David Vrabel | fdfd811 | 2015-02-19 15:23:17 +0000 | [diff] [blame] | 668 | #ifndef CONFIG_PREEMPT |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 669 | call xen_maybe_preempt_hcall |
David Vrabel | fdfd811 | 2015-02-19 15:23:17 +0000 | [diff] [blame] | 670 | #endif |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 671 | jmp ret_from_intr |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 672 | ENDPROC(xen_hypervisor_callback) |
| 673 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 674 | /* |
| 675 | * Hypervisor uses this for application faults while it executes. |
| 676 | * We get here for two reasons: |
| 677 | * 1. Fault while reloading DS, ES, FS or GS |
| 678 | * 2. Fault while executing IRET |
| 679 | * Category 1 we fix up by reattempting the load, and zeroing the segment |
| 680 | * register if the load fails. |
| 681 | * Category 2 we fix up by jumping to do_iret_error. We cannot use the |
| 682 | * normal Linux return path in this case because if we use the IRET hypercall |
| 683 | * to pop the stack frame we end up in an infinite loop of failsafe callbacks. |
| 684 | * We distinguish between categories by maintaining a status value in EAX. |
| 685 | */ |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 686 | ENTRY(xen_failsafe_callback) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 687 | pushl %eax |
| 688 | movl $1, %eax |
| 689 | 1: mov 4(%esp), %ds |
| 690 | 2: mov 8(%esp), %es |
| 691 | 3: mov 12(%esp), %fs |
| 692 | 4: mov 16(%esp), %gs |
David Vrabel | a349e23d1 | 2012-10-19 17:29:07 +0100 | [diff] [blame] | 693 | /* EAX == 0 => Category 1 (Bad segment) |
| 694 | EAX != 0 => Category 2 (Bad IRET) */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 695 | testl %eax, %eax |
| 696 | popl %eax |
| 697 | lea 16(%esp), %esp |
| 698 | jz 5f |
| 699 | jmp iret_exc |
| 700 | 5: pushl $-1 /* orig_ax = -1 => not a system call */ |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 701 | SAVE_ALL |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 702 | jmp ret_from_exception |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 703 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 704 | .section .fixup, "ax" |
| 705 | 6: xorl %eax, %eax |
| 706 | movl %eax, 4(%esp) |
| 707 | jmp 1b |
| 708 | 7: xorl %eax, %eax |
| 709 | movl %eax, 8(%esp) |
| 710 | jmp 2b |
| 711 | 8: xorl %eax, %eax |
| 712 | movl %eax, 12(%esp) |
| 713 | jmp 3b |
| 714 | 9: xorl %eax, %eax |
| 715 | movl %eax, 16(%esp) |
| 716 | jmp 4b |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 717 | .previous |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 718 | _ASM_EXTABLE(1b, 6b) |
| 719 | _ASM_EXTABLE(2b, 7b) |
| 720 | _ASM_EXTABLE(3b, 8b) |
| 721 | _ASM_EXTABLE(4b, 9b) |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 722 | ENDPROC(xen_failsafe_callback) |
| 723 | |
K. Y. Srinivasan | bc2b033 | 2013-02-03 17:22:39 -0800 | [diff] [blame] | 724 | BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, |
Sheng Yang | 38e20b0 | 2010-05-14 12:40:51 +0100 | [diff] [blame] | 725 | xen_evtchn_do_upcall) |
| 726 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 727 | #endif /* CONFIG_XEN */ |
Jeremy Fitzhardinge | 5ead97c | 2007-07-17 18:37:04 -0700 | [diff] [blame] | 728 | |
K. Y. Srinivasan | bc2b033 | 2013-02-03 17:22:39 -0800 | [diff] [blame] | 729 | #if IS_ENABLED(CONFIG_HYPERV) |
| 730 | |
| 731 | BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, |
| 732 | hyperv_vector_handler) |
| 733 | |
| 734 | #endif /* CONFIG_HYPERV */ |
| 735 | |
Steven Rostedt | 606576c | 2008-10-06 19:06:12 -0400 | [diff] [blame] | 736 | #ifdef CONFIG_FUNCTION_TRACER |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 737 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 738 | |
| 739 | ENTRY(mcount) |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 740 | ret |
| 741 | END(mcount) |
| 742 | |
| 743 | ENTRY(ftrace_caller) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 744 | pushl %eax |
| 745 | pushl %ecx |
| 746 | pushl %edx |
| 747 | pushl $0 /* Pass NULL as regs pointer */ |
| 748 | movl 4*4(%esp), %eax |
| 749 | movl 0x4(%ebp), %edx |
| 750 | movl function_trace_op, %ecx |
| 751 | subl $MCOUNT_INSN_SIZE, %eax |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 752 | |
| 753 | .globl ftrace_call |
| 754 | ftrace_call: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 755 | call ftrace_stub |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 756 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 757 | addl $4, %esp /* skip NULL pointer */ |
| 758 | popl %edx |
| 759 | popl %ecx |
| 760 | popl %eax |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 761 | ftrace_ret: |
Steven Rostedt | 5a45cfe | 2008-11-26 00:16:24 -0500 | [diff] [blame] | 762 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 763 | .globl ftrace_graph_call |
| 764 | ftrace_graph_call: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 765 | jmp ftrace_stub |
Steven Rostedt | 5a45cfe | 2008-11-26 00:16:24 -0500 | [diff] [blame] | 766 | #endif |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 767 | |
| 768 | .globl ftrace_stub |
| 769 | ftrace_stub: |
| 770 | ret |
| 771 | END(ftrace_caller) |
| 772 | |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 773 | ENTRY(ftrace_regs_caller) |
| 774 | pushf /* push flags before compare (in cs location) */ |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 775 | |
| 776 | /* |
| 777 | * i386 does not save SS and ESP when coming from kernel. |
| 778 | * Instead, to get sp, ®s->sp is used (see ptrace.h). |
| 779 | * Unfortunately, that means eflags must be at the same location |
| 780 | * as the current return ip is. We move the return ip into the |
| 781 | * ip location, and move flags into the return ip location. |
| 782 | */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 783 | pushl 4(%esp) /* save return ip into ip slot */ |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 784 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 785 | pushl $0 /* Load 0 into orig_ax */ |
| 786 | pushl %gs |
| 787 | pushl %fs |
| 788 | pushl %es |
| 789 | pushl %ds |
| 790 | pushl %eax |
| 791 | pushl %ebp |
| 792 | pushl %edi |
| 793 | pushl %esi |
| 794 | pushl %edx |
| 795 | pushl %ecx |
| 796 | pushl %ebx |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 797 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 798 | movl 13*4(%esp), %eax /* Get the saved flags */ |
| 799 | movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ |
| 800 | /* clobbering return ip */ |
| 801 | movl $__KERNEL_CS, 13*4(%esp) |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 802 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 803 | movl 12*4(%esp), %eax /* Load ip (1st parameter) */ |
| 804 | subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ |
| 805 | movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ |
| 806 | movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ |
| 807 | pushl %esp /* Save pt_regs as 4th parameter */ |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 808 | |
| 809 | GLOBAL(ftrace_regs_call) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 810 | call ftrace_stub |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 811 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 812 | addl $4, %esp /* Skip pt_regs */ |
| 813 | movl 14*4(%esp), %eax /* Move flags back into cs */ |
| 814 | movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ |
| 815 | movl 12*4(%esp), %eax /* Get return ip from regs->ip */ |
| 816 | movl %eax, 14*4(%esp) /* Put return ip back for ret */ |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 817 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 818 | popl %ebx |
| 819 | popl %ecx |
| 820 | popl %edx |
| 821 | popl %esi |
| 822 | popl %edi |
| 823 | popl %ebp |
| 824 | popl %eax |
| 825 | popl %ds |
| 826 | popl %es |
| 827 | popl %fs |
| 828 | popl %gs |
| 829 | addl $8, %esp /* Skip orig_ax and ip */ |
| 830 | popf /* Pop flags at end (no addl to corrupt flags) */ |
| 831 | jmp ftrace_ret |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 832 | |
Steven Rostedt | 4de7239 | 2012-06-05 20:00:11 -0400 | [diff] [blame] | 833 | popf |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 834 | jmp ftrace_stub |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 835 | #else /* ! CONFIG_DYNAMIC_FTRACE */ |
| 836 | |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 837 | ENTRY(mcount) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 838 | cmpl $__PAGE_OFFSET, %esp |
| 839 | jb ftrace_stub /* Paging not enabled yet? */ |
H. Peter Anvin | af058ab | 2013-08-30 17:29:29 -0700 | [diff] [blame] | 840 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 841 | cmpl $ftrace_stub, ftrace_trace_function |
| 842 | jnz trace |
Frederic Weisbecker | fb52607 | 2008-11-25 21:07:04 +0100 | [diff] [blame] | 843 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 844 | cmpl $ftrace_stub, ftrace_graph_return |
| 845 | jnz ftrace_graph_caller |
Steven Rostedt | e49dc19 | 2008-12-02 23:50:05 -0500 | [diff] [blame] | 846 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 847 | cmpl $ftrace_graph_entry_stub, ftrace_graph_entry |
| 848 | jnz ftrace_graph_caller |
Frederic Weisbecker | caf4b32 | 2008-11-11 07:03:45 +0100 | [diff] [blame] | 849 | #endif |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 850 | .globl ftrace_stub |
| 851 | ftrace_stub: |
| 852 | ret |
| 853 | |
| 854 | /* taken from glibc */ |
| 855 | trace: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 856 | pushl %eax |
| 857 | pushl %ecx |
| 858 | pushl %edx |
| 859 | movl 0xc(%esp), %eax |
| 860 | movl 0x4(%ebp), %edx |
| 861 | subl $MCOUNT_INSN_SIZE, %eax |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 862 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 863 | call *ftrace_trace_function |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 864 | |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 865 | popl %edx |
| 866 | popl %ecx |
| 867 | popl %eax |
| 868 | jmp ftrace_stub |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 869 | END(mcount) |
Steven Rostedt | d61f82d | 2008-05-12 21:20:43 +0200 | [diff] [blame] | 870 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
Steven Rostedt | 606576c | 2008-10-06 19:06:12 -0400 | [diff] [blame] | 871 | #endif /* CONFIG_FUNCTION_TRACER */ |
Arnaldo Carvalho de Melo | 16444a8 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 872 | |
Frederic Weisbecker | fb52607 | 2008-11-25 21:07:04 +0100 | [diff] [blame] | 873 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 874 | ENTRY(ftrace_graph_caller) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 875 | pushl %eax |
| 876 | pushl %ecx |
| 877 | pushl %edx |
| 878 | movl 0xc(%esp), %eax |
| 879 | lea 0x4(%ebp), %edx |
| 880 | movl (%ebp), %ecx |
| 881 | subl $MCOUNT_INSN_SIZE, %eax |
| 882 | call prepare_ftrace_return |
| 883 | popl %edx |
| 884 | popl %ecx |
| 885 | popl %eax |
Frederic Weisbecker | e7d3737 | 2008-11-16 06:02:06 +0100 | [diff] [blame] | 886 | ret |
Frederic Weisbecker | fb52607 | 2008-11-25 21:07:04 +0100 | [diff] [blame] | 887 | END(ftrace_graph_caller) |
Frederic Weisbecker | caf4b32 | 2008-11-11 07:03:45 +0100 | [diff] [blame] | 888 | |
| 889 | .globl return_to_handler |
| 890 | return_to_handler: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 891 | pushl %eax |
| 892 | pushl %edx |
| 893 | movl %ebp, %eax |
| 894 | call ftrace_return_to_handler |
| 895 | movl %eax, %ecx |
| 896 | popl %edx |
| 897 | popl %eax |
| 898 | jmp *%ecx |
Frederic Weisbecker | e7d3737 | 2008-11-16 06:02:06 +0100 | [diff] [blame] | 899 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 900 | |
Seiji Aguchi | 25c74b1 | 2013-10-30 16:37:00 -0400 | [diff] [blame] | 901 | #ifdef CONFIG_TRACING |
| 902 | ENTRY(trace_page_fault) |
Seiji Aguchi | 25c74b1 | 2013-10-30 16:37:00 -0400 | [diff] [blame] | 903 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 904 | pushl $trace_do_page_fault |
| 905 | jmp error_code |
Seiji Aguchi | 25c74b1 | 2013-10-30 16:37:00 -0400 | [diff] [blame] | 906 | END(trace_page_fault) |
| 907 | #endif |
| 908 | |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 909 | ENTRY(page_fault) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 910 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 911 | pushl $do_page_fault |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 912 | ALIGN |
| 913 | error_code: |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 914 | /* the function address is in %gs's slot on the stack */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 915 | pushl %fs |
| 916 | pushl %es |
| 917 | pushl %ds |
| 918 | pushl %eax |
| 919 | pushl %ebp |
| 920 | pushl %edi |
| 921 | pushl %esi |
| 922 | pushl %edx |
| 923 | pushl %ecx |
| 924 | pushl %ebx |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 925 | cld |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 926 | movl $(__KERNEL_PERCPU), %ecx |
| 927 | movl %ecx, %fs |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 928 | UNWIND_ESPFIX_STACK |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 929 | GS_TO_REG %ecx |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 930 | movl PT_GS(%esp), %edi # get the function address |
| 931 | movl PT_ORIG_EAX(%esp), %edx # get the error code |
| 932 | movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart |
Tejun Heo | ccbeed3 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 933 | REG_TO_PTGS %ecx |
| 934 | SET_KERNEL_GS %ecx |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 935 | movl $(__USER_DS), %ecx |
| 936 | movl %ecx, %ds |
| 937 | movl %ecx, %es |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 938 | TRACE_IRQS_OFF |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 939 | movl %esp, %eax # pt_regs pointer |
| 940 | call *%edi |
| 941 | jmp ret_from_exception |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 942 | END(page_fault) |
| 943 | |
| 944 | /* |
| 945 | * Debug traps and NMI can happen at the one SYSENTER instruction |
| 946 | * that sets up the real kernel stack. Check here, since we can't |
| 947 | * allow the wrong stack to be used. |
| 948 | * |
| 949 | * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have |
| 950 | * already pushed 3 words if it hits on the sysenter instruction: |
| 951 | * eflags, cs and eip. |
| 952 | * |
| 953 | * We just load the right stack, and push the three (known) values |
| 954 | * by hand onto the new stack - while updating the return eip past |
| 955 | * the instruction that would have done it for sysenter. |
| 956 | */ |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 957 | .macro FIX_STACK offset ok label |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 958 | cmpw $__KERNEL_CS, 4(%esp) |
| 959 | jne \ok |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 960 | \label: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 961 | movl TSS_sysenter_sp0 + \offset(%esp), %esp |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 962 | pushfl |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 963 | pushl $__KERNEL_CS |
| 964 | pushl $sysenter_past_esp |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 965 | .endm |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 966 | |
| 967 | ENTRY(debug) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 968 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 969 | cmpl $entry_SYSENTER_32, (%esp) |
| 970 | jne debug_stack_correct |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 971 | FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 972 | debug_stack_correct: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 973 | pushl $-1 # mark this as an int |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 974 | SAVE_ALL |
| 975 | TRACE_IRQS_OFF |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 976 | xorl %edx, %edx # error code 0 |
| 977 | movl %esp, %eax # pt_regs pointer |
| 978 | call do_debug |
| 979 | jmp ret_from_exception |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 980 | END(debug) |
| 981 | |
| 982 | /* |
| 983 | * NMI is doubly nasty. It can happen _while_ we're handling |
| 984 | * a debug fault, and the debug fault hasn't yet been able to |
| 985 | * clear up the stack. So we first check whether we got an |
| 986 | * NMI on the sysenter entry path, but after that we need to |
| 987 | * check whether we got an NMI on the debug path where the debug |
| 988 | * fault happened on the sysenter path. |
| 989 | */ |
| 990 | ENTRY(nmi) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 991 | ASM_CLAC |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 992 | #ifdef CONFIG_X86_ESPFIX32 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 993 | pushl %eax |
| 994 | movl %ss, %eax |
| 995 | cmpw $__ESPFIX_SS, %ax |
| 996 | popl %eax |
| 997 | je nmi_espfix_stack |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 998 | #endif |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 999 | cmpl $entry_SYSENTER_32, (%esp) |
| 1000 | je nmi_stack_fixup |
| 1001 | pushl %eax |
| 1002 | movl %esp, %eax |
| 1003 | /* |
| 1004 | * Do not access memory above the end of our stack page, |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1005 | * it might not exist. |
| 1006 | */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1007 | andl $(THREAD_SIZE-1), %eax |
| 1008 | cmpl $(THREAD_SIZE-20), %eax |
| 1009 | popl %eax |
| 1010 | jae nmi_stack_correct |
| 1011 | cmpl $entry_SYSENTER_32, 12(%esp) |
| 1012 | je nmi_debug_stack_check |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1013 | nmi_stack_correct: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1014 | pushl %eax |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1015 | SAVE_ALL |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1016 | xorl %edx, %edx # zero error code |
| 1017 | movl %esp, %eax # pt_regs pointer |
| 1018 | call do_nmi |
| 1019 | jmp restore_all_notrace |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1020 | |
| 1021 | nmi_stack_fixup: |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 1022 | FIX_STACK 12, nmi_stack_correct, 1 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1023 | jmp nmi_stack_correct |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1024 | |
| 1025 | nmi_debug_stack_check: |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1026 | cmpw $__KERNEL_CS, 16(%esp) |
| 1027 | jne nmi_stack_correct |
| 1028 | cmpl $debug, (%esp) |
| 1029 | jb nmi_stack_correct |
| 1030 | cmpl $debug_esp_fix_insn, (%esp) |
| 1031 | ja nmi_stack_correct |
Tejun Heo | f0d9611 | 2009-02-09 22:17:40 +0900 | [diff] [blame] | 1032 | FIX_STACK 24, nmi_stack_correct, 1 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1033 | jmp nmi_stack_correct |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1034 | |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 1035 | #ifdef CONFIG_X86_ESPFIX32 |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1036 | nmi_espfix_stack: |
Ingo Molnar | 131484c | 2015-05-28 12:21:47 +0200 | [diff] [blame] | 1037 | /* |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1038 | * create the pointer to lss back |
| 1039 | */ |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1040 | pushl %ss |
| 1041 | pushl %esp |
| 1042 | addl $4, (%esp) |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1043 | /* copy the iret frame of 12 bytes */ |
| 1044 | .rept 3 |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1045 | pushl 16(%esp) |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1046 | .endr |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1047 | pushl %eax |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1048 | SAVE_ALL |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1049 | FIXUP_ESPFIX_STACK # %eax == %esp |
| 1050 | xorl %edx, %edx # zero error code |
| 1051 | call do_nmi |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1052 | RESTORE_REGS |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1053 | lss 12+4(%esp), %esp # back to espfix stack |
| 1054 | jmp irq_return |
H. Peter Anvin | 34273f4 | 2014-05-04 10:36:22 -0700 | [diff] [blame] | 1055 | #endif |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1056 | END(nmi) |
| 1057 | |
| 1058 | ENTRY(int3) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 1059 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1060 | pushl $-1 # mark this as an int |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1061 | SAVE_ALL |
| 1062 | TRACE_IRQS_OFF |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1063 | xorl %edx, %edx # zero error code |
| 1064 | movl %esp, %eax # pt_regs pointer |
| 1065 | call do_int3 |
| 1066 | jmp ret_from_exception |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1067 | END(int3) |
| 1068 | |
| 1069 | ENTRY(general_protection) |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1070 | pushl $do_general_protection |
| 1071 | jmp error_code |
Alexander van Heukelum | d211af0 | 2008-11-24 15:38:45 +0100 | [diff] [blame] | 1072 | END(general_protection) |
| 1073 | |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 1074 | #ifdef CONFIG_KVM_GUEST |
| 1075 | ENTRY(async_page_fault) |
H. Peter Anvin | e59d1b0 | 2012-09-21 13:58:10 -0700 | [diff] [blame] | 1076 | ASM_CLAC |
Ingo Molnar | a49976d | 2015-06-08 09:49:11 +0200 | [diff] [blame] | 1077 | pushl $do_async_page_fault |
| 1078 | jmp error_code |
Sedat Dilek | 2ae9d29 | 2011-03-08 22:39:24 +0100 | [diff] [blame] | 1079 | END(async_page_fault) |
Gleb Natapov | 631bc48 | 2010-10-14 11:22:52 +0200 | [diff] [blame] | 1080 | #endif |