Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 1 | #ifndef _X86_IRQFLAGS_H_ |
| 2 | #define _X86_IRQFLAGS_H_ |
| 3 | |
| 4 | #include <asm/processor-flags.h> |
| 5 | |
| 6 | #ifndef __ASSEMBLY__ |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 7 | |
Thomas Gleixner | 2394f59 | 2019-02-18 23:04:01 +0100 | [diff] [blame] | 8 | #include <asm/nospec-branch.h> |
| 9 | |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 10 | /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ |
| 11 | #define __cpuidle __attribute__((__section__(".cpuidle.text"))) |
| 12 | |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 13 | /* |
| 14 | * Interrupt control: |
| 15 | */ |
| 16 | |
Nick Desaulniers | 329d815 | 2018-08-03 10:05:50 -0700 | [diff] [blame] | 17 | /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ |
| 18 | extern inline unsigned long native_save_fl(void); |
Nick Desaulniers | 1919f3f | 2018-06-21 09:23:24 -0700 | [diff] [blame] | 19 | extern inline unsigned long native_save_fl(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 20 | { |
| 21 | unsigned long flags; |
| 22 | |
H. Peter Anvin | f1f029c | 2009-08-03 16:33:40 -0700 | [diff] [blame] | 23 | /* |
H. Peter Anvin | ab94fcf | 2009-08-25 16:47:16 -0700 | [diff] [blame] | 24 | * "=rm" is safe here, because "pop" adjusts the stack before |
| 25 | * it evaluates its effective address -- this is part of the |
| 26 | * documented behavior of the "pop" instruction. |
H. Peter Anvin | f1f029c | 2009-08-03 16:33:40 -0700 | [diff] [blame] | 27 | */ |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 28 | asm volatile("# __raw_save_flags\n\t" |
| 29 | "pushf ; pop %0" |
H. Peter Anvin | ab94fcf | 2009-08-25 16:47:16 -0700 | [diff] [blame] | 30 | : "=rm" (flags) |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 31 | : /* no input */ |
| 32 | : "memory"); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 33 | |
| 34 | return flags; |
| 35 | } |
| 36 | |
Nick Desaulniers | c148246 | 2018-08-27 14:40:09 -0700 | [diff] [blame] | 37 | extern inline void native_restore_fl(unsigned long flags); |
| 38 | extern inline void native_restore_fl(unsigned long flags) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 39 | { |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 40 | asm volatile("push %0 ; popf" |
| 41 | : /* no output */ |
| 42 | :"g" (flags) |
| 43 | :"memory", "cc"); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 44 | } |
| 45 | |
| 46 | static inline void native_irq_disable(void) |
| 47 | { |
| 48 | asm volatile("cli": : :"memory"); |
| 49 | } |
| 50 | |
| 51 | static inline void native_irq_enable(void) |
| 52 | { |
| 53 | asm volatile("sti": : :"memory"); |
| 54 | } |
| 55 | |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 56 | static inline __cpuidle void native_safe_halt(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 57 | { |
Thomas Gleixner | 2394f59 | 2019-02-18 23:04:01 +0100 | [diff] [blame] | 58 | mds_idle_clear_cpu_buffers(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 59 | asm volatile("sti; hlt": : :"memory"); |
| 60 | } |
| 61 | |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 62 | static inline __cpuidle void native_halt(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 63 | { |
Thomas Gleixner | 2394f59 | 2019-02-18 23:04:01 +0100 | [diff] [blame] | 64 | mds_idle_clear_cpu_buffers(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 65 | asm volatile("hlt": : :"memory"); |
| 66 | } |
| 67 | |
| 68 | #endif |
| 69 | |
| 70 | #ifdef CONFIG_PARAVIRT |
| 71 | #include <asm/paravirt.h> |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 72 | #else |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 73 | #ifndef __ASSEMBLY__ |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 74 | #include <linux/types.h> |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 75 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 76 | static inline notrace unsigned long arch_local_save_flags(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 77 | { |
| 78 | return native_save_fl(); |
| 79 | } |
| 80 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 81 | static inline notrace void arch_local_irq_restore(unsigned long flags) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 82 | { |
| 83 | native_restore_fl(flags); |
| 84 | } |
| 85 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 86 | static inline notrace void arch_local_irq_disable(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 87 | { |
| 88 | native_irq_disable(); |
| 89 | } |
| 90 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 91 | static inline notrace void arch_local_irq_enable(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 92 | { |
| 93 | native_irq_enable(); |
| 94 | } |
| 95 | |
| 96 | /* |
| 97 | * Used in the idle loop; sti takes one instruction cycle |
| 98 | * to complete: |
| 99 | */ |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 100 | static inline __cpuidle void arch_safe_halt(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 101 | { |
| 102 | native_safe_halt(); |
| 103 | } |
| 104 | |
| 105 | /* |
| 106 | * Used when interrupts are already enabled or to |
| 107 | * shutdown the processor: |
| 108 | */ |
Chris Metcalf | 6727ad9 | 2016-10-07 17:02:55 -0700 | [diff] [blame] | 109 | static inline __cpuidle void halt(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 110 | { |
| 111 | native_halt(); |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * For spinlocks, etc: |
| 116 | */ |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 117 | static inline notrace unsigned long arch_local_irq_save(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 118 | { |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 119 | unsigned long flags = arch_local_save_flags(); |
| 120 | arch_local_irq_disable(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 121 | return flags; |
| 122 | } |
| 123 | #else |
| 124 | |
| 125 | #define ENABLE_INTERRUPTS(x) sti |
| 126 | #define DISABLE_INTERRUPTS(x) cli |
| 127 | |
| 128 | #ifdef CONFIG_X86_64 |
Jeremy Fitzhardinge | df366e9 | 2008-06-27 12:04:03 -0700 | [diff] [blame] | 129 | #define SWAPGS swapgs |
| 130 | /* |
| 131 | * Currently paravirt can't handle swapgs nicely when we |
| 132 | * don't have a stack we can rely on (such as a user space |
| 133 | * stack). So we either find a way around these or just fault |
| 134 | * and emulate if a guest tries to call swapgs directly. |
| 135 | * |
| 136 | * Either way, this is a good way to document that we don't |
| 137 | * have a reliable stack. x86_64 only. |
| 138 | */ |
Jeremy Fitzhardinge | a00394f | 2008-06-25 00:19:30 -0400 | [diff] [blame] | 139 | #define SWAPGS_UNSAFE_STACK swapgs |
Jeremy Fitzhardinge | df366e9 | 2008-06-27 12:04:03 -0700 | [diff] [blame] | 140 | |
| 141 | #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ |
| 142 | |
Andy Lutomirski | 7209a75 | 2014-07-23 08:34:11 -0700 | [diff] [blame] | 143 | #define INTERRUPT_RETURN jmp native_iret |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 144 | #define USERGS_SYSRET64 \ |
| 145 | swapgs; \ |
| 146 | sysretq; |
| 147 | #define USERGS_SYSRET32 \ |
| 148 | swapgs; \ |
| 149 | sysretl |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 150 | |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 151 | #else |
| 152 | #define INTERRUPT_RETURN iret |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 153 | #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 154 | #define GET_CR0_INTO_EAX movl %cr0, %eax |
| 155 | #endif |
| 156 | |
| 157 | |
| 158 | #endif /* __ASSEMBLY__ */ |
| 159 | #endif /* CONFIG_PARAVIRT */ |
| 160 | |
| 161 | #ifndef __ASSEMBLY__ |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 162 | static inline int arch_irqs_disabled_flags(unsigned long flags) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 163 | { |
| 164 | return !(flags & X86_EFLAGS_IF); |
| 165 | } |
| 166 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 167 | static inline int arch_irqs_disabled(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 168 | { |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 169 | unsigned long flags = arch_local_save_flags(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 170 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 171 | return arch_irqs_disabled_flags(flags); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 172 | } |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 173 | #endif /* !__ASSEMBLY__ */ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 174 | |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 175 | #ifdef __ASSEMBLY__ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 176 | #ifdef CONFIG_TRACE_IRQFLAGS |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 177 | # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; |
| 178 | # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 179 | #else |
| 180 | # define TRACE_IRQS_ON |
| 181 | # define TRACE_IRQS_OFF |
| 182 | #endif |
| 183 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 184 | # ifdef CONFIG_X86_64 |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 185 | # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
| 186 | # define LOCKDEP_SYS_EXIT_IRQ \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 187 | TRACE_IRQS_ON; \ |
| 188 | sti; \ |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 189 | call lockdep_sys_exit_thunk; \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 190 | cli; \ |
| 191 | TRACE_IRQS_OFF; |
| 192 | # else |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 193 | # define LOCKDEP_SYS_EXIT \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 194 | pushl %eax; \ |
| 195 | pushl %ecx; \ |
| 196 | pushl %edx; \ |
| 197 | call lockdep_sys_exit; \ |
| 198 | popl %edx; \ |
| 199 | popl %ecx; \ |
| 200 | popl %eax; |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 201 | # define LOCKDEP_SYS_EXIT_IRQ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 202 | # endif |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 203 | #else |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 204 | # define LOCKDEP_SYS_EXIT |
| 205 | # define LOCKDEP_SYS_EXIT_IRQ |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 206 | #endif |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 207 | #endif /* __ASSEMBLY__ */ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 208 | |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 209 | #endif |