Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 1 | #ifndef _X86_IRQFLAGS_H_ |
| 2 | #define _X86_IRQFLAGS_H_ |
| 3 | |
| 4 | #include <asm/processor-flags.h> |
| 5 | |
| 6 | #ifndef __ASSEMBLY__ |
| 7 | /* |
| 8 | * Interrupt control: |
| 9 | */ |
| 10 | |
| 11 | static inline unsigned long native_save_fl(void) |
| 12 | { |
| 13 | unsigned long flags; |
| 14 | |
H. Peter Anvin | f1f029c | 2009-08-03 16:33:40 -0700 | [diff] [blame] | 15 | /* |
H. Peter Anvin | ab94fcf | 2009-08-25 16:47:16 -0700 | [diff] [blame] | 16 | * "=rm" is safe here, because "pop" adjusts the stack before |
| 17 | * it evaluates its effective address -- this is part of the |
| 18 | * documented behavior of the "pop" instruction. |
H. Peter Anvin | f1f029c | 2009-08-03 16:33:40 -0700 | [diff] [blame] | 19 | */ |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 20 | asm volatile("# __raw_save_flags\n\t" |
| 21 | "pushf ; pop %0" |
H. Peter Anvin | ab94fcf | 2009-08-25 16:47:16 -0700 | [diff] [blame] | 22 | : "=rm" (flags) |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 23 | : /* no input */ |
| 24 | : "memory"); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 25 | |
| 26 | return flags; |
| 27 | } |
| 28 | |
| 29 | static inline void native_restore_fl(unsigned long flags) |
| 30 | { |
Joe Perches | cf7f719 | 2008-03-23 01:02:30 -0700 | [diff] [blame] | 31 | asm volatile("push %0 ; popf" |
| 32 | : /* no output */ |
| 33 | :"g" (flags) |
| 34 | :"memory", "cc"); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 35 | } |
| 36 | |
| 37 | static inline void native_irq_disable(void) |
| 38 | { |
| 39 | asm volatile("cli": : :"memory"); |
| 40 | } |
| 41 | |
| 42 | static inline void native_irq_enable(void) |
| 43 | { |
| 44 | asm volatile("sti": : :"memory"); |
| 45 | } |
| 46 | |
| 47 | static inline void native_safe_halt(void) |
| 48 | { |
| 49 | asm volatile("sti; hlt": : :"memory"); |
| 50 | } |
| 51 | |
| 52 | static inline void native_halt(void) |
| 53 | { |
| 54 | asm volatile("hlt": : :"memory"); |
| 55 | } |
| 56 | |
| 57 | #endif |
| 58 | |
| 59 | #ifdef CONFIG_PARAVIRT |
| 60 | #include <asm/paravirt.h> |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 61 | #else |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 62 | #ifndef __ASSEMBLY__ |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 63 | #include <linux/types.h> |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 64 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 65 | static inline notrace unsigned long arch_local_save_flags(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 66 | { |
| 67 | return native_save_fl(); |
| 68 | } |
| 69 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 70 | static inline notrace void arch_local_irq_restore(unsigned long flags) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 71 | { |
| 72 | native_restore_fl(flags); |
| 73 | } |
| 74 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 75 | static inline notrace void arch_local_irq_disable(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 76 | { |
| 77 | native_irq_disable(); |
| 78 | } |
| 79 | |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 80 | static inline notrace void arch_local_irq_enable(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 81 | { |
| 82 | native_irq_enable(); |
| 83 | } |
| 84 | |
| 85 | /* |
| 86 | * Used in the idle loop; sti takes one instruction cycle |
| 87 | * to complete: |
| 88 | */ |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 89 | static inline void arch_safe_halt(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 90 | { |
| 91 | native_safe_halt(); |
| 92 | } |
| 93 | |
| 94 | /* |
| 95 | * Used when interrupts are already enabled or to |
| 96 | * shutdown the processor: |
| 97 | */ |
| 98 | static inline void halt(void) |
| 99 | { |
| 100 | native_halt(); |
| 101 | } |
| 102 | |
| 103 | /* |
| 104 | * For spinlocks, etc: |
| 105 | */ |
Steven Rostedt | e08fbb7 | 2011-07-01 23:04:36 -0400 | [diff] [blame] | 106 | static inline notrace unsigned long arch_local_irq_save(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 107 | { |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 108 | unsigned long flags = arch_local_save_flags(); |
| 109 | arch_local_irq_disable(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 110 | return flags; |
| 111 | } |
| 112 | #else |
| 113 | |
| 114 | #define ENABLE_INTERRUPTS(x) sti |
| 115 | #define DISABLE_INTERRUPTS(x) cli |
| 116 | |
| 117 | #ifdef CONFIG_X86_64 |
Jeremy Fitzhardinge | df366e9 | 2008-06-27 12:04:03 -0700 | [diff] [blame] | 118 | #define SWAPGS swapgs |
| 119 | /* |
| 120 | * Currently paravirt can't handle swapgs nicely when we |
| 121 | * don't have a stack we can rely on (such as a user space |
| 122 | * stack). So we either find a way around these or just fault |
| 123 | * and emulate if a guest tries to call swapgs directly. |
| 124 | * |
| 125 | * Either way, this is a good way to document that we don't |
| 126 | * have a reliable stack. x86_64 only. |
| 127 | */ |
Jeremy Fitzhardinge | a00394f | 2008-06-25 00:19:30 -0400 | [diff] [blame] | 128 | #define SWAPGS_UNSAFE_STACK swapgs |
Jeremy Fitzhardinge | df366e9 | 2008-06-27 12:04:03 -0700 | [diff] [blame] | 129 | |
| 130 | #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ |
| 131 | |
Andy Lutomirski | 7209a75 | 2014-07-23 08:34:11 -0700 | [diff] [blame] | 132 | #define INTERRUPT_RETURN jmp native_iret |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 133 | #define USERGS_SYSRET64 \ |
| 134 | swapgs; \ |
| 135 | sysretq; |
| 136 | #define USERGS_SYSRET32 \ |
| 137 | swapgs; \ |
| 138 | sysretl |
Jeremy Fitzhardinge | 2be2998 | 2008-06-25 00:19:28 -0400 | [diff] [blame] | 139 | |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 140 | #else |
| 141 | #define INTERRUPT_RETURN iret |
Jeremy Fitzhardinge | d75cd22f | 2008-06-25 00:19:26 -0400 | [diff] [blame] | 142 | #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 143 | #define GET_CR0_INTO_EAX movl %cr0, %eax |
| 144 | #endif |
| 145 | |
| 146 | |
| 147 | #endif /* __ASSEMBLY__ */ |
| 148 | #endif /* CONFIG_PARAVIRT */ |
| 149 | |
| 150 | #ifndef __ASSEMBLY__ |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 151 | static inline int arch_irqs_disabled_flags(unsigned long flags) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 152 | { |
| 153 | return !(flags & X86_EFLAGS_IF); |
| 154 | } |
| 155 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 156 | static inline int arch_irqs_disabled(void) |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 157 | { |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 158 | unsigned long flags = arch_local_save_flags(); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 159 | |
David Howells | df9ee29 | 2010-10-07 14:08:55 +0100 | [diff] [blame] | 160 | return arch_irqs_disabled_flags(flags); |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 161 | } |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 162 | #endif /* !__ASSEMBLY__ */ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 163 | |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 164 | #ifdef __ASSEMBLY__ |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 165 | #ifdef CONFIG_TRACE_IRQFLAGS |
Steven Rostedt | 81d68a9 | 2008-05-12 21:20:42 +0200 | [diff] [blame] | 166 | # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; |
| 167 | # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 168 | #else |
| 169 | # define TRACE_IRQS_ON |
| 170 | # define TRACE_IRQS_OFF |
| 171 | #endif |
| 172 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 173 | # ifdef CONFIG_X86_64 |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 174 | # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
| 175 | # define LOCKDEP_SYS_EXIT_IRQ \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 176 | TRACE_IRQS_ON; \ |
| 177 | sti; \ |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 178 | call lockdep_sys_exit_thunk; \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 179 | cli; \ |
| 180 | TRACE_IRQS_OFF; |
| 181 | # else |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 182 | # define LOCKDEP_SYS_EXIT \ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 183 | pushl %eax; \ |
| 184 | pushl %ecx; \ |
| 185 | pushl %edx; \ |
| 186 | call lockdep_sys_exit; \ |
| 187 | popl %edx; \ |
| 188 | popl %ecx; \ |
| 189 | popl %eax; |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 190 | # define LOCKDEP_SYS_EXIT_IRQ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 191 | # endif |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 192 | #else |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 193 | # define LOCKDEP_SYS_EXIT |
| 194 | # define LOCKDEP_SYS_EXIT_IRQ |
Denys Vlasenko | 7dc7cc0 | 2015-03-25 21:14:27 +0100 | [diff] [blame] | 195 | #endif |
Glauber de Oliveira Costa | 6abcd98 | 2008-01-30 13:30:33 +0100 | [diff] [blame] | 196 | #endif /* __ASSEMBLY__ */ |
Denys Vlasenko | 40e2ec6 | 2015-03-25 21:14:26 +0100 | [diff] [blame] | 197 | |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 198 | #endif |