Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1994 Linus Torvalds |
| 3 | * |
| 4 | * Pentium III FXSR, SSE support |
| 5 | * General FPU state handling cleanups |
| 6 | * Gareth Hughes <gareth@valinux.com>, May 2000 |
| 7 | * x86-64 work by Andi Kleen 2002 |
| 8 | */ |
| 9 | |
| 10 | #ifndef _ASM_X86_I387_H |
| 11 | #define _ASM_X86_I387_H |
| 12 | |
| 13 | #include <linux/sched.h> |
| 14 | #include <linux/kernel_stat.h> |
| 15 | #include <linux/regset.h> |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 16 | #include <linux/hardirq.h> |
H. Peter Anvin | 92c37fa | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 17 | #include <asm/asm.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 18 | #include <asm/processor.h> |
| 19 | #include <asm/sigcontext.h> |
| 20 | #include <asm/user.h> |
| 21 | #include <asm/uaccess.h> |
| 22 | |
| 23 | extern void fpu_init(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 24 | extern void mxcsr_feature_mask_init(void); |
Suresh Siddha | aa283f4 | 2008-03-10 15:28:05 -0700 | [diff] [blame] | 25 | extern int init_fpu(struct task_struct *child); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 26 | extern asmlinkage void math_state_restore(void); |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 27 | extern void init_thread_xstate(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 28 | |
| 29 | extern user_regset_active_fn fpregs_active, xfpregs_active; |
| 30 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; |
| 31 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set; |
| 32 | |
| 33 | #ifdef CONFIG_IA32_EMULATION |
| 34 | struct _fpstate_ia32; |
| 35 | extern int save_i387_ia32(struct _fpstate_ia32 __user *buf); |
| 36 | extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf); |
| 37 | #endif |
| 38 | |
| 39 | #ifdef CONFIG_X86_64 |
| 40 | |
| 41 | /* Ignore delayed exceptions from user space */ |
| 42 | static inline void tolerant_fwait(void) |
| 43 | { |
| 44 | asm volatile("1: fwait\n" |
| 45 | "2:\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 46 | _ASM_EXTABLE(1b, 2b)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 47 | } |
| 48 | |
| 49 | static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) |
| 50 | { |
| 51 | int err; |
| 52 | |
| 53 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" |
| 54 | "2:\n" |
| 55 | ".section .fixup,\"ax\"\n" |
| 56 | "3: movl $-1,%[err]\n" |
| 57 | " jmp 2b\n" |
| 58 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 59 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 60 | : [err] "=r" (err) |
| 61 | #if 0 /* See comment in __save_init_fpu() below. */ |
| 62 | : [fx] "r" (fx), "m" (*fx), "0" (0)); |
| 63 | #else |
| 64 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); |
| 65 | #endif |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 66 | return err; |
| 67 | } |
| 68 | |
| 69 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
| 70 | |
| 71 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 72 | is pending. Clear the x87 state here by setting it to fixed |
| 73 | values. The kernel data segment can be sometimes 0 and sometimes |
| 74 | new user value. Both should be ok. |
| 75 | Use the PDA as safe address because it should be already in L1. */ |
| 76 | static inline void clear_fpu_state(struct i387_fxsave_struct *fx) |
| 77 | { |
| 78 | if (unlikely(fx->swd & X87_FSW_ES)) |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 79 | asm volatile("fnclex"); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 80 | alternative_input(ASM_NOP8 ASM_NOP2, |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 81 | " emms\n" /* clear stack tags */ |
| 82 | " fildl %%gs:0", /* load to clear state */ |
| 83 | X86_FEATURE_FXSAVE_LEAK); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) |
| 87 | { |
| 88 | int err; |
| 89 | |
| 90 | asm volatile("1: rex64/fxsave (%[fx])\n\t" |
| 91 | "2:\n" |
| 92 | ".section .fixup,\"ax\"\n" |
| 93 | "3: movl $-1,%[err]\n" |
| 94 | " jmp 2b\n" |
| 95 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 96 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 97 | : [err] "=r" (err), "=m" (*fx) |
| 98 | #if 0 /* See comment in __fxsave_clear() below. */ |
| 99 | : [fx] "r" (fx), "0" (0)); |
| 100 | #else |
| 101 | : [fx] "cdaSDb" (fx), "0" (0)); |
| 102 | #endif |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 103 | if (unlikely(err) && |
| 104 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 105 | err = -EFAULT; |
| 106 | /* No need to clear here because the caller clears USED_MATH */ |
| 107 | return err; |
| 108 | } |
| 109 | |
| 110 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 111 | { |
| 112 | /* Using "rex64; fxsave %0" is broken because, if the memory operand |
| 113 | uses any extended registers for addressing, a second REX prefix |
| 114 | will be generated (to the assembler, rex64 followed by semicolon |
| 115 | is a separate instruction), and hence the 64-bitness is lost. */ |
| 116 | #if 0 |
| 117 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported |
| 118 | starting with gas 2.16. */ |
| 119 | __asm__ __volatile__("fxsaveq %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 120 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 121 | #elif 0 |
| 122 | /* Using, as a workaround, the properly prefixed form below isn't |
| 123 | accepted by any binutils version so far released, complaining that |
| 124 | the same type of prefix is used twice if an extended register is |
| 125 | needed for addressing (fix submitted to mainline 2005-11-21). */ |
| 126 | __asm__ __volatile__("rex64/fxsave %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 127 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 128 | #else |
| 129 | /* This, however, we can work around by forcing the compiler to select |
| 130 | an addressing mode that doesn't require extended registers. */ |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 131 | __asm__ __volatile__("rex64/fxsave (%1)" |
| 132 | : "=m" (tsk->thread.xstate->fxsave) |
| 133 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 134 | #endif |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 135 | clear_fpu_state(&tsk->thread.xstate->fxsave); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 136 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 137 | } |
| 138 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 139 | #else /* CONFIG_X86_32 */ |
| 140 | |
Suresh Siddha | e8a496a | 2008-05-23 16:26:37 -0700 | [diff] [blame] | 141 | extern void finit(void); |
| 142 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 143 | static inline void tolerant_fwait(void) |
| 144 | { |
| 145 | asm volatile("fnclex ; fwait"); |
| 146 | } |
| 147 | |
| 148 | static inline void restore_fpu(struct task_struct *tsk) |
| 149 | { |
| 150 | /* |
| 151 | * The "nop" is needed to make the instructions the same |
| 152 | * length. |
| 153 | */ |
| 154 | alternative_input( |
| 155 | "nop ; frstor %1", |
| 156 | "fxrstor %1", |
| 157 | X86_FEATURE_FXSR, |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 158 | "m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 159 | } |
| 160 | |
| 161 | /* We need a safe address that is cheap to find and that is already |
| 162 | in L1 during context switch. The best choices are unfortunately |
| 163 | different for UP and SMP */ |
| 164 | #ifdef CONFIG_SMP |
| 165 | #define safe_address (__per_cpu_offset[0]) |
| 166 | #else |
| 167 | #define safe_address (kstat_cpu(0).cpustat.user) |
| 168 | #endif |
| 169 | |
| 170 | /* |
| 171 | * These must be called with preempt disabled |
| 172 | */ |
| 173 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 174 | { |
| 175 | /* Use more nops than strictly needed in case the compiler |
| 176 | varies code */ |
| 177 | alternative_input( |
| 178 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, |
| 179 | "fxsave %[fx]\n" |
| 180 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", |
| 181 | X86_FEATURE_FXSR, |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 182 | [fx] "m" (tsk->thread.xstate->fxsave), |
| 183 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 184 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 185 | is pending. Clear the x87 state here by setting it to fixed |
| 186 | values. safe_address is a random variable that should be in L1 */ |
| 187 | alternative_input( |
| 188 | GENERIC_NOP8 GENERIC_NOP2, |
| 189 | "emms\n\t" /* clear stack tags */ |
| 190 | "fildl %[addr]", /* set F?P to defined value */ |
| 191 | X86_FEATURE_FXSAVE_LEAK, |
| 192 | [addr] "m" (safe_address)); |
| 193 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 194 | } |
| 195 | |
| 196 | /* |
| 197 | * Signal frame handlers... |
| 198 | */ |
| 199 | extern int save_i387(struct _fpstate __user *buf); |
| 200 | extern int restore_i387(struct _fpstate __user *buf); |
| 201 | |
| 202 | #endif /* CONFIG_X86_64 */ |
| 203 | |
| 204 | static inline void __unlazy_fpu(struct task_struct *tsk) |
| 205 | { |
| 206 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 207 | __save_init_fpu(tsk); |
| 208 | stts(); |
| 209 | } else |
| 210 | tsk->fpu_counter = 0; |
| 211 | } |
| 212 | |
| 213 | static inline void __clear_fpu(struct task_struct *tsk) |
| 214 | { |
| 215 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 216 | tolerant_fwait(); |
| 217 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 218 | stts(); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | static inline void kernel_fpu_begin(void) |
| 223 | { |
| 224 | struct thread_info *me = current_thread_info(); |
| 225 | preempt_disable(); |
| 226 | if (me->status & TS_USEDFPU) |
| 227 | __save_init_fpu(me->task); |
| 228 | else |
| 229 | clts(); |
| 230 | } |
| 231 | |
| 232 | static inline void kernel_fpu_end(void) |
| 233 | { |
| 234 | stts(); |
| 235 | preempt_enable(); |
| 236 | } |
| 237 | |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 238 | /* |
| 239 | * Some instructions like VIA's padlock instructions generate a spurious |
| 240 | * DNA fault but don't modify SSE registers. And these instructions |
| 241 | * get used from interrupt context aswell. To prevent these kernel instructions |
| 242 | * in interrupt context interact wrongly with other user/kernel fpu usage, we |
| 243 | * should use them only in the context of irq_ts_save/restore() |
| 244 | */ |
| 245 | static inline int irq_ts_save(void) |
| 246 | { |
| 247 | /* |
| 248 | * If we are in process context, we are ok to take a spurious DNA fault. |
| 249 | * Otherwise, doing clts() in process context require pre-emption to |
| 250 | * be disabled or some heavy lifting like kernel_fpu_begin() |
| 251 | */ |
| 252 | if (!in_interrupt()) |
| 253 | return 0; |
| 254 | |
| 255 | if (read_cr0() & X86_CR0_TS) { |
| 256 | clts(); |
| 257 | return 1; |
| 258 | } |
| 259 | |
| 260 | return 0; |
| 261 | } |
| 262 | |
| 263 | static inline void irq_ts_restore(int TS_state) |
| 264 | { |
| 265 | if (TS_state) |
| 266 | stts(); |
| 267 | } |
| 268 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 269 | #ifdef CONFIG_X86_64 |
| 270 | |
| 271 | static inline void save_init_fpu(struct task_struct *tsk) |
| 272 | { |
| 273 | __save_init_fpu(tsk); |
| 274 | stts(); |
| 275 | } |
| 276 | |
| 277 | #define unlazy_fpu __unlazy_fpu |
| 278 | #define clear_fpu __clear_fpu |
| 279 | |
| 280 | #else /* CONFIG_X86_32 */ |
| 281 | |
| 282 | /* |
| 283 | * These disable preemption on their own and are safe |
| 284 | */ |
| 285 | static inline void save_init_fpu(struct task_struct *tsk) |
| 286 | { |
| 287 | preempt_disable(); |
| 288 | __save_init_fpu(tsk); |
| 289 | stts(); |
| 290 | preempt_enable(); |
| 291 | } |
| 292 | |
| 293 | static inline void unlazy_fpu(struct task_struct *tsk) |
| 294 | { |
| 295 | preempt_disable(); |
| 296 | __unlazy_fpu(tsk); |
| 297 | preempt_enable(); |
| 298 | } |
| 299 | |
| 300 | static inline void clear_fpu(struct task_struct *tsk) |
| 301 | { |
| 302 | preempt_disable(); |
| 303 | __clear_fpu(tsk); |
| 304 | preempt_enable(); |
| 305 | } |
| 306 | |
| 307 | #endif /* CONFIG_X86_64 */ |
| 308 | |
| 309 | /* |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 310 | * i387 state interaction |
| 311 | */ |
| 312 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) |
| 313 | { |
| 314 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 315 | return tsk->thread.xstate->fxsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 316 | } else { |
Suresh Siddha | 1679f27 | 2008-04-16 10:27:53 +0200 | [diff] [blame] | 317 | return (unsigned short)tsk->thread.xstate->fsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 318 | } |
| 319 | } |
| 320 | |
| 321 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) |
| 322 | { |
| 323 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 324 | return tsk->thread.xstate->fxsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 325 | } else { |
Suresh Siddha | 1679f27 | 2008-04-16 10:27:53 +0200 | [diff] [blame] | 326 | return (unsigned short)tsk->thread.xstate->fsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 327 | } |
| 328 | } |
| 329 | |
| 330 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) |
| 331 | { |
| 332 | if (cpu_has_xmm) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 333 | return tsk->thread.xstate->fxsave.mxcsr; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 334 | } else { |
| 335 | return MXCSR_DEFAULT; |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | #endif /* _ASM_X86_I387_H */ |