Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1994 Linus Torvalds |
| 3 | * |
| 4 | * Pentium III FXSR, SSE support |
| 5 | * General FPU state handling cleanups |
| 6 | * Gareth Hughes <gareth@valinux.com>, May 2000 |
| 7 | * x86-64 work by Andi Kleen 2002 |
| 8 | */ |
| 9 | |
| 10 | #ifndef _ASM_X86_I387_H |
| 11 | #define _ASM_X86_I387_H |
| 12 | |
| 13 | #include <linux/sched.h> |
| 14 | #include <linux/kernel_stat.h> |
| 15 | #include <linux/regset.h> |
H. Peter Anvin | 92c37fa | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 16 | #include <asm/asm.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 17 | #include <asm/processor.h> |
| 18 | #include <asm/sigcontext.h> |
| 19 | #include <asm/user.h> |
| 20 | #include <asm/uaccess.h> |
| 21 | |
| 22 | extern void fpu_init(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 23 | extern void mxcsr_feature_mask_init(void); |
Suresh Siddha | aa283f4 | 2008-03-10 15:28:05 -0700 | [diff] [blame^] | 24 | extern int init_fpu(struct task_struct *child); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 25 | extern asmlinkage void math_state_restore(void); |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 26 | extern void init_thread_xstate(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 27 | |
| 28 | extern user_regset_active_fn fpregs_active, xfpregs_active; |
| 29 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; |
| 30 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set; |
| 31 | |
| 32 | #ifdef CONFIG_IA32_EMULATION |
| 33 | struct _fpstate_ia32; |
| 34 | extern int save_i387_ia32(struct _fpstate_ia32 __user *buf); |
| 35 | extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf); |
| 36 | #endif |
| 37 | |
| 38 | #ifdef CONFIG_X86_64 |
| 39 | |
| 40 | /* Ignore delayed exceptions from user space */ |
| 41 | static inline void tolerant_fwait(void) |
| 42 | { |
| 43 | asm volatile("1: fwait\n" |
| 44 | "2:\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 45 | _ASM_EXTABLE(1b, 2b)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 46 | } |
| 47 | |
| 48 | static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) |
| 49 | { |
| 50 | int err; |
| 51 | |
| 52 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" |
| 53 | "2:\n" |
| 54 | ".section .fixup,\"ax\"\n" |
| 55 | "3: movl $-1,%[err]\n" |
| 56 | " jmp 2b\n" |
| 57 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 58 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 59 | : [err] "=r" (err) |
| 60 | #if 0 /* See comment in __save_init_fpu() below. */ |
| 61 | : [fx] "r" (fx), "m" (*fx), "0" (0)); |
| 62 | #else |
| 63 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); |
| 64 | #endif |
| 65 | if (unlikely(err)) |
| 66 | init_fpu(current); |
| 67 | return err; |
| 68 | } |
| 69 | |
| 70 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
| 71 | |
| 72 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 73 | is pending. Clear the x87 state here by setting it to fixed |
| 74 | values. The kernel data segment can be sometimes 0 and sometimes |
| 75 | new user value. Both should be ok. |
| 76 | Use the PDA as safe address because it should be already in L1. */ |
| 77 | static inline void clear_fpu_state(struct i387_fxsave_struct *fx) |
| 78 | { |
| 79 | if (unlikely(fx->swd & X87_FSW_ES)) |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 80 | asm volatile("fnclex"); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 81 | alternative_input(ASM_NOP8 ASM_NOP2, |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 82 | " emms\n" /* clear stack tags */ |
| 83 | " fildl %%gs:0", /* load to clear state */ |
| 84 | X86_FEATURE_FXSAVE_LEAK); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 85 | } |
| 86 | |
| 87 | static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) |
| 88 | { |
| 89 | int err; |
| 90 | |
| 91 | asm volatile("1: rex64/fxsave (%[fx])\n\t" |
| 92 | "2:\n" |
| 93 | ".section .fixup,\"ax\"\n" |
| 94 | "3: movl $-1,%[err]\n" |
| 95 | " jmp 2b\n" |
| 96 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 97 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 98 | : [err] "=r" (err), "=m" (*fx) |
| 99 | #if 0 /* See comment in __fxsave_clear() below. */ |
| 100 | : [fx] "r" (fx), "0" (0)); |
| 101 | #else |
| 102 | : [fx] "cdaSDb" (fx), "0" (0)); |
| 103 | #endif |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 104 | if (unlikely(err) && |
| 105 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 106 | err = -EFAULT; |
| 107 | /* No need to clear here because the caller clears USED_MATH */ |
| 108 | return err; |
| 109 | } |
| 110 | |
| 111 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 112 | { |
| 113 | /* Using "rex64; fxsave %0" is broken because, if the memory operand |
| 114 | uses any extended registers for addressing, a second REX prefix |
| 115 | will be generated (to the assembler, rex64 followed by semicolon |
| 116 | is a separate instruction), and hence the 64-bitness is lost. */ |
| 117 | #if 0 |
| 118 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported |
| 119 | starting with gas 2.16. */ |
| 120 | __asm__ __volatile__("fxsaveq %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 121 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 122 | #elif 0 |
| 123 | /* Using, as a workaround, the properly prefixed form below isn't |
| 124 | accepted by any binutils version so far released, complaining that |
| 125 | the same type of prefix is used twice if an extended register is |
| 126 | needed for addressing (fix submitted to mainline 2005-11-21). */ |
| 127 | __asm__ __volatile__("rex64/fxsave %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 128 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 129 | #else |
| 130 | /* This, however, we can work around by forcing the compiler to select |
| 131 | an addressing mode that doesn't require extended registers. */ |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 132 | __asm__ __volatile__("rex64/fxsave (%1)" |
| 133 | : "=m" (tsk->thread.xstate->fxsave) |
| 134 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 135 | #endif |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 136 | clear_fpu_state(&tsk->thread.xstate->fxsave); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 137 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 138 | } |
| 139 | |
| 140 | /* |
| 141 | * Signal frame handlers. |
| 142 | */ |
| 143 | |
| 144 | static inline int save_i387(struct _fpstate __user *buf) |
| 145 | { |
| 146 | struct task_struct *tsk = current; |
| 147 | int err = 0; |
| 148 | |
| 149 | BUILD_BUG_ON(sizeof(struct user_i387_struct) != |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 150 | sizeof(tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 151 | |
| 152 | if ((unsigned long)buf % 16) |
| 153 | printk("save_i387: bad fpstate %p\n", buf); |
| 154 | |
| 155 | if (!used_math()) |
| 156 | return 0; |
| 157 | clear_used_math(); /* trigger finit */ |
| 158 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 159 | err = save_i387_checking((struct i387_fxsave_struct __user *) |
| 160 | buf); |
| 161 | if (err) |
| 162 | return err; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 163 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 164 | stts(); |
| 165 | } else { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 166 | if (__copy_to_user(buf, &tsk->thread.xstate->fxsave, |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 167 | sizeof(struct i387_fxsave_struct))) |
| 168 | return -1; |
| 169 | } |
| 170 | return 1; |
| 171 | } |
| 172 | |
| 173 | /* |
| 174 | * This restores directly out of user space. Exceptions are handled. |
| 175 | */ |
| 176 | static inline int restore_i387(struct _fpstate __user *buf) |
| 177 | { |
| 178 | set_used_math(); |
| 179 | if (!(task_thread_info(current)->status & TS_USEDFPU)) { |
| 180 | clts(); |
| 181 | task_thread_info(current)->status |= TS_USEDFPU; |
| 182 | } |
| 183 | return restore_fpu_checking((__force struct i387_fxsave_struct *)buf); |
| 184 | } |
| 185 | |
| 186 | #else /* CONFIG_X86_32 */ |
| 187 | |
| 188 | static inline void tolerant_fwait(void) |
| 189 | { |
| 190 | asm volatile("fnclex ; fwait"); |
| 191 | } |
| 192 | |
| 193 | static inline void restore_fpu(struct task_struct *tsk) |
| 194 | { |
| 195 | /* |
| 196 | * The "nop" is needed to make the instructions the same |
| 197 | * length. |
| 198 | */ |
| 199 | alternative_input( |
| 200 | "nop ; frstor %1", |
| 201 | "fxrstor %1", |
| 202 | X86_FEATURE_FXSR, |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 203 | "m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | /* We need a safe address that is cheap to find and that is already |
| 207 | in L1 during context switch. The best choices are unfortunately |
| 208 | different for UP and SMP */ |
| 209 | #ifdef CONFIG_SMP |
| 210 | #define safe_address (__per_cpu_offset[0]) |
| 211 | #else |
| 212 | #define safe_address (kstat_cpu(0).cpustat.user) |
| 213 | #endif |
| 214 | |
| 215 | /* |
| 216 | * These must be called with preempt disabled |
| 217 | */ |
| 218 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 219 | { |
| 220 | /* Use more nops than strictly needed in case the compiler |
| 221 | varies code */ |
| 222 | alternative_input( |
| 223 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, |
| 224 | "fxsave %[fx]\n" |
| 225 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", |
| 226 | X86_FEATURE_FXSR, |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 227 | [fx] "m" (tsk->thread.xstate->fxsave), |
| 228 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 229 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 230 | is pending. Clear the x87 state here by setting it to fixed |
| 231 | values. safe_address is a random variable that should be in L1 */ |
| 232 | alternative_input( |
| 233 | GENERIC_NOP8 GENERIC_NOP2, |
| 234 | "emms\n\t" /* clear stack tags */ |
| 235 | "fildl %[addr]", /* set F?P to defined value */ |
| 236 | X86_FEATURE_FXSAVE_LEAK, |
| 237 | [addr] "m" (safe_address)); |
| 238 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 239 | } |
| 240 | |
| 241 | /* |
| 242 | * Signal frame handlers... |
| 243 | */ |
| 244 | extern int save_i387(struct _fpstate __user *buf); |
| 245 | extern int restore_i387(struct _fpstate __user *buf); |
| 246 | |
| 247 | #endif /* CONFIG_X86_64 */ |
| 248 | |
| 249 | static inline void __unlazy_fpu(struct task_struct *tsk) |
| 250 | { |
| 251 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 252 | __save_init_fpu(tsk); |
| 253 | stts(); |
| 254 | } else |
| 255 | tsk->fpu_counter = 0; |
| 256 | } |
| 257 | |
| 258 | static inline void __clear_fpu(struct task_struct *tsk) |
| 259 | { |
| 260 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 261 | tolerant_fwait(); |
| 262 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 263 | stts(); |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | static inline void kernel_fpu_begin(void) |
| 268 | { |
| 269 | struct thread_info *me = current_thread_info(); |
| 270 | preempt_disable(); |
| 271 | if (me->status & TS_USEDFPU) |
| 272 | __save_init_fpu(me->task); |
| 273 | else |
| 274 | clts(); |
| 275 | } |
| 276 | |
| 277 | static inline void kernel_fpu_end(void) |
| 278 | { |
| 279 | stts(); |
| 280 | preempt_enable(); |
| 281 | } |
| 282 | |
| 283 | #ifdef CONFIG_X86_64 |
| 284 | |
| 285 | static inline void save_init_fpu(struct task_struct *tsk) |
| 286 | { |
| 287 | __save_init_fpu(tsk); |
| 288 | stts(); |
| 289 | } |
| 290 | |
| 291 | #define unlazy_fpu __unlazy_fpu |
| 292 | #define clear_fpu __clear_fpu |
| 293 | |
| 294 | #else /* CONFIG_X86_32 */ |
| 295 | |
| 296 | /* |
| 297 | * These disable preemption on their own and are safe |
| 298 | */ |
| 299 | static inline void save_init_fpu(struct task_struct *tsk) |
| 300 | { |
| 301 | preempt_disable(); |
| 302 | __save_init_fpu(tsk); |
| 303 | stts(); |
| 304 | preempt_enable(); |
| 305 | } |
| 306 | |
| 307 | static inline void unlazy_fpu(struct task_struct *tsk) |
| 308 | { |
| 309 | preempt_disable(); |
| 310 | __unlazy_fpu(tsk); |
| 311 | preempt_enable(); |
| 312 | } |
| 313 | |
| 314 | static inline void clear_fpu(struct task_struct *tsk) |
| 315 | { |
| 316 | preempt_disable(); |
| 317 | __clear_fpu(tsk); |
| 318 | preempt_enable(); |
| 319 | } |
| 320 | |
| 321 | #endif /* CONFIG_X86_64 */ |
| 322 | |
| 323 | /* |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 324 | * i387 state interaction |
| 325 | */ |
| 326 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) |
| 327 | { |
| 328 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 329 | return tsk->thread.xstate->fxsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 330 | } else { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 331 | return (unsigned short) tsk->thread.xstate->fsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 332 | } |
| 333 | } |
| 334 | |
| 335 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) |
| 336 | { |
| 337 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 338 | return tsk->thread.xstate->fxsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 339 | } else { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 340 | return (unsigned short) tsk->thread.xstate->fsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 341 | } |
| 342 | } |
| 343 | |
| 344 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) |
| 345 | { |
| 346 | if (cpu_has_xmm) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 347 | return tsk->thread.xstate->fxsave.mxcsr; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 348 | } else { |
| 349 | return MXCSR_DEFAULT; |
| 350 | } |
| 351 | } |
| 352 | |
| 353 | #endif /* _ASM_X86_I387_H */ |