Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1994 Linus Torvalds |
| 3 | * |
| 4 | * Pentium III FXSR, SSE support |
| 5 | * General FPU state handling cleanups |
| 6 | * Gareth Hughes <gareth@valinux.com>, May 2000 |
| 7 | * x86-64 work by Andi Kleen 2002 |
| 8 | */ |
| 9 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 10 | #ifndef _ASM_X86_I387_H |
| 11 | #define _ASM_X86_I387_H |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 12 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 13 | #ifndef __ASSEMBLY__ |
| 14 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 15 | #include <linux/sched.h> |
| 16 | #include <linux/kernel_stat.h> |
| 17 | #include <linux/regset.h> |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 18 | #include <linux/hardirq.h> |
H. Peter Anvin | 92c37fa | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 19 | #include <asm/asm.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 20 | #include <asm/processor.h> |
| 21 | #include <asm/sigcontext.h> |
| 22 | #include <asm/user.h> |
| 23 | #include <asm/uaccess.h> |
Suresh Siddha | dc1e35c | 2008-07-29 10:29:19 -0700 | [diff] [blame] | 24 | #include <asm/xsave.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 25 | |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 26 | extern unsigned int sig_xstate_size; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 27 | extern void fpu_init(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 28 | extern void mxcsr_feature_mask_init(void); |
Suresh Siddha | aa283f4 | 2008-03-10 15:28:05 -0700 | [diff] [blame] | 29 | extern int init_fpu(struct task_struct *child); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 30 | extern asmlinkage void math_state_restore(void); |
Jeremy Fitzhardinge | e6e9cac | 2009-04-24 00:40:59 -0700 | [diff] [blame] | 31 | extern void __math_state_restore(void); |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 32 | extern void init_thread_xstate(void); |
Jaswinder Singh | 3645493 | 2008-07-21 22:31:57 +0530 | [diff] [blame] | 33 | extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 34 | |
| 35 | extern user_regset_active_fn fpregs_active, xfpregs_active; |
Suresh Siddha | 5b3efd5 | 2010-02-11 11:50:59 -0800 | [diff] [blame^] | 36 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, |
| 37 | xstateregs_get; |
| 38 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, |
| 39 | xstateregs_set; |
| 40 | |
| 41 | /* |
| 42 | * xstateregs_active == fpregs_active. Please refer to the comment |
| 43 | * at the definition of fpregs_active. |
| 44 | */ |
| 45 | #define xstateregs_active fpregs_active |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 46 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 47 | extern struct _fpx_sw_bytes fx_sw_reserved; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 48 | #ifdef CONFIG_IA32_EMULATION |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 49 | extern unsigned int sig_xstate_ia32_size; |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 50 | extern struct _fpx_sw_bytes fx_sw_reserved_ia32; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 51 | struct _fpstate_ia32; |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 52 | struct _xstate_ia32; |
| 53 | extern int save_i387_xstate_ia32(void __user *buf); |
| 54 | extern int restore_i387_xstate_ia32(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 55 | #endif |
| 56 | |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 57 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
| 58 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 59 | #ifdef CONFIG_X86_64 |
| 60 | |
| 61 | /* Ignore delayed exceptions from user space */ |
| 62 | static inline void tolerant_fwait(void) |
| 63 | { |
| 64 | asm volatile("1: fwait\n" |
| 65 | "2:\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 66 | _ASM_EXTABLE(1b, 2b)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 67 | } |
| 68 | |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 69 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 70 | { |
| 71 | int err; |
| 72 | |
| 73 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" |
| 74 | "2:\n" |
| 75 | ".section .fixup,\"ax\"\n" |
| 76 | "3: movl $-1,%[err]\n" |
| 77 | " jmp 2b\n" |
| 78 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 79 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 80 | : [err] "=r" (err) |
Jiri Slaby | 4ecf458 | 2009-04-08 13:32:00 +0200 | [diff] [blame] | 81 | #if 0 /* See comment in fxsave() below. */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 82 | : [fx] "r" (fx), "m" (*fx), "0" (0)); |
| 83 | #else |
| 84 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); |
| 85 | #endif |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 86 | return err; |
| 87 | } |
| 88 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 89 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 90 | is pending. Clear the x87 state here by setting it to fixed |
| 91 | values. The kernel data segment can be sometimes 0 and sometimes |
| 92 | new user value. Both should be ok. |
| 93 | Use the PDA as safe address because it should be already in L1. */ |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 94 | static inline void clear_fpu_state(struct task_struct *tsk) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 95 | { |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 96 | struct xsave_struct *xstate = &tsk->thread.xstate->xsave; |
| 97 | struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; |
| 98 | |
| 99 | /* |
| 100 | * xsave header may indicate the init state of the FP. |
| 101 | */ |
| 102 | if ((task_thread_info(tsk)->status & TS_XSAVE) && |
| 103 | !(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) |
| 104 | return; |
| 105 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 106 | if (unlikely(fx->swd & X87_FSW_ES)) |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 107 | asm volatile("fnclex"); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 108 | alternative_input(ASM_NOP8 ASM_NOP2, |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 109 | " emms\n" /* clear stack tags */ |
| 110 | " fildl %%gs:0", /* load to clear state */ |
| 111 | X86_FEATURE_FXSAVE_LEAK); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 112 | } |
| 113 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 114 | static inline int fxsave_user(struct i387_fxsave_struct __user *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 115 | { |
| 116 | int err; |
| 117 | |
| 118 | asm volatile("1: rex64/fxsave (%[fx])\n\t" |
| 119 | "2:\n" |
| 120 | ".section .fixup,\"ax\"\n" |
| 121 | "3: movl $-1,%[err]\n" |
| 122 | " jmp 2b\n" |
| 123 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 124 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 125 | : [err] "=r" (err), "=m" (*fx) |
Jiri Slaby | 4ecf458 | 2009-04-08 13:32:00 +0200 | [diff] [blame] | 126 | #if 0 /* See comment in fxsave() below. */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 127 | : [fx] "r" (fx), "0" (0)); |
| 128 | #else |
| 129 | : [fx] "cdaSDb" (fx), "0" (0)); |
| 130 | #endif |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 131 | if (unlikely(err) && |
| 132 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 133 | err = -EFAULT; |
| 134 | /* No need to clear here because the caller clears USED_MATH */ |
| 135 | return err; |
| 136 | } |
| 137 | |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 138 | static inline void fxsave(struct task_struct *tsk) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 139 | { |
| 140 | /* Using "rex64; fxsave %0" is broken because, if the memory operand |
| 141 | uses any extended registers for addressing, a second REX prefix |
| 142 | will be generated (to the assembler, rex64 followed by semicolon |
| 143 | is a separate instruction), and hence the 64-bitness is lost. */ |
| 144 | #if 0 |
| 145 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported |
| 146 | starting with gas 2.16. */ |
| 147 | __asm__ __volatile__("fxsaveq %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 148 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 149 | #elif 0 |
| 150 | /* Using, as a workaround, the properly prefixed form below isn't |
| 151 | accepted by any binutils version so far released, complaining that |
| 152 | the same type of prefix is used twice if an extended register is |
| 153 | needed for addressing (fix submitted to mainline 2005-11-21). */ |
| 154 | __asm__ __volatile__("rex64/fxsave %0" |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 155 | : "=m" (tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 156 | #else |
| 157 | /* This, however, we can work around by forcing the compiler to select |
| 158 | an addressing mode that doesn't require extended registers. */ |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 159 | __asm__ __volatile__("rex64/fxsave (%1)" |
| 160 | : "=m" (tsk->thread.xstate->fxsave) |
| 161 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 162 | #endif |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 166 | { |
| 167 | if (task_thread_info(tsk)->status & TS_XSAVE) |
| 168 | xsave(tsk); |
| 169 | else |
| 170 | fxsave(tsk); |
| 171 | |
| 172 | clear_fpu_state(tsk); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 173 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 174 | } |
| 175 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 176 | #else /* CONFIG_X86_32 */ |
| 177 | |
Daniel Glöckner | ab9e185 | 2009-03-04 19:42:27 +0100 | [diff] [blame] | 178 | #ifdef CONFIG_MATH_EMULATION |
| 179 | extern void finit_task(struct task_struct *tsk); |
| 180 | #else |
| 181 | static inline void finit_task(struct task_struct *tsk) |
| 182 | { |
| 183 | } |
| 184 | #endif |
Suresh Siddha | e8a496a | 2008-05-23 16:26:37 -0700 | [diff] [blame] | 185 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 186 | static inline void tolerant_fwait(void) |
| 187 | { |
| 188 | asm volatile("fnclex ; fwait"); |
| 189 | } |
| 190 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 191 | /* perform fxrstor iff the processor has extended states, otherwise frstor */ |
| 192 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 193 | { |
| 194 | /* |
| 195 | * The "nop" is needed to make the instructions the same |
| 196 | * length. |
| 197 | */ |
| 198 | alternative_input( |
| 199 | "nop ; frstor %1", |
| 200 | "fxrstor %1", |
| 201 | X86_FEATURE_FXSR, |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 202 | "m" (*fx)); |
| 203 | |
Jiri Slaby | fcb2ac5 | 2009-04-08 13:31:58 +0200 | [diff] [blame] | 204 | return 0; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 205 | } |
| 206 | |
| 207 | /* We need a safe address that is cheap to find and that is already |
| 208 | in L1 during context switch. The best choices are unfortunately |
| 209 | different for UP and SMP */ |
| 210 | #ifdef CONFIG_SMP |
| 211 | #define safe_address (__per_cpu_offset[0]) |
| 212 | #else |
| 213 | #define safe_address (kstat_cpu(0).cpustat.user) |
| 214 | #endif |
| 215 | |
| 216 | /* |
| 217 | * These must be called with preempt disabled |
| 218 | */ |
| 219 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 220 | { |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 221 | if (task_thread_info(tsk)->status & TS_XSAVE) { |
| 222 | struct xsave_struct *xstate = &tsk->thread.xstate->xsave; |
| 223 | struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; |
| 224 | |
| 225 | xsave(tsk); |
| 226 | |
| 227 | /* |
| 228 | * xsave header may indicate the init state of the FP. |
| 229 | */ |
| 230 | if (!(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) |
| 231 | goto end; |
| 232 | |
| 233 | if (unlikely(fx->swd & X87_FSW_ES)) |
| 234 | asm volatile("fnclex"); |
| 235 | |
| 236 | /* |
| 237 | * we can do a simple return here or be paranoid :) |
| 238 | */ |
| 239 | goto clear_state; |
| 240 | } |
| 241 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 242 | /* Use more nops than strictly needed in case the compiler |
| 243 | varies code */ |
| 244 | alternative_input( |
| 245 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, |
| 246 | "fxsave %[fx]\n" |
| 247 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", |
| 248 | X86_FEATURE_FXSR, |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 249 | [fx] "m" (tsk->thread.xstate->fxsave), |
| 250 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 251 | clear_state: |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 252 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 253 | is pending. Clear the x87 state here by setting it to fixed |
| 254 | values. safe_address is a random variable that should be in L1 */ |
| 255 | alternative_input( |
| 256 | GENERIC_NOP8 GENERIC_NOP2, |
| 257 | "emms\n\t" /* clear stack tags */ |
| 258 | "fildl %[addr]", /* set F?P to defined value */ |
| 259 | X86_FEATURE_FXSAVE_LEAK, |
| 260 | [addr] "m" (safe_address)); |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 261 | end: |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 262 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 263 | } |
| 264 | |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 265 | #endif /* CONFIG_X86_64 */ |
| 266 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 267 | static inline int restore_fpu_checking(struct task_struct *tsk) |
| 268 | { |
| 269 | if (task_thread_info(tsk)->status & TS_XSAVE) |
| 270 | return xrstor_checking(&tsk->thread.xstate->xsave); |
| 271 | else |
| 272 | return fxrstor_checking(&tsk->thread.xstate->fxsave); |
| 273 | } |
| 274 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 275 | /* |
| 276 | * Signal frame handlers... |
| 277 | */ |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 278 | extern int save_i387_xstate(void __user *buf); |
| 279 | extern int restore_i387_xstate(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 280 | |
| 281 | static inline void __unlazy_fpu(struct task_struct *tsk) |
| 282 | { |
| 283 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 284 | __save_init_fpu(tsk); |
| 285 | stts(); |
| 286 | } else |
| 287 | tsk->fpu_counter = 0; |
| 288 | } |
| 289 | |
| 290 | static inline void __clear_fpu(struct task_struct *tsk) |
| 291 | { |
| 292 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 293 | tolerant_fwait(); |
| 294 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 295 | stts(); |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | static inline void kernel_fpu_begin(void) |
| 300 | { |
| 301 | struct thread_info *me = current_thread_info(); |
| 302 | preempt_disable(); |
| 303 | if (me->status & TS_USEDFPU) |
| 304 | __save_init_fpu(me->task); |
| 305 | else |
| 306 | clts(); |
| 307 | } |
| 308 | |
| 309 | static inline void kernel_fpu_end(void) |
| 310 | { |
| 311 | stts(); |
| 312 | preempt_enable(); |
| 313 | } |
| 314 | |
Huang Ying | ae4b688 | 2009-08-31 13:11:54 +0800 | [diff] [blame] | 315 | static inline bool irq_fpu_usable(void) |
| 316 | { |
| 317 | struct pt_regs *regs; |
| 318 | |
| 319 | return !in_interrupt() || !(regs = get_irq_regs()) || \ |
| 320 | user_mode(regs) || (read_cr0() & X86_CR0_TS); |
| 321 | } |
| 322 | |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 323 | /* |
| 324 | * Some instructions like VIA's padlock instructions generate a spurious |
| 325 | * DNA fault but don't modify SSE registers. And these instructions |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 326 | * get used from interrupt context as well. To prevent these kernel instructions |
| 327 | * in interrupt context interacting wrongly with other user/kernel fpu usage, we |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 328 | * should use them only in the context of irq_ts_save/restore() |
| 329 | */ |
| 330 | static inline int irq_ts_save(void) |
| 331 | { |
| 332 | /* |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 333 | * If in process context and not atomic, we can take a spurious DNA fault. |
| 334 | * Otherwise, doing clts() in process context requires disabling preemption |
| 335 | * or some heavy lifting like kernel_fpu_begin() |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 336 | */ |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 337 | if (!in_atomic()) |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 338 | return 0; |
| 339 | |
| 340 | if (read_cr0() & X86_CR0_TS) { |
| 341 | clts(); |
| 342 | return 1; |
| 343 | } |
| 344 | |
| 345 | return 0; |
| 346 | } |
| 347 | |
| 348 | static inline void irq_ts_restore(int TS_state) |
| 349 | { |
| 350 | if (TS_state) |
| 351 | stts(); |
| 352 | } |
| 353 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 354 | #ifdef CONFIG_X86_64 |
| 355 | |
| 356 | static inline void save_init_fpu(struct task_struct *tsk) |
| 357 | { |
| 358 | __save_init_fpu(tsk); |
| 359 | stts(); |
| 360 | } |
| 361 | |
| 362 | #define unlazy_fpu __unlazy_fpu |
| 363 | #define clear_fpu __clear_fpu |
| 364 | |
| 365 | #else /* CONFIG_X86_32 */ |
| 366 | |
| 367 | /* |
| 368 | * These disable preemption on their own and are safe |
| 369 | */ |
| 370 | static inline void save_init_fpu(struct task_struct *tsk) |
| 371 | { |
| 372 | preempt_disable(); |
| 373 | __save_init_fpu(tsk); |
| 374 | stts(); |
| 375 | preempt_enable(); |
| 376 | } |
| 377 | |
| 378 | static inline void unlazy_fpu(struct task_struct *tsk) |
| 379 | { |
| 380 | preempt_disable(); |
| 381 | __unlazy_fpu(tsk); |
| 382 | preempt_enable(); |
| 383 | } |
| 384 | |
| 385 | static inline void clear_fpu(struct task_struct *tsk) |
| 386 | { |
| 387 | preempt_disable(); |
| 388 | __clear_fpu(tsk); |
| 389 | preempt_enable(); |
| 390 | } |
| 391 | |
| 392 | #endif /* CONFIG_X86_64 */ |
| 393 | |
| 394 | /* |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 395 | * i387 state interaction |
| 396 | */ |
| 397 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) |
| 398 | { |
| 399 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 400 | return tsk->thread.xstate->fxsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 401 | } else { |
Suresh Siddha | 1679f27 | 2008-04-16 10:27:53 +0200 | [diff] [blame] | 402 | return (unsigned short)tsk->thread.xstate->fsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 403 | } |
| 404 | } |
| 405 | |
| 406 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) |
| 407 | { |
| 408 | if (cpu_has_fxsr) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 409 | return tsk->thread.xstate->fxsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 410 | } else { |
Suresh Siddha | 1679f27 | 2008-04-16 10:27:53 +0200 | [diff] [blame] | 411 | return (unsigned short)tsk->thread.xstate->fsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 412 | } |
| 413 | } |
| 414 | |
| 415 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) |
| 416 | { |
| 417 | if (cpu_has_xmm) { |
Suresh Siddha | 61c4628 | 2008-03-10 15:28:04 -0700 | [diff] [blame] | 418 | return tsk->thread.xstate->fxsave.mxcsr; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 419 | } else { |
| 420 | return MXCSR_DEFAULT; |
| 421 | } |
| 422 | } |
| 423 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 424 | #endif /* __ASSEMBLY__ */ |
| 425 | |
| 426 | #define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 |
| 427 | #define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 |
| 428 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 429 | #endif /* _ASM_X86_I387_H */ |