Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1994 Linus Torvalds |
| 3 | * |
| 4 | * Pentium III FXSR, SSE support |
| 5 | * General FPU state handling cleanups |
| 6 | * Gareth Hughes <gareth@valinux.com>, May 2000 |
| 7 | * x86-64 work by Andi Kleen 2002 |
| 8 | */ |
| 9 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 10 | #ifndef _ASM_X86_I387_H |
| 11 | #define _ASM_X86_I387_H |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 12 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 13 | #ifndef __ASSEMBLY__ |
| 14 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 15 | #include <linux/sched.h> |
| 16 | #include <linux/kernel_stat.h> |
| 17 | #include <linux/regset.h> |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 18 | #include <linux/hardirq.h> |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 19 | #include <linux/slab.h> |
H. Peter Anvin | 92c37fa | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 20 | #include <asm/asm.h> |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 21 | #include <asm/cpufeature.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 22 | #include <asm/processor.h> |
| 23 | #include <asm/sigcontext.h> |
| 24 | #include <asm/user.h> |
| 25 | #include <asm/uaccess.h> |
Suresh Siddha | dc1e35c | 2008-07-29 10:29:19 -0700 | [diff] [blame] | 26 | #include <asm/xsave.h> |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 27 | |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 28 | extern unsigned int sig_xstate_size; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 29 | extern void fpu_init(void); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 30 | extern void mxcsr_feature_mask_init(void); |
Suresh Siddha | aa283f4 | 2008-03-10 15:28:05 -0700 | [diff] [blame] | 31 | extern int init_fpu(struct task_struct *child); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 32 | extern asmlinkage void math_state_restore(void); |
Jeremy Fitzhardinge | e6e9cac | 2009-04-24 00:40:59 -0700 | [diff] [blame] | 33 | extern void __math_state_restore(void); |
Jaswinder Singh | 3645493 | 2008-07-21 22:31:57 +0530 | [diff] [blame] | 34 | extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 35 | |
| 36 | extern user_regset_active_fn fpregs_active, xfpregs_active; |
Suresh Siddha | 5b3efd5 | 2010-02-11 11:50:59 -0800 | [diff] [blame] | 37 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, |
| 38 | xstateregs_get; |
| 39 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, |
| 40 | xstateregs_set; |
| 41 | |
| 42 | /* |
| 43 | * xstateregs_active == fpregs_active. Please refer to the comment |
| 44 | * at the definition of fpregs_active. |
| 45 | */ |
| 46 | #define xstateregs_active fpregs_active |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 47 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 48 | extern struct _fpx_sw_bytes fx_sw_reserved; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 49 | #ifdef CONFIG_IA32_EMULATION |
Suresh Siddha | 3c1c7f1 | 2008-07-29 10:29:21 -0700 | [diff] [blame] | 50 | extern unsigned int sig_xstate_ia32_size; |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 51 | extern struct _fpx_sw_bytes fx_sw_reserved_ia32; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 52 | struct _fpstate_ia32; |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 53 | struct _xstate_ia32; |
| 54 | extern int save_i387_xstate_ia32(void __user *buf); |
| 55 | extern int restore_i387_xstate_ia32(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 56 | #endif |
| 57 | |
Brian Gerst | 8eb91a5 | 2010-09-03 21:17:16 -0400 | [diff] [blame] | 58 | #ifdef CONFIG_MATH_EMULATION |
| 59 | extern void finit_soft_fpu(struct i387_soft_struct *soft); |
| 60 | #else |
| 61 | static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} |
| 62 | #endif |
| 63 | |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 64 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
| 65 | |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 66 | static __always_inline __pure bool use_xsaveopt(void) |
| 67 | { |
Suresh Siddha | 6bad06b | 2010-07-19 16:05:52 -0700 | [diff] [blame] | 68 | return static_cpu_has(X86_FEATURE_XSAVEOPT); |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 69 | } |
| 70 | |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 71 | static __always_inline __pure bool use_xsave(void) |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 72 | { |
H. Peter Anvin | c9775b4 | 2010-05-11 17:49:54 -0700 | [diff] [blame] | 73 | return static_cpu_has(X86_FEATURE_XSAVE); |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 74 | } |
| 75 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 76 | static __always_inline __pure bool use_fxsr(void) |
| 77 | { |
| 78 | return static_cpu_has(X86_FEATURE_FXSR); |
| 79 | } |
| 80 | |
Suresh Siddha | 29104e1 | 2010-07-19 16:05:49 -0700 | [diff] [blame] | 81 | extern void __sanitize_i387_state(struct task_struct *); |
| 82 | |
| 83 | static inline void sanitize_i387_state(struct task_struct *tsk) |
| 84 | { |
| 85 | if (!use_xsaveopt()) |
| 86 | return; |
| 87 | __sanitize_i387_state(tsk); |
| 88 | } |
| 89 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 90 | #ifdef CONFIG_X86_64 |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 91 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 92 | { |
| 93 | int err; |
| 94 | |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 95 | /* See comment in fxsave() below. */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 96 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" |
| 97 | "2:\n" |
| 98 | ".section .fixup,\"ax\"\n" |
| 99 | "3: movl $-1,%[err]\n" |
| 100 | " jmp 2b\n" |
| 101 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 102 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 103 | : [err] "=r" (err) |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 104 | : [fx] "R" (fx), "m" (*fx), "0" (0)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 105 | return err; |
| 106 | } |
| 107 | |
Suresh Siddha | c37b5ef | 2008-07-29 10:29:25 -0700 | [diff] [blame] | 108 | static inline int fxsave_user(struct i387_fxsave_struct __user *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 109 | { |
| 110 | int err; |
| 111 | |
Suresh Siddha | 8e221b6 | 2010-06-22 16:23:37 -0700 | [diff] [blame] | 112 | /* |
| 113 | * Clear the bytes not touched by the fxsave and reserved |
| 114 | * for the SW usage. |
| 115 | */ |
| 116 | err = __clear_user(&fx->sw_reserved, |
| 117 | sizeof(struct _fpx_sw_bytes)); |
| 118 | if (unlikely(err)) |
| 119 | return -EFAULT; |
| 120 | |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 121 | /* See comment in fxsave() below. */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 122 | asm volatile("1: rex64/fxsave (%[fx])\n\t" |
| 123 | "2:\n" |
| 124 | ".section .fixup,\"ax\"\n" |
| 125 | "3: movl $-1,%[err]\n" |
| 126 | " jmp 2b\n" |
| 127 | ".previous\n" |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 128 | _ASM_EXTABLE(1b, 3b) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 129 | : [err] "=r" (err), "=m" (*fx) |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 130 | : [fx] "R" (fx), "0" (0)); |
Joe Perches | affe663 | 2008-03-23 01:02:18 -0700 | [diff] [blame] | 131 | if (unlikely(err) && |
| 132 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 133 | err = -EFAULT; |
| 134 | /* No need to clear here because the caller clears USED_MATH */ |
| 135 | return err; |
| 136 | } |
| 137 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 138 | static inline void fpu_fxsave(struct fpu *fpu) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 139 | { |
| 140 | /* Using "rex64; fxsave %0" is broken because, if the memory operand |
| 141 | uses any extended registers for addressing, a second REX prefix |
| 142 | will be generated (to the assembler, rex64 followed by semicolon |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 143 | is a separate instruction), and hence the 64-bitness is lost. |
| 144 | Using "fxsaveq %0" would be the ideal choice, but is only supported |
| 145 | starting with gas 2.16. |
| 146 | asm volatile("fxsaveq %0" |
| 147 | : "=m" (fpu->state->fxsave)); |
| 148 | Using, as a workaround, the properly prefixed form below isn't |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 149 | accepted by any binutils version so far released, complaining that |
| 150 | the same type of prefix is used twice if an extended register is |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 151 | needed for addressing (fix submitted to mainline 2005-11-21). |
| 152 | asm volatile("rex64/fxsave %0" |
| 153 | : "=m" (fpu->state->fxsave)); |
| 154 | This, however, we can work around by forcing the compiler to select |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 155 | an addressing mode that doesn't require extended registers. */ |
Brian Gerst | 8202413 | 2010-09-03 21:17:14 -0400 | [diff] [blame] | 156 | asm volatile("rex64/fxsave (%[fx])" |
| 157 | : "=m" (fpu->state->fxsave) |
| 158 | : [fx] "R" (&fpu->state->fxsave)); |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 159 | } |
| 160 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 161 | #else /* CONFIG_X86_32 */ |
| 162 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 163 | /* perform fxrstor iff the processor has extended states, otherwise frstor */ |
| 164 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 165 | { |
| 166 | /* |
| 167 | * The "nop" is needed to make the instructions the same |
| 168 | * length. |
| 169 | */ |
| 170 | alternative_input( |
| 171 | "nop ; frstor %1", |
| 172 | "fxrstor %1", |
| 173 | X86_FEATURE_FXSR, |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 174 | "m" (*fx)); |
| 175 | |
Jiri Slaby | fcb2ac5 | 2009-04-08 13:31:58 +0200 | [diff] [blame] | 176 | return 0; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 177 | } |
| 178 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 179 | static inline void fpu_fxsave(struct fpu *fpu) |
| 180 | { |
| 181 | asm volatile("fxsave %[fx]" |
| 182 | : [fx] "=m" (fpu->state->fxsave)); |
| 183 | } |
| 184 | |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame^] | 185 | #endif /* CONFIG_X86_64 */ |
| 186 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 187 | /* We need a safe address that is cheap to find and that is already |
| 188 | in L1 during context switch. The best choices are unfortunately |
| 189 | different for UP and SMP */ |
| 190 | #ifdef CONFIG_SMP |
| 191 | #define safe_address (__per_cpu_offset[0]) |
| 192 | #else |
| 193 | #define safe_address (kstat_cpu(0).cpustat.user) |
| 194 | #endif |
| 195 | |
| 196 | /* |
| 197 | * These must be called with preempt disabled |
| 198 | */ |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 199 | static inline void fpu_save_init(struct fpu *fpu) |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 200 | { |
Avi Kivity | c9ad488 | 2010-05-06 11:45:45 +0300 | [diff] [blame] | 201 | if (use_xsave()) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 202 | fpu_xsave(fpu); |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 203 | |
| 204 | /* |
| 205 | * xsave header may indicate the init state of the FP. |
| 206 | */ |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 207 | if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP)) |
| 208 | return; |
| 209 | } else if (use_fxsr()) { |
| 210 | fpu_fxsave(fpu); |
| 211 | } else { |
| 212 | asm volatile("fsave %[fx]; fwait" |
| 213 | : [fx] "=m" (fpu->state->fsave)); |
| 214 | return; |
Suresh Siddha | b359e8a | 2008-07-29 10:29:20 -0700 | [diff] [blame] | 215 | } |
| 216 | |
Brian Gerst | 58a992b | 2010-09-03 21:17:18 -0400 | [diff] [blame] | 217 | if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) |
| 218 | asm volatile("fnclex"); |
| 219 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 220 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
| 221 | is pending. Clear the x87 state here by setting it to fixed |
| 222 | values. safe_address is a random variable that should be in L1 */ |
| 223 | alternative_input( |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame^] | 224 | ASM_NOP8 ASM_NOP2, |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 225 | "emms\n\t" /* clear stack tags */ |
Brian Gerst | b2b57fe | 2010-09-03 21:17:19 -0400 | [diff] [blame^] | 226 | "fildl %P[addr]", /* set F?P to defined value */ |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 227 | X86_FEATURE_FXSAVE_LEAK, |
| 228 | [addr] "m" (safe_address)); |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 229 | } |
| 230 | |
| 231 | static inline void __save_init_fpu(struct task_struct *tsk) |
| 232 | { |
| 233 | fpu_save_init(&tsk->thread.fpu); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 234 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 235 | } |
| 236 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 237 | static inline int fpu_fxrstor_checking(struct fpu *fpu) |
| 238 | { |
| 239 | return fxrstor_checking(&fpu->state->fxsave); |
| 240 | } |
| 241 | |
| 242 | static inline int fpu_restore_checking(struct fpu *fpu) |
| 243 | { |
| 244 | if (use_xsave()) |
| 245 | return fpu_xrstor_checking(fpu); |
| 246 | else |
| 247 | return fpu_fxrstor_checking(fpu); |
| 248 | } |
| 249 | |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 250 | static inline int restore_fpu_checking(struct task_struct *tsk) |
| 251 | { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 252 | return fpu_restore_checking(&tsk->thread.fpu); |
Jiri Slaby | 34ba476 | 2009-04-08 13:31:59 +0200 | [diff] [blame] | 253 | } |
| 254 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 255 | /* |
| 256 | * Signal frame handlers... |
| 257 | */ |
Suresh Siddha | ab51370 | 2008-07-29 10:29:22 -0700 | [diff] [blame] | 258 | extern int save_i387_xstate(void __user *buf); |
| 259 | extern int restore_i387_xstate(void __user *buf); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 260 | |
| 261 | static inline void __unlazy_fpu(struct task_struct *tsk) |
| 262 | { |
| 263 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
| 264 | __save_init_fpu(tsk); |
| 265 | stts(); |
| 266 | } else |
| 267 | tsk->fpu_counter = 0; |
| 268 | } |
| 269 | |
| 270 | static inline void __clear_fpu(struct task_struct *tsk) |
| 271 | { |
| 272 | if (task_thread_info(tsk)->status & TS_USEDFPU) { |
Brian Gerst | 51115d4 | 2010-09-03 21:17:10 -0400 | [diff] [blame] | 273 | /* Ignore delayed exceptions from user space */ |
| 274 | asm volatile("1: fwait\n" |
| 275 | "2:\n" |
| 276 | _ASM_EXTABLE(1b, 2b)); |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 277 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
| 278 | stts(); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | static inline void kernel_fpu_begin(void) |
| 283 | { |
| 284 | struct thread_info *me = current_thread_info(); |
| 285 | preempt_disable(); |
| 286 | if (me->status & TS_USEDFPU) |
| 287 | __save_init_fpu(me->task); |
| 288 | else |
| 289 | clts(); |
| 290 | } |
| 291 | |
| 292 | static inline void kernel_fpu_end(void) |
| 293 | { |
| 294 | stts(); |
| 295 | preempt_enable(); |
| 296 | } |
| 297 | |
Huang Ying | ae4b688 | 2009-08-31 13:11:54 +0800 | [diff] [blame] | 298 | static inline bool irq_fpu_usable(void) |
| 299 | { |
| 300 | struct pt_regs *regs; |
| 301 | |
| 302 | return !in_interrupt() || !(regs = get_irq_regs()) || \ |
| 303 | user_mode(regs) || (read_cr0() & X86_CR0_TS); |
| 304 | } |
| 305 | |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 306 | /* |
| 307 | * Some instructions like VIA's padlock instructions generate a spurious |
| 308 | * DNA fault but don't modify SSE registers. And these instructions |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 309 | * get used from interrupt context as well. To prevent these kernel instructions |
| 310 | * in interrupt context interacting wrongly with other user/kernel fpu usage, we |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 311 | * should use them only in the context of irq_ts_save/restore() |
| 312 | */ |
| 313 | static inline int irq_ts_save(void) |
| 314 | { |
| 315 | /* |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 316 | * If in process context and not atomic, we can take a spurious DNA fault. |
| 317 | * Otherwise, doing clts() in process context requires disabling preemption |
| 318 | * or some heavy lifting like kernel_fpu_begin() |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 319 | */ |
Chuck Ebbert | 0b8c3d5 | 2009-06-09 10:40:50 -0400 | [diff] [blame] | 320 | if (!in_atomic()) |
Suresh Siddha | e491401 | 2008-08-13 22:02:26 +1000 | [diff] [blame] | 321 | return 0; |
| 322 | |
| 323 | if (read_cr0() & X86_CR0_TS) { |
| 324 | clts(); |
| 325 | return 1; |
| 326 | } |
| 327 | |
| 328 | return 0; |
| 329 | } |
| 330 | |
| 331 | static inline void irq_ts_restore(int TS_state) |
| 332 | { |
| 333 | if (TS_state) |
| 334 | stts(); |
| 335 | } |
| 336 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 337 | /* |
| 338 | * These disable preemption on their own and are safe |
| 339 | */ |
| 340 | static inline void save_init_fpu(struct task_struct *tsk) |
| 341 | { |
| 342 | preempt_disable(); |
| 343 | __save_init_fpu(tsk); |
| 344 | stts(); |
| 345 | preempt_enable(); |
| 346 | } |
| 347 | |
| 348 | static inline void unlazy_fpu(struct task_struct *tsk) |
| 349 | { |
| 350 | preempt_disable(); |
| 351 | __unlazy_fpu(tsk); |
| 352 | preempt_enable(); |
| 353 | } |
| 354 | |
| 355 | static inline void clear_fpu(struct task_struct *tsk) |
| 356 | { |
| 357 | preempt_disable(); |
| 358 | __clear_fpu(tsk); |
| 359 | preempt_enable(); |
| 360 | } |
| 361 | |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 362 | /* |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 363 | * i387 state interaction |
| 364 | */ |
| 365 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) |
| 366 | { |
| 367 | if (cpu_has_fxsr) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 368 | return tsk->thread.fpu.state->fxsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 369 | } else { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 370 | return (unsigned short)tsk->thread.fpu.state->fsave.cwd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 371 | } |
| 372 | } |
| 373 | |
| 374 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) |
| 375 | { |
| 376 | if (cpu_has_fxsr) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 377 | return tsk->thread.fpu.state->fxsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 378 | } else { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 379 | return (unsigned short)tsk->thread.fpu.state->fsave.swd; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 380 | } |
| 381 | } |
| 382 | |
| 383 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) |
| 384 | { |
| 385 | if (cpu_has_xmm) { |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 386 | return tsk->thread.fpu.state->fxsave.mxcsr; |
Roland McGrath | 1eeaed7 | 2008-01-30 13:31:51 +0100 | [diff] [blame] | 387 | } else { |
| 388 | return MXCSR_DEFAULT; |
| 389 | } |
| 390 | } |
| 391 | |
Avi Kivity | 8660328 | 2010-05-06 11:45:46 +0300 | [diff] [blame] | 392 | static bool fpu_allocated(struct fpu *fpu) |
| 393 | { |
| 394 | return fpu->state != NULL; |
| 395 | } |
| 396 | |
| 397 | static inline int fpu_alloc(struct fpu *fpu) |
| 398 | { |
| 399 | if (fpu_allocated(fpu)) |
| 400 | return 0; |
| 401 | fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); |
| 402 | if (!fpu->state) |
| 403 | return -ENOMEM; |
| 404 | WARN_ON((unsigned long)fpu->state & 15); |
| 405 | return 0; |
| 406 | } |
| 407 | |
| 408 | static inline void fpu_free(struct fpu *fpu) |
| 409 | { |
| 410 | if (fpu->state) { |
| 411 | kmem_cache_free(task_xstate_cachep, fpu->state); |
| 412 | fpu->state = NULL; |
| 413 | } |
| 414 | } |
| 415 | |
| 416 | static inline void fpu_copy(struct fpu *dst, struct fpu *src) |
| 417 | { |
| 418 | memcpy(dst->state, src->state, xstate_size); |
| 419 | } |
| 420 | |
Sheng Yang | 5ee481d | 2010-05-17 17:22:23 +0800 | [diff] [blame] | 421 | extern void fpu_finit(struct fpu *fpu); |
| 422 | |
Herbert Xu | 3b0d659 | 2009-11-03 09:11:15 -0500 | [diff] [blame] | 423 | #endif /* __ASSEMBLY__ */ |
| 424 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 425 | #endif /* _ASM_X86_I387_H */ |