blob: 52b8f40b1c73d48d206d497a6fadc381b5ddd888 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/arm/vfp/vfpmodule.c
3 *
4 * Copyright (C) 2004 ARM Limited.
5 * Written by Deep Blue Solutions Limited.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/types.h>
Russell King90b44192010-12-18 10:59:49 +000012#include <linux/cpu.h>
Colin Cross746a9d12011-02-10 02:08:32 -080013#include <linux/cpu_pm.h>
Will Deacon998de4a2012-05-11 17:42:37 +010014#include <linux/hardirq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/kernel.h>
Russell King90b44192010-12-18 10:59:49 +000016#include <linux/notifier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/signal.h>
18#include <linux/sched.h>
Russell King90b44192010-12-18 10:59:49 +000019#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/init.h>
Will Deacon24988142012-04-23 15:38:28 +010021#include <linux/uaccess.h>
22#include <linux/user.h>
Ard Biesheuvel73c132c2013-05-16 11:41:48 +020023#include <linux/export.h>
Russell Kingd6551e82006-06-21 13:31:52 +010024
Russell King15d07dc2012-03-28 18:30:01 +010025#include <asm/cp15.h>
Tony Lindgren5aaf2542010-07-01 13:41:05 +010026#include <asm/cputype.h>
David Howells9f97da72012-03-28 18:30:01 +010027#include <asm/system_info.h>
Russell Kingd6551e82006-06-21 13:31:52 +010028#include <asm/thread_notify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/vfp.h>
30
31#include "vfpinstr.h"
32#include "vfp.h"
33
34/*
35 * Our undef handlers (in entry.S)
36 */
37void vfp_testing_entry(void);
38void vfp_support_entry(void);
Russell King5d4cae52007-06-10 12:22:20 +010039void vfp_null_entry(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Russell King5d4cae52007-06-10 12:22:20 +010041void (*vfp_vector)(void) = vfp_null_entry;
Russell Kingaf61bdf2011-07-09 13:44:04 +010042
43/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070044 * Dual-use variable.
45 * Used in startup: set to non-zero if VFP checks fail
46 * After startup, holds VFP architecture
47 */
48unsigned int VFP_arch;
49
Russell King0d782dc2009-12-12 14:47:40 +000050/*
Russell Kingf8f2a852011-07-09 16:09:43 +010051 * The pointer to the vfpstate structure of the thread which currently
52 * owns the context held in the VFP hardware, or NULL if the hardware
53 * context is invalid.
54 *
55 * For UP, this is sufficient to tell which thread owns the VFP context.
56 * However, for SMP, we also need to check the CPU number stored in the
57 * saved state too to catch migrations.
58 */
59union vfp_state *vfp_current_hw_state[NR_CPUS];
60
61/*
62 * Is 'thread's most up to date state stored in this CPUs hardware?
63 * Must be called from non-preemptible context.
64 */
65static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread)
66{
67#ifdef CONFIG_SMP
68 if (thread->vfpstate.hard.cpu != cpu)
69 return false;
70#endif
71 return vfp_current_hw_state[cpu] == &thread->vfpstate;
72}
73
74/*
75 * Force a reload of the VFP context from the thread structure. We do
76 * this by ensuring that access to the VFP hardware is disabled, and
Santosh Shilimkar48af9fe2011-09-02 21:12:36 +053077 * clear vfp_current_hw_state. Must be called from non-preemptible context.
Russell Kingf8f2a852011-07-09 16:09:43 +010078 */
79static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
80{
81 if (vfp_state_in_hw(cpu, thread)) {
82 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
83 vfp_current_hw_state[cpu] = NULL;
84 }
85#ifdef CONFIG_SMP
86 thread->vfpstate.hard.cpu = NR_CPUS;
87#endif
88}
89
90/*
Russell King0d782dc2009-12-12 14:47:40 +000091 * Per-thread VFP initialization.
92 */
93static void vfp_thread_flush(struct thread_info *thread)
94{
95 union vfp_state *vfp = &thread->vfpstate;
96 unsigned int cpu;
97
Russell King19dad352011-07-09 17:41:33 +010098 /*
99 * Disable VFP to ensure we initialize it first. We must ensure
100 * that the modification of vfp_current_hw_state[] and hardware
101 * disable are done for the same CPU and without preemption.
102 *
103 * Do this first to ensure that preemption won't overwrite our
104 * state saving should access to the VFP be enabled at this point.
105 */
106 cpu = get_cpu();
107 if (vfp_current_hw_state[cpu] == vfp)
108 vfp_current_hw_state[cpu] = NULL;
109 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
110 put_cpu();
111
Russell King0d782dc2009-12-12 14:47:40 +0000112 memset(vfp, 0, sizeof(union vfp_state));
113
114 vfp->hard.fpexc = FPEXC_EN;
115 vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
Russell Kingf8f2a852011-07-09 16:09:43 +0100116#ifdef CONFIG_SMP
117 vfp->hard.cpu = NR_CPUS;
118#endif
Russell King0d782dc2009-12-12 14:47:40 +0000119}
120
Russell King797245f2009-12-18 14:34:43 +0000121static void vfp_thread_exit(struct thread_info *thread)
Russell King0d782dc2009-12-12 14:47:40 +0000122{
123 /* release case: Per-thread VFP cleanup. */
124 union vfp_state *vfp = &thread->vfpstate;
Russell King797245f2009-12-18 14:34:43 +0000125 unsigned int cpu = get_cpu();
Russell King0d782dc2009-12-12 14:47:40 +0000126
Russell Kingaf61bdf2011-07-09 13:44:04 +0100127 if (vfp_current_hw_state[cpu] == vfp)
128 vfp_current_hw_state[cpu] = NULL;
Russell King797245f2009-12-18 14:34:43 +0000129 put_cpu();
Russell King0d782dc2009-12-12 14:47:40 +0000130}
131
Catalin Marinasc98c0972011-04-06 16:17:17 +0100132static void vfp_thread_copy(struct thread_info *thread)
133{
134 struct thread_info *parent = current_thread_info();
135
136 vfp_sync_hwstate(parent);
137 thread->vfpstate = parent->vfpstate;
Russell Kingf8f2a852011-07-09 16:09:43 +0100138#ifdef CONFIG_SMP
139 thread->vfpstate.hard.cpu = NR_CPUS;
140#endif
Catalin Marinasc98c0972011-04-06 16:17:17 +0100141}
142
Russell King0d782dc2009-12-12 14:47:40 +0000143/*
144 * When this function is called with the following 'cmd's, the following
145 * is true while this function is being run:
146 * THREAD_NOFTIFY_SWTICH:
147 * - the previously running thread will not be scheduled onto another CPU.
148 * - the next thread to be run (v) will not be running on another CPU.
149 * - thread->cpu is the local CPU number
150 * - not preemptible as we're called in the middle of a thread switch
151 * THREAD_NOTIFY_FLUSH:
152 * - the thread (v) will be running on the local CPU, so
153 * v === current_thread_info()
154 * - thread->cpu is the local CPU number at the time it is accessed,
155 * but may change at any time.
156 * - we could be preempted if tree preempt rcu is enabled, so
157 * it is unsafe to use thread->cpu.
Russell King797245f2009-12-18 14:34:43 +0000158 * THREAD_NOTIFY_EXIT
159 * - the thread (v) will be running on the local CPU, so
160 * v === current_thread_info()
161 * - thread->cpu is the local CPU number at the time it is accessed,
162 * but may change at any time.
163 * - we could be preempted if tree preempt rcu is enabled, so
164 * it is unsafe to use thread->cpu.
Russell King0d782dc2009-12-12 14:47:40 +0000165 */
Russell Kingd6551e82006-06-21 13:31:52 +0100166static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167{
Russell Kingd6551e82006-06-21 13:31:52 +0100168 struct thread_info *thread = v;
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100169 u32 fpexc;
170#ifdef CONFIG_SMP
171 unsigned int cpu;
172#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100174 switch (cmd) {
175 case THREAD_NOTIFY_SWITCH:
176 fpexc = fmrx(FPEXC);
Catalin Marinasc6428462007-01-24 18:47:08 +0100177
178#ifdef CONFIG_SMP
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100179 cpu = thread->cpu;
Russell King0d782dc2009-12-12 14:47:40 +0000180
Catalin Marinasc6428462007-01-24 18:47:08 +0100181 /*
182 * On SMP, if VFP is enabled, save the old state in
183 * case the thread migrates to a different CPU. The
184 * restoring is done lazily.
185 */
Russell Kingf8f2a852011-07-09 16:09:43 +0100186 if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
Russell Kingaf61bdf2011-07-09 13:44:04 +0100187 vfp_save_state(vfp_current_hw_state[cpu], fpexc);
Catalin Marinasc6428462007-01-24 18:47:08 +0100188#endif
189
Russell King681a4992006-08-27 12:38:34 +0100190 /*
191 * Always disable VFP so we can lazily save/restore the
192 * old state.
193 */
Russell King228adef2007-07-18 09:37:10 +0100194 fmxr(FPEXC, fpexc & ~FPEXC_EN);
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100195 break;
Russell King681a4992006-08-27 12:38:34 +0100196
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100197 case THREAD_NOTIFY_FLUSH:
Russell King0d782dc2009-12-12 14:47:40 +0000198 vfp_thread_flush(thread);
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100199 break;
200
201 case THREAD_NOTIFY_EXIT:
Russell King797245f2009-12-18 14:34:43 +0000202 vfp_thread_exit(thread);
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100203 break;
Catalin Marinasc98c0972011-04-06 16:17:17 +0100204
205 case THREAD_NOTIFY_COPY:
206 vfp_thread_copy(thread);
207 break;
Catalin Marinas2e82669a2011-04-06 16:16:29 +0100208 }
Russell King681a4992006-08-27 12:38:34 +0100209
Russell Kingd6551e82006-06-21 13:31:52 +0100210 return NOTIFY_DONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211}
212
Russell Kingd6551e82006-06-21 13:31:52 +0100213static struct notifier_block vfp_notifier_block = {
214 .notifier_call = vfp_notifier,
215};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217/*
218 * Raise a SIGFPE for the current process.
219 * sicode describes the signal being raised.
220 */
Russell King2bbd7e92011-01-08 12:05:09 +0000221static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222{
223 siginfo_t info;
224
225 memset(&info, 0, sizeof(info));
226
227 info.si_signo = SIGFPE;
228 info.si_code = sicode;
Al Viro35d59fc2006-10-11 17:22:44 +0100229 info.si_addr = (void __user *)(instruction_pointer(regs) - 4);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
231 /*
232 * This is the same as NWFPE, because it's not clear what
233 * this is used for
234 */
235 current->thread.error_code = 0;
236 current->thread.trap_no = 6;
237
Russell Kingda411192005-06-29 23:02:02 +0100238 send_sig_info(SIGFPE, &info, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239}
240
Catalin Marinasc98929c2007-11-22 18:32:01 +0100241static void vfp_panic(char *reason, u32 inst)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242{
243 int i;
244
Nicolas Pitredc457072012-05-09 22:32:43 +0100245 pr_err("VFP: Error: %s\n", reason);
246 pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n",
Catalin Marinasc98929c2007-11-22 18:32:01 +0100247 fmrx(FPEXC), fmrx(FPSCR), inst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 for (i = 0; i < 32; i += 2)
Nicolas Pitredc457072012-05-09 22:32:43 +0100249 pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250 i, vfp_get_float(i), i+1, vfp_get_float(i+1));
251}
252
253/*
254 * Process bitmask of exception conditions.
255 */
256static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)
257{
258 int si_code = 0;
259
260 pr_debug("VFP: raising exceptions %08x\n", exceptions);
261
Daniel Jacobowitz7c6f2512006-08-27 12:42:08 +0100262 if (exceptions == VFP_EXCEPTION_ERROR) {
Catalin Marinasc98929c2007-11-22 18:32:01 +0100263 vfp_panic("unhandled bounce", inst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 vfp_raise_sigfpe(0, regs);
265 return;
266 }
267
268 /*
Catalin Marinasdbead402010-02-01 18:50:40 +0100269 * If any of the status flags are set, update the FPSCR.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 * Comparison instructions always return at least one of
271 * these flags set.
272 */
Catalin Marinasdbead402010-02-01 18:50:40 +0100273 if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
274 fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 fpscr |= exceptions;
277
278 fmxr(FPSCR, fpscr);
279
280#define RAISE(stat,en,sig) \
281 if (exceptions & stat && fpscr & en) \
282 si_code = sig;
283
284 /*
285 * These are arranged in priority order, least to highest.
286 */
Takashi Ohmasae0f205d2006-10-23 11:19:40 +0100287 RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
289 RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
290 RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
291 RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
292
293 if (si_code)
294 vfp_raise_sigfpe(si_code, regs);
295}
296
297/*
298 * Emulate a VFP instruction.
299 */
300static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
301{
Daniel Jacobowitz7c6f2512006-08-27 12:42:08 +0100302 u32 exceptions = VFP_EXCEPTION_ERROR;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
304 pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr);
305
306 if (INST_CPRTDO(inst)) {
307 if (!INST_CPRT(inst)) {
308 /*
309 * CPDO
310 */
311 if (vfp_single(inst)) {
312 exceptions = vfp_single_cpdo(inst, fpscr);
313 } else {
314 exceptions = vfp_double_cpdo(inst, fpscr);
315 }
316 } else {
317 /*
318 * A CPRT instruction can not appear in FPINST2, nor
319 * can it cause an exception. Therefore, we do not
320 * have to emulate it.
321 */
322 }
323 } else {
324 /*
325 * A CPDT instruction can not appear in FPINST2, nor can
326 * it cause an exception. Therefore, we do not have to
327 * emulate it.
328 */
329 }
Russell King928bd1b2006-04-25 20:41:27 +0100330 return exceptions & ~VFP_NAN_FLAG;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331}
332
333/*
334 * Package up a bounce condition.
335 */
Catalin Marinasc98929c2007-11-22 18:32:01 +0100336void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337{
Catalin Marinasc98929c2007-11-22 18:32:01 +0100338 u32 fpscr, orig_fpscr, fpsid, exceptions;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
340 pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
341
342 /*
Catalin Marinasc98929c2007-11-22 18:32:01 +0100343 * At this point, FPEXC can have the following configuration:
344 *
345 * EX DEX IXE
346 * 0 1 x - synchronous exception
347 * 1 x 0 - asynchronous exception
348 * 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later
349 * 0 0 1 - synchronous on VFP9 (non-standard subarch 1
350 * implementation), undefined otherwise
351 *
352 * Clear various bits and enable access to the VFP so we can
353 * handle the bounce.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 */
Catalin Marinasc98929c2007-11-22 18:32:01 +0100355 fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356
Catalin Marinasc98929c2007-11-22 18:32:01 +0100357 fpsid = fmrx(FPSID);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 orig_fpscr = fpscr = fmrx(FPSCR);
359
360 /*
Catalin Marinasc98929c2007-11-22 18:32:01 +0100361 * Check for the special VFP subarch 1 and FPSCR.IXE bit case
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 */
Catalin Marinasc98929c2007-11-22 18:32:01 +0100363 if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
364 && (fpscr & FPSCR_IXE)) {
365 /*
366 * Synchronous exception, emulate the trigger instruction
367 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 goto emulate;
369 }
370
Catalin Marinasc98929c2007-11-22 18:32:01 +0100371 if (fpexc & FPEXC_EX) {
Catalin Marinas85d69432009-05-30 14:00:18 +0100372#ifndef CONFIG_CPU_FEROCEON
Catalin Marinasc98929c2007-11-22 18:32:01 +0100373 /*
374 * Asynchronous exception. The instruction is read from FPINST
375 * and the interrupted instruction has to be restarted.
376 */
377 trigger = fmrx(FPINST);
378 regs->ARM_pc -= 4;
Catalin Marinas85d69432009-05-30 14:00:18 +0100379#endif
Catalin Marinasc98929c2007-11-22 18:32:01 +0100380 } else if (!(fpexc & FPEXC_DEX)) {
381 /*
382 * Illegal combination of bits. It can be caused by an
383 * unallocated VFP instruction but with FPSCR.IXE set and not
384 * on VFP subarch 1.
385 */
386 vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);
George G. Davisf2255be2009-04-01 20:27:18 +0100387 goto exit;
Catalin Marinasc98929c2007-11-22 18:32:01 +0100388 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
390 /*
Catalin Marinasc98929c2007-11-22 18:32:01 +0100391 * Modify fpscr to indicate the number of iterations remaining.
392 * If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates
393 * whether FPEXC.VECITR or FPSCR.LEN is used.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 */
Catalin Marinasc98929c2007-11-22 18:32:01 +0100395 if (fpexc & (FPEXC_EX | FPEXC_VV)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 u32 len;
397
398 len = fpexc + (1 << FPEXC_LENGTH_BIT);
399
400 fpscr &= ~FPSCR_LENGTH_MASK;
401 fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
402 }
403
404 /*
405 * Handle the first FP instruction. We used to take note of the
406 * FPEXC bounce reason, but this appears to be unreliable.
407 * Emulate the bounced instruction instead.
408 */
Catalin Marinasc98929c2007-11-22 18:32:01 +0100409 exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 if (exceptions)
Catalin Marinasc98929c2007-11-22 18:32:01 +0100411 vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
413 /*
Catalin Marinasc98929c2007-11-22 18:32:01 +0100414 * If there isn't a second FP instruction, exit now. Note that
415 * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 */
Russell King5e4ba612013-02-25 16:09:12 +0000417 if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
George G. Davisf2255be2009-04-01 20:27:18 +0100418 goto exit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
420 /*
421 * The barrier() here prevents fpinst2 being read
422 * before the condition above.
423 */
424 barrier();
425 trigger = fmrx(FPINST2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426
427 emulate:
Catalin Marinasc98929c2007-11-22 18:32:01 +0100428 exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 if (exceptions)
430 vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
George G. Davisf2255be2009-04-01 20:27:18 +0100431 exit:
432 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433}
Russell Kingefe90d22006-12-08 15:22:20 +0000434
Russell King8e140362007-01-02 23:40:30 +0000435static void vfp_enable(void *unused)
436{
Will Deacon998de4a2012-05-11 17:42:37 +0100437 u32 access;
438
439 BUG_ON(preemptible());
440 access = get_copro_access();
Russell King8e140362007-01-02 23:40:30 +0000441
442 /*
443 * Enable full access to VFP (cp10 and cp11)
444 */
445 set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11));
446}
447
Colin Cross746a9d12011-02-10 02:08:32 -0800448#ifdef CONFIG_CPU_PM
Rafael J. Wysocki328f5cc2011-04-22 22:02:33 +0200449static int vfp_pm_suspend(void)
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100450{
451 struct thread_info *ti = current_thread_info();
452 u32 fpexc = fmrx(FPEXC);
453
454 /* if vfp is on, then save state for resumption */
455 if (fpexc & FPEXC_EN) {
Nicolas Pitredc457072012-05-09 22:32:43 +0100456 pr_debug("%s: saving vfp state\n", __func__);
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100457 vfp_save_state(&ti->vfpstate, fpexc);
458
459 /* disable, just in case */
460 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
Colin Cross24b35522012-07-20 02:03:42 +0100461 } else if (vfp_current_hw_state[ti->cpu]) {
462#ifndef CONFIG_SMP
463 fmxr(FPEXC, fpexc | FPEXC_EN);
464 vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc);
465 fmxr(FPEXC, fpexc);
466#endif
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100467 }
468
469 /* clear any information we had about last context state */
Colin Crossa84b8952012-07-20 02:03:43 +0100470 vfp_current_hw_state[ti->cpu] = NULL;
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100471
472 return 0;
473}
474
Rafael J. Wysocki328f5cc2011-04-22 22:02:33 +0200475static void vfp_pm_resume(void)
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100476{
477 /* ensure we have access to the vfp */
478 vfp_enable(NULL);
479
480 /* and disable it to ensure the next usage restores the state */
481 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100482}
483
Colin Cross746a9d12011-02-10 02:08:32 -0800484static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd,
485 void *v)
486{
487 switch (cmd) {
488 case CPU_PM_ENTER:
489 vfp_pm_suspend();
490 break;
491 case CPU_PM_ENTER_FAILED:
492 case CPU_PM_EXIT:
493 vfp_pm_resume();
494 break;
495 }
496 return NOTIFY_OK;
497}
498
499static struct notifier_block vfp_cpu_pm_notifier_block = {
500 .notifier_call = vfp_cpu_pm_notifier,
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100501};
502
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100503static void vfp_pm_init(void)
504{
Colin Cross746a9d12011-02-10 02:08:32 -0800505 cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block);
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100506}
507
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100508#else
509static inline void vfp_pm_init(void) { }
Colin Cross746a9d12011-02-10 02:08:32 -0800510#endif /* CONFIG_CPU_PM */
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100511
Russell Kingf8f2a852011-07-09 16:09:43 +0100512/*
513 * Ensure that the VFP state stored in 'thread->vfpstate' is up to date
514 * with the hardware state.
515 */
Russell Kingad187f92010-02-06 11:36:23 +0000516void vfp_sync_hwstate(struct thread_info *thread)
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100517{
518 unsigned int cpu = get_cpu();
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100519
Russell Kingf8f2a852011-07-09 16:09:43 +0100520 if (vfp_state_in_hw(cpu, thread)) {
Russell King54cb3db2010-02-06 11:27:45 +0000521 u32 fpexc = fmrx(FPEXC);
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100522
Russell King54cb3db2010-02-06 11:27:45 +0000523 /*
524 * Save the last VFP state on this CPU.
525 */
526 fmxr(FPEXC, fpexc | FPEXC_EN);
527 vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
Russell Kingad187f92010-02-06 11:36:23 +0000528 fmxr(FPEXC, fpexc);
529 }
530
531 put_cpu();
532}
533
Russell Kingf8f2a852011-07-09 16:09:43 +0100534/* Ensure that the thread reloads the hardware VFP state on the next use. */
Russell Kingad187f92010-02-06 11:36:23 +0000535void vfp_flush_hwstate(struct thread_info *thread)
536{
537 unsigned int cpu = get_cpu();
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100538
Russell Kingf8f2a852011-07-09 16:09:43 +0100539 vfp_force_reload(cpu, thread);
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100540
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100541 put_cpu();
542}
Catalin Marinas3d1228e2009-02-11 13:12:56 +0100543
Russell King90b44192010-12-18 10:59:49 +0000544/*
Will Deacon24988142012-04-23 15:38:28 +0100545 * Save the current VFP state into the provided structures and prepare
546 * for entry into a new function (signal handler).
547 */
548int vfp_preserve_user_clear_hwstate(struct user_vfp __user *ufp,
549 struct user_vfp_exc __user *ufp_exc)
550{
551 struct thread_info *thread = current_thread_info();
552 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
553 int err = 0;
554
555 /* Ensure that the saved hwstate is up-to-date. */
556 vfp_sync_hwstate(thread);
557
558 /*
559 * Copy the floating point registers. There can be unused
560 * registers see asm/hwcap.h for details.
561 */
562 err |= __copy_to_user(&ufp->fpregs, &hwstate->fpregs,
563 sizeof(hwstate->fpregs));
564 /*
565 * Copy the status and control register.
566 */
567 __put_user_error(hwstate->fpscr, &ufp->fpscr, err);
568
569 /*
570 * Copy the exception registers.
571 */
572 __put_user_error(hwstate->fpexc, &ufp_exc->fpexc, err);
573 __put_user_error(hwstate->fpinst, &ufp_exc->fpinst, err);
574 __put_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err);
575
576 if (err)
577 return -EFAULT;
Will Deaconff9a1842012-04-23 15:42:16 +0100578
579 /* Ensure that VFP is disabled. */
580 vfp_flush_hwstate(thread);
581
582 /*
583 * As per the PCS, clear the length and stride bits for function
584 * entry.
585 */
586 hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK);
Will Deacon24988142012-04-23 15:38:28 +0100587 return 0;
588}
589
590/* Sanitise and restore the current VFP state from the provided structures. */
591int vfp_restore_user_hwstate(struct user_vfp __user *ufp,
592 struct user_vfp_exc __user *ufp_exc)
593{
594 struct thread_info *thread = current_thread_info();
595 struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
596 unsigned long fpexc;
597 int err = 0;
598
Will Deacon56cb2482012-05-15 15:51:54 +0100599 /* Disable VFP to avoid corrupting the new thread state. */
600 vfp_flush_hwstate(thread);
Will Deacon24988142012-04-23 15:38:28 +0100601
602 /*
603 * Copy the floating point registers. There can be unused
604 * registers see asm/hwcap.h for details.
605 */
606 err |= __copy_from_user(&hwstate->fpregs, &ufp->fpregs,
607 sizeof(hwstate->fpregs));
608 /*
609 * Copy the status and control register.
610 */
611 __get_user_error(hwstate->fpscr, &ufp->fpscr, err);
612
613 /*
614 * Sanitise and restore the exception registers.
615 */
616 __get_user_error(fpexc, &ufp_exc->fpexc, err);
617
618 /* Ensure the VFP is enabled. */
619 fpexc |= FPEXC_EN;
620
621 /* Ensure FPINST2 is invalid and the exception flag is cleared. */
622 fpexc &= ~(FPEXC_EX | FPEXC_FP2V);
623 hwstate->fpexc = fpexc;
624
625 __get_user_error(hwstate->fpinst, &ufp_exc->fpinst, err);
626 __get_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err);
627
628 return err ? -EFAULT : 0;
629}
630
631/*
Russell King90b44192010-12-18 10:59:49 +0000632 * VFP hardware can lose all context when a CPU goes offline.
Russell King74c25be2011-01-31 14:43:25 +0000633 * As we will be running in SMP mode with CPU hotplug, we will save the
634 * hardware state at every thread switch. We clear our held state when
635 * a CPU has been killed, indicating that the VFP hardware doesn't contain
636 * a threads VFP state. When a CPU starts up, we re-enable access to the
637 * VFP hardware.
Russell King90b44192010-12-18 10:59:49 +0000638 *
639 * Both CPU_DYING and CPU_STARTING are called on the CPU which
640 * is being offlined/onlined.
641 */
642static int vfp_hotplug(struct notifier_block *b, unsigned long action,
643 void *hcpu)
644{
645 if (action == CPU_DYING || action == CPU_DYING_FROZEN) {
Russell Kingf8f2a852011-07-09 16:09:43 +0100646 vfp_force_reload((long)hcpu, current_thread_info());
Russell King90b44192010-12-18 10:59:49 +0000647 } else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
648 vfp_enable(NULL);
649 return NOTIFY_OK;
650}
Russell King8e140362007-01-02 23:40:30 +0000651
Ard Biesheuvelab3da152013-05-24 16:23:28 +0200652void vfp_kmode_exception(void)
653{
654 /*
655 * If we reach this point, a floating point exception has been raised
656 * while running in kernel mode. If the NEON/VFP unit was enabled at the
657 * time, it means a VFP instruction has been issued that requires
658 * software assistance to complete, something which is not currently
659 * supported in kernel mode.
660 * If the NEON/VFP unit was disabled, and the location pointed to below
661 * is properly preceded by a call to kernel_neon_begin(), something has
662 * caused the task to be scheduled out and back in again. In this case,
663 * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should
664 * be helpful in localizing the problem.
665 */
666 if (fmrx(FPEXC) & FPEXC_EN)
667 pr_crit("BUG: unsupported FP instruction in kernel mode\n");
668 else
669 pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n");
670}
671
Ard Biesheuvel73c132c2013-05-16 11:41:48 +0200672#ifdef CONFIG_KERNEL_MODE_NEON
673
674/*
675 * Kernel-side NEON support functions
676 */
677void kernel_neon_begin(void)
678{
679 struct thread_info *thread = current_thread_info();
680 unsigned int cpu;
681 u32 fpexc;
682
683 /*
684 * Kernel mode NEON is only allowed outside of interrupt context
685 * with preemption disabled. This will make sure that the kernel
686 * mode NEON register contents never need to be preserved.
687 */
688 BUG_ON(in_interrupt());
689 cpu = get_cpu();
690
691 fpexc = fmrx(FPEXC) | FPEXC_EN;
692 fmxr(FPEXC, fpexc);
693
694 /*
695 * Save the userland NEON/VFP state. Under UP,
696 * the owner could be a task other than 'current'
697 */
698 if (vfp_state_in_hw(cpu, thread))
699 vfp_save_state(&thread->vfpstate, fpexc);
700#ifndef CONFIG_SMP
701 else if (vfp_current_hw_state[cpu] != NULL)
702 vfp_save_state(vfp_current_hw_state[cpu], fpexc);
703#endif
704 vfp_current_hw_state[cpu] = NULL;
705}
706EXPORT_SYMBOL(kernel_neon_begin);
707
708void kernel_neon_end(void)
709{
710 /* Disable the NEON/VFP unit. */
711 fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
712 put_cpu();
713}
714EXPORT_SYMBOL(kernel_neon_end);
715
716#endif /* CONFIG_KERNEL_MODE_NEON */
717
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718/*
719 * VFP support code initialisation.
720 */
721static int __init vfp_init(void)
722{
723 unsigned int vfpsid;
Russell Kingefe90d22006-12-08 15:22:20 +0000724 unsigned int cpu_arch = cpu_architecture();
Russell Kingefe90d22006-12-08 15:22:20 +0000725
Catalin Marinasc98929c2007-11-22 18:32:01 +0100726 if (cpu_arch >= CPU_ARCH_ARMv6)
Will Deacon998de4a2012-05-11 17:42:37 +0100727 on_each_cpu(vfp_enable, NULL, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
729 /*
730 * First check that there is a VFP that we can use.
731 * The handler is already setup to just log calls, so
732 * we just need to read the VFPSID register.
733 */
Russell King5d4cae52007-06-10 12:22:20 +0100734 vfp_vector = vfp_testing_entry;
Tzachi Perelsteinb9338a72007-09-09 14:24:59 +0100735 barrier();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 vfpsid = fmrx(FPSID);
Russell King8e140362007-01-02 23:40:30 +0000737 barrier();
Russell King5d4cae52007-06-10 12:22:20 +0100738 vfp_vector = vfp_null_entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Nicolas Pitredc457072012-05-09 22:32:43 +0100740 pr_info("VFP support v0.3: ");
Catalin Marinasc98929c2007-11-22 18:32:01 +0100741 if (VFP_arch)
Nicolas Pitredc457072012-05-09 22:32:43 +0100742 pr_cont("not present\n");
Catalin Marinasc98929c2007-11-22 18:32:01 +0100743 else if (vfpsid & FPSID_NODOUBLE) {
Nicolas Pitredc457072012-05-09 22:32:43 +0100744 pr_cont("no double precision support\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745 } else {
Russell King90b44192010-12-18 10:59:49 +0000746 hotcpu_notifier(vfp_hotplug, 0);
747
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; /* Extract the architecture version */
Nicolas Pitredc457072012-05-09 22:32:43 +0100749 pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT,
751 (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT,
752 (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT,
753 (vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT,
754 (vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT);
Russell Kingefe90d22006-12-08 15:22:20 +0000755
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 vfp_vector = vfp_support_entry;
Russell Kingd6551e82006-06-21 13:31:52 +0100757
758 thread_register_notifier(&vfp_notifier_block);
Ben Dooksfc0b7a22008-12-18 12:26:54 +0100759 vfp_pm_init();
Russell Kingefe90d22006-12-08 15:22:20 +0000760
761 /*
762 * We detected VFP, and the support code is
763 * in place; report VFP support to userspace.
764 */
765 elf_hwcap |= HWCAP_VFP;
Catalin Marinas7279dc32009-02-11 13:13:56 +0100766#ifdef CONFIG_VFPv3
Catalin Marinas325ffc32010-03-26 15:44:57 +0100767 if (VFP_arch >= 2) {
Catalin Marinas7279dc32009-02-11 13:13:56 +0100768 elf_hwcap |= HWCAP_VFPv3;
769
770 /*
Paul Walmsley39141dd2012-10-23 20:32:59 +0100771 * Check for VFPv3 D16 and VFPv4 D16. CPUs in
772 * this configuration only have 16 x 64bit
773 * registers.
Catalin Marinas7279dc32009-02-11 13:13:56 +0100774 */
775 if (((fmrx(MVFR0) & MVFR0_A_SIMD_MASK)) == 1)
Paul Walmsley39141dd2012-10-23 20:32:59 +0100776 elf_hwcap |= HWCAP_VFPv3D16; /* also v4-D16 */
777 else
778 elf_hwcap |= HWCAP_VFPD32;
Catalin Marinas7279dc32009-02-11 13:13:56 +0100779 }
780#endif
Catalin Marinas2bedbdf2008-11-06 13:23:07 +0000781 /*
782 * Check for the presence of the Advanced SIMD
783 * load/store instructions, integer and single
Tony Lindgren5aaf2542010-07-01 13:41:05 +0100784 * precision floating point operations. Only check
785 * for NEON if the hardware has the MVFR registers.
Catalin Marinas2bedbdf2008-11-06 13:23:07 +0000786 */
Tony Lindgren5aaf2542010-07-01 13:41:05 +0100787 if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
Will Deacon18b9dc12011-06-08 11:42:43 +0100788#ifdef CONFIG_NEON
Tony Lindgren5aaf2542010-07-01 13:41:05 +0100789 if ((fmrx(MVFR1) & 0x000fff00) == 0x00011100)
790 elf_hwcap |= HWCAP_NEON;
Catalin Marinas2bedbdf2008-11-06 13:23:07 +0000791#endif
Will Deacon3d9fb002012-08-03 17:24:14 +0100792#ifdef CONFIG_VFPv3
Will Deacon18b9dc12011-06-08 11:42:43 +0100793 if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000)
794 elf_hwcap |= HWCAP_VFPv4;
Will Deacon3d9fb002012-08-03 17:24:14 +0100795#endif
Will Deacon18b9dc12011-06-08 11:42:43 +0100796 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 }
798 return 0;
799}
800
Ard Biesheuvel0773d732013-05-22 10:38:53 +0200801core_initcall(vfp_init);