blob: 46ef3fd0663b5c1dfca0ca3b0d175c7a5d4fe990 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * arch/s390/mm/fault.c
3 *
4 * S390 version
5 * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation
6 * Author(s): Hartmut Penner (hp@de.ibm.com)
7 * Ulrich Weigand (uweigand@de.ibm.com)
8 *
9 * Derived from "arch/i386/mm/fault.c"
10 * Copyright (C) 1995 Linus Torvalds
11 */
12
Heiko Carstens052ff462011-01-05 12:47:28 +010013#include <linux/kernel_stat.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020014#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/signal.h>
16#include <linux/sched.h>
17#include <linux/kernel.h>
18#include <linux/errno.h>
19#include <linux/string.h>
20#include <linux/types.h>
21#include <linux/ptrace.h>
22#include <linux/mman.h>
23#include <linux/mm.h>
Heiko Carstens77575912009-06-12 10:26:25 +020024#include <linux/compat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/smp.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070026#include <linux/kdebug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/init.h>
28#include <linux/console.h>
29#include <linux/module.h>
30#include <linux/hardirq.h>
Michael Grundy4ba069b2006-09-20 15:58:39 +020031#include <linux/kprobes.h>
Martin Schwidefskybe5ec362007-04-27 16:01:44 +020032#include <linux/uaccess.h>
Gerald Schaefer53492b12008-04-30 13:38:46 +020033#include <linux/hugetlb.h>
Heiko Carstenscbb870c2010-02-26 22:37:43 +010034#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/pgtable.h>
Heiko Carstensd7b250e2011-05-26 09:48:24 +020036#include <asm/irq.h>
Martin Schwidefsky6252d702008-02-09 18:24:37 +010037#include <asm/mmu_context.h>
David Howellsa0616cd2012-03-28 18:30:02 +010038#include <asm/facility.h>
Heiko Carstensa8061702008-04-17 07:46:26 +020039#include "../kernel/entry.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080041#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#define __FAIL_ADDR_MASK 0x7ffff000
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#define __SUBCODE_MASK 0x0200
44#define __PF_RES_FIELD 0ULL
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080045#else /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070046#define __FAIL_ADDR_MASK -4096L
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#define __SUBCODE_MASK 0x0600
48#define __PF_RES_FIELD 0x8000000000000000ULL
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080049#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
Martin Schwidefsky50d72802009-12-07 12:51:45 +010051#define VM_FAULT_BADCONTEXT 0x010000
52#define VM_FAULT_BADMAP 0x020000
53#define VM_FAULT_BADACCESS 0x040000
54
Martin Schwidefsky92f842e2010-10-25 16:10:13 +020055static unsigned long store_indication;
56
57void fault_init(void)
58{
Martin Schwidefsky14375bc2010-10-25 16:10:51 +020059 if (test_facility(2) && test_facility(75))
Martin Schwidefsky92f842e2010-10-25 16:10:13 +020060 store_indication = 0xc00;
61}
62
Martin Schwidefsky7ecb3442009-12-07 12:51:44 +010063static inline int notify_page_fault(struct pt_regs *regs)
Martin Schwidefsky10c10312007-04-27 16:01:43 +020064{
Christoph Hellwig33464e32007-05-04 18:47:46 +020065 int ret = 0;
66
67 /* kprobe_running() needs smp_processor_id() */
Heiko Carstens22e0a042010-02-26 22:37:45 +010068 if (kprobes_built_in() && !user_mode(regs)) {
Christoph Hellwig33464e32007-05-04 18:47:46 +020069 preempt_disable();
70 if (kprobe_running() && kprobe_fault_handler(regs, 14))
71 ret = 1;
72 preempt_enable();
73 }
Christoph Hellwig33464e32007-05-04 18:47:46 +020074 return ret;
Michael Grundy4ba069b2006-09-20 15:58:39 +020075}
Michael Grundy4ba069b2006-09-20 15:58:39 +020076
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78/*
79 * Unlock any spinlocks which will prevent us from getting the
Kirill Korotaevcefc8be2007-02-10 01:46:18 -080080 * message out.
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 */
82void bust_spinlocks(int yes)
83{
84 if (yes) {
85 oops_in_progress = 1;
86 } else {
87 int loglevel_save = console_loglevel;
88 console_unblank();
89 oops_in_progress = 0;
90 /*
91 * OK, the message is on the console. Now we call printk()
92 * without oops_in_progress set so that printk will give klogd
93 * a poke. Hold onto your hats...
94 */
95 console_loglevel = 15;
96 printk(" ");
97 console_loglevel = loglevel_save;
98 }
99}
100
101/*
Gerald Schaefer482b05d2007-03-05 23:35:54 +0100102 * Returns the address space associated with the fault.
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100103 * Returns 0 for kernel space and 1 for user space.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100105static inline int user_space_fault(unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106{
107 /*
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100108 * The lowest two bits of the translation exception
109 * identification indicate which paging table was used.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100111 trans_exc_code &= 3;
112 if (trans_exc_code == 2)
113 /* Access via secondary space, set_fs setting decides */
114 return current->thread.mm_segment.ar4;
Martin Schwidefskyb11b5332009-12-07 12:51:43 +0100115 if (user_mode == HOME_SPACE_MODE)
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100116 /* User space if the access has been done via home space. */
117 return trans_exc_code == 3;
118 /*
119 * If the user space is not the home space the kernel runs in home
120 * space. Access via secondary space has already been covered,
121 * access via primary space or access register is from user space
122 * and access via home space is from the kernel.
123 */
124 return trans_exc_code != 3;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125}
126
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100127static inline void report_user_fault(struct pt_regs *regs, long signr)
Heiko Carstensab3c68e2010-05-17 10:00:21 +0200128{
129 if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
130 return;
131 if (!unhandled_signal(current, signr))
132 return;
133 if (!printk_ratelimit())
134 return;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100135 printk(KERN_ALERT "User process fault: interruption code 0x%X ",
136 regs->int_code);
Heiko Carstensab3c68e2010-05-17 10:00:21 +0200137 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN);
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100138 printk(KERN_CONT "\n");
139 printk(KERN_ALERT "failing address: %lX\n",
140 regs->int_parm_long & __FAIL_ADDR_MASK);
Heiko Carstensab3c68e2010-05-17 10:00:21 +0200141 show_regs(regs);
142}
143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
145 * Send SIGSEGV to task. This is an external routine
146 * to keep the stack usage of do_page_fault small.
147 */
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100148static noinline void do_sigsegv(struct pt_regs *regs, int si_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149{
150 struct siginfo si;
151
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100152 report_user_fault(regs, SIGSEGV);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 si.si_signo = SIGSEGV;
154 si.si_code = si_code;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100155 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 force_sig_info(SIGSEGV, &si, current);
157}
158
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100159static noinline void do_no_context(struct pt_regs *regs)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200160{
161 const struct exception_table_entry *fixup;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100162 unsigned long address;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200163
164 /* Are we prepared to handle this kernel fault? */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100165 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200166 if (fixup) {
167 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
168 return;
169 }
170
171 /*
172 * Oops. The kernel tried to access some bad page. We'll have to
173 * terminate things with extreme prejudice.
174 */
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100175 address = regs->int_parm_long & __FAIL_ADDR_MASK;
176 if (!user_space_fault(regs->int_parm_long))
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200177 printk(KERN_ALERT "Unable to handle kernel pointer dereference"
178 " at virtual kernel address %p\n", (void *)address);
179 else
180 printk(KERN_ALERT "Unable to handle kernel paging request"
181 " at virtual user address %p\n", (void *)address);
182
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100183 die(regs, "Oops");
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200184 do_exit(SIGKILL);
185}
186
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100187static noinline void do_low_address(struct pt_regs *regs)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200188{
189 /* Low-address protection hit in kernel mode means
190 NULL pointer write access in kernel mode. */
191 if (regs->psw.mask & PSW_MASK_PSTATE) {
192 /* Low-address protection hit in user mode 'cannot happen'. */
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100193 die (regs, "Low-address protection");
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200194 do_exit(SIGKILL);
195 }
196
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100197 do_no_context(regs);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200198}
199
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100200static noinline void do_sigbus(struct pt_regs *regs)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200201{
202 struct task_struct *tsk = current;
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200203 struct siginfo si;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200204
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200205 /*
206 * Send a sigbus, regardless of whether we were in kernel
207 * or user mode.
208 */
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200209 si.si_signo = SIGBUS;
210 si.si_errno = 0;
211 si.si_code = BUS_ADRERR;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100212 si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200213 force_sig_info(SIGBUS, &si, tsk);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200214}
215
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100216static noinline void do_fault_error(struct pt_regs *regs, int fault)
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100217{
218 int si_code;
219
220 switch (fault) {
221 case VM_FAULT_BADACCESS:
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100222 case VM_FAULT_BADMAP:
223 /* Bad memory access. Check if it is kernel or user space. */
224 if (regs->psw.mask & PSW_MASK_PSTATE) {
225 /* User mode accesses just cause a SIGSEGV */
226 si_code = (fault == VM_FAULT_BADMAP) ?
227 SEGV_MAPERR : SEGV_ACCERR;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100228 do_sigsegv(regs, si_code);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100229 return;
230 }
231 case VM_FAULT_BADCONTEXT:
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100232 do_no_context(regs);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100233 break;
234 default: /* fault & VM_FAULT_ERROR */
Heiko Carstens99583182011-05-26 09:48:29 +0200235 if (fault & VM_FAULT_OOM) {
236 if (!(regs->psw.mask & PSW_MASK_PSTATE))
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100237 do_no_context(regs);
Heiko Carstens99583182011-05-26 09:48:29 +0200238 else
239 pagefault_out_of_memory();
240 } else if (fault & VM_FAULT_SIGBUS) {
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100241 /* Kernel mode? Handle exceptions or die */
242 if (!(regs->psw.mask & PSW_MASK_PSTATE))
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100243 do_no_context(regs);
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200244 else
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100245 do_sigbus(regs);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100246 } else
247 BUG();
248 break;
249 }
250}
251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252/*
253 * This routine handles page faults. It determines the address,
254 * and the problem, and then passes it off to one of the appropriate
255 * routines.
256 *
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100257 * interruption code (int_code):
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 * 04 Protection -> Write-Protection (suprression)
259 * 10 Segment translation -> Not present (nullification)
260 * 11 Page translation -> Not present (nullification)
261 * 3b Region third trans. -> Not present (nullification)
262 */
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100263static inline int do_exception(struct pt_regs *regs, int access)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264{
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200265 struct task_struct *tsk;
266 struct mm_struct *mm;
267 struct vm_area_struct *vma;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100268 unsigned long trans_exc_code;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200269 unsigned long address;
Heiko Carstens33ce6142011-05-26 09:48:30 +0200270 unsigned int flags;
271 int fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272
Martin Schwidefsky7ecb3442009-12-07 12:51:44 +0100273 if (notify_page_fault(regs))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100274 return 0;
Michael Grundy4ba069b2006-09-20 15:58:39 +0200275
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200276 tsk = current;
277 mm = tsk->mm;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100278 trans_exc_code = regs->int_parm_long;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 /*
281 * Verify that the fault happened in user space, that
282 * we are not in an interrupt and that there is a
283 * user context.
284 */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100285 fault = VM_FAULT_BADCONTEXT;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100286 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100287 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100289 address = trans_exc_code & __FAIL_ADDR_MASK;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200290 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
Heiko Carstens33ce6142011-05-26 09:48:30 +0200291 flags = FAULT_FLAG_ALLOW_RETRY;
292 if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400)
293 flags |= FAULT_FLAG_WRITE;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200294 down_read(&mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200296#ifdef CONFIG_PGSTE
297 if (test_tsk_thread_flag(current, TIF_SIE) && S390_lowcore.gmap) {
Carsten Otte499069e2011-10-30 15:17:02 +0100298 address = __gmap_fault(address,
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200299 (struct gmap *) S390_lowcore.gmap);
300 if (address == -EFAULT) {
301 fault = VM_FAULT_BADMAP;
302 goto out_up;
303 }
304 if (address == -ENOMEM) {
305 fault = VM_FAULT_OOM;
306 goto out_up;
307 }
308 }
309#endif
310
311retry:
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100312 fault = VM_FAULT_BADMAP;
Gerald Schaefer482b05d2007-03-05 23:35:54 +0100313 vma = find_vma(mm, address);
314 if (!vma)
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100315 goto out_up;
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100316
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100317 if (unlikely(vma->vm_start > address)) {
318 if (!(vma->vm_flags & VM_GROWSDOWN))
319 goto out_up;
320 if (expand_stack(vma, address))
321 goto out_up;
322 }
323
324 /*
325 * Ok, we have a good vm_area for this memory access, so
326 * we can handle it..
327 */
328 fault = VM_FAULT_BADACCESS;
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100329 if (unlikely(!(vma->vm_flags & access)))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100330 goto out_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
Gerald Schaefer53492b12008-04-30 13:38:46 +0200332 if (is_vm_hugetlb_page(vma))
333 address &= HPAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 /*
335 * If for any reason at all we couldn't handle the fault,
336 * make sure we exit gracefully rather than endlessly redo
337 * the fault.
338 */
Heiko Carstens33ce6142011-05-26 09:48:30 +0200339 fault = handle_mm_fault(mm, vma, address, flags);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100340 if (unlikely(fault & VM_FAULT_ERROR))
341 goto out_up;
342
Heiko Carstens33ce6142011-05-26 09:48:30 +0200343 /*
344 * Major/minor page fault accounting is only done on the
345 * initial attempt. If we go through a retry, it is extremely
346 * likely that the page will be found in page cache at that point.
347 */
348 if (flags & FAULT_FLAG_ALLOW_RETRY) {
349 if (fault & VM_FAULT_MAJOR) {
350 tsk->maj_flt++;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200351 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
Heiko Carstens33ce6142011-05-26 09:48:30 +0200352 regs, address);
353 } else {
354 tsk->min_flt++;
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200355 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
Heiko Carstens33ce6142011-05-26 09:48:30 +0200356 regs, address);
357 }
358 if (fault & VM_FAULT_RETRY) {
359 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
360 * of starvation. */
361 flags &= ~FAULT_FLAG_ALLOW_RETRY;
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200362 down_read(&mm->mmap_sem);
Heiko Carstens33ce6142011-05-26 09:48:30 +0200363 goto retry;
364 }
Heiko Carstensbde69af2009-09-11 10:29:06 +0200365 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 /*
367 * The instruction that caused the program check will
368 * be repeated. Don't signal single step via SIGTRAP.
369 */
Martin Schwidefsky5e9a2692011-01-05 12:48:10 +0100370 clear_tsk_thread_flag(tsk, TIF_PER_TRAP);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100371 fault = 0;
372out_up:
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200373 up_read(&mm->mmap_sem);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100374out:
375 return fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376}
377
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100378void __kprobes do_protection_exception(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379{
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100380 unsigned long trans_exc_code;
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100381 int fault;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100382
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100383 trans_exc_code = regs->int_parm_long;
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300384 /* Protection exception is suppressing, decrement psw address. */
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100385 regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200386 /*
387 * Check for low-address protection. This needs to be treated
388 * as a special case because the translation exception code
389 * field is not guaranteed to contain valid data in this case.
390 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100391 if (unlikely(!(trans_exc_code & 4))) {
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100392 do_low_address(regs);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200393 return;
394 }
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100395 fault = do_exception(regs, VM_WRITE);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100396 if (unlikely(fault))
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100397 do_fault_error(regs, fault);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398}
399
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100400void __kprobes do_dat_exception(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401{
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100402 int access, fault;
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100403
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100404 access = VM_READ | VM_EXEC | VM_WRITE;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100405 fault = do_exception(regs, access);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100406 if (unlikely(fault))
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100407 do_fault_error(regs, fault);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408}
409
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100410#ifdef CONFIG_64BIT
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100411void __kprobes do_asce_exception(struct pt_regs *regs)
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100412{
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100413 struct mm_struct *mm = current->mm;
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100414 struct vm_area_struct *vma;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100415 unsigned long trans_exc_code;
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100416
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100417 trans_exc_code = regs->int_parm_long;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100418 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100419 goto no_context;
420
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100421 down_read(&mm->mmap_sem);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100422 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100423 up_read(&mm->mmap_sem);
424
425 if (vma) {
426 update_mm(mm, current);
427 return;
428 }
429
430 /* User mode accesses just cause a SIGSEGV */
431 if (regs->psw.mask & PSW_MASK_PSTATE) {
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100432 do_sigsegv(regs, SEGV_MAPERR);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100433 return;
434 }
435
436no_context:
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100437 do_no_context(regs);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100438}
439#endif
440
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200441int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write)
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100442{
443 struct pt_regs regs;
444 int access, fault;
445
Martin Schwidefskyb50511e2011-10-30 15:16:50 +0100446 regs.psw.mask = psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_MCHECK;
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100447 if (!irqs_disabled())
448 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT;
449 regs.psw.addr = (unsigned long) __builtin_return_address(0);
450 regs.psw.addr |= PSW_ADDR_AMODE;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100451 regs.int_code = pgm_int_code;
452 regs.int_parm_long = (uaddr & PAGE_MASK) | 2;
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200453 access = write ? VM_WRITE : VM_READ;
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100454 fault = do_exception(&regs, access);
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100455 if (unlikely(fault)) {
Heiko Carstens99583182011-05-26 09:48:29 +0200456 if (fault & VM_FAULT_OOM)
457 return -EFAULT;
458 else if (fault & VM_FAULT_SIGBUS)
Martin Schwidefskyaa33c8c2011-12-27 11:27:18 +0100459 do_sigbus(&regs);
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100460 }
461 return fault ? -EFAULT : 0;
462}
463
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464#ifdef CONFIG_PFAULT
465/*
466 * 'pfault' pseudo page faults routines.
467 */
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100468static int pfault_disable;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
470static int __init nopfault(char *str)
471{
472 pfault_disable = 1;
473 return 1;
474}
475
476__setup("nopfault", nopfault);
477
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200478struct pfault_refbk {
479 u16 refdiagc;
480 u16 reffcode;
481 u16 refdwlen;
482 u16 refversn;
483 u64 refgaddr;
484 u64 refselmk;
485 u64 refcmpmk;
486 u64 reserved;
487} __attribute__ ((packed, aligned(8)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488
489int pfault_init(void)
490{
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200491 struct pfault_refbk refbk = {
492 .refdiagc = 0x258,
493 .reffcode = 0,
494 .refdwlen = 5,
495 .refversn = 2,
496 .refgaddr = __LC_CURRENT_PID,
497 .refselmk = 1ULL << 48,
498 .refcmpmk = 1ULL << 48,
499 .reserved = __PF_RES_FIELD };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 int rc;
501
Carsten Ottef32269a2011-12-27 11:27:11 +0100502 if (pfault_disable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 return -1;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200504 asm volatile(
505 " diag %1,%0,0x258\n"
506 "0: j 2f\n"
507 "1: la %0,8\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 "2:\n"
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200509 EX_TABLE(0b,1b)
510 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 return rc;
512}
513
514void pfault_fini(void)
515{
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200516 struct pfault_refbk refbk = {
517 .refdiagc = 0x258,
518 .reffcode = 1,
519 .refdwlen = 5,
520 .refversn = 2,
521 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Carsten Ottef32269a2011-12-27 11:27:11 +0100523 if (pfault_disable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 return;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200525 asm volatile(
526 " diag %0,0,0x258\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 "0:\n"
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200528 EX_TABLE(0b,0b)
529 : : "a" (&refbk), "m" (refbk) : "cc");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530}
531
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200532static DEFINE_SPINLOCK(pfault_lock);
533static LIST_HEAD(pfault_list);
534
Heiko Carstensfde15c32012-03-11 11:59:31 -0400535static void pfault_interrupt(struct ext_code ext_code,
Martin Schwidefskyf6649a72010-10-25 16:10:38 +0200536 unsigned int param32, unsigned long param64)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537{
538 struct task_struct *tsk;
539 __u16 subcode;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200540 pid_t pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541
542 /*
543 * Get the external interruption subcode & pfault
544 * initial/completion signal bit. VM stores this
545 * in the 'cpu address' field associated with the
546 * external interrupt.
547 */
Heiko Carstensfde15c32012-03-11 11:59:31 -0400548 subcode = ext_code.subcode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 if ((subcode & 0xff00) != __SUBCODE_MASK)
550 return;
Heiko Carstensa9851832011-04-29 10:42:19 +0200551 kstat_cpu(smp_processor_id()).irqs[EXTINT_PFL]++;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200552 if (subcode & 0x0080) {
553 /* Get the token (= pid of the affected task). */
554 pid = sizeof(void *) == 4 ? param32 : param64;
555 rcu_read_lock();
556 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
557 if (tsk)
558 get_task_struct(tsk);
559 rcu_read_unlock();
560 if (!tsk)
561 return;
562 } else {
563 tsk = current;
564 }
565 spin_lock(&pfault_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 if (subcode & 0x0080) {
567 /* signal bit is set -> a page has been swapped in by VM */
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200568 if (tsk->thread.pfault_wait == 1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 /* Initial interrupt was faster than the completion
570 * interrupt. pfault_wait is valid. Set pfault_wait
571 * back to zero and wake up the process. This can
572 * safely be done because the task is still sleeping
Martin Schwidefskyb6d09442005-09-03 15:58:02 -0700573 * and can't produce new pfaults. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 tsk->thread.pfault_wait = 0;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200575 list_del(&tsk->thread.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 wake_up_process(tsk);
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200577 } else {
578 /* Completion interrupt was faster than initial
579 * interrupt. Set pfault_wait to -1 so the initial
Heiko Carstensfa2fb2f2011-11-14 11:19:01 +0100580 * interrupt doesn't put the task to sleep.
581 * If the task is not running, ignore the completion
582 * interrupt since it must be a leftover of a PFAULT
583 * CANCEL operation which didn't remove all pending
584 * completion interrupts. */
585 if (tsk->state == TASK_RUNNING)
586 tsk->thread.pfault_wait = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 }
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200588 put_task_struct(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 } else {
590 /* signal bit not set -> a real page is missing. */
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200591 if (tsk->thread.pfault_wait == -1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 /* Completion interrupt was faster than the initial
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200593 * interrupt (pfault_wait == -1). Set pfault_wait
594 * back to zero and exit. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 tsk->thread.pfault_wait = 0;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200596 } else {
597 /* Initial interrupt arrived before completion
598 * interrupt. Let the task sleep. */
599 tsk->thread.pfault_wait = 1;
600 list_add(&tsk->thread.list, &pfault_list);
601 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 set_tsk_need_resched(tsk);
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200603 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 }
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200605 spin_unlock(&pfault_lock);
606}
607
608static int __cpuinit pfault_cpu_notify(struct notifier_block *self,
609 unsigned long action, void *hcpu)
610{
611 struct thread_struct *thread, *next;
612 struct task_struct *tsk;
613
614 switch (action) {
615 case CPU_DEAD:
616 case CPU_DEAD_FROZEN:
617 spin_lock_irq(&pfault_lock);
618 list_for_each_entry_safe(thread, next, &pfault_list, list) {
619 thread->pfault_wait = 0;
620 list_del(&thread->list);
621 tsk = container_of(thread, struct task_struct, thread);
622 wake_up_process(tsk);
623 }
624 spin_unlock_irq(&pfault_lock);
625 break;
626 default:
627 break;
628 }
629 return NOTIFY_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100632static int __init pfault_irq_init(void)
Heiko Carstens29b08d22006-12-04 15:40:40 +0100633{
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100634 int rc;
Heiko Carstens29b08d22006-12-04 15:40:40 +0100635
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100636 rc = register_external_interrupt(0x2603, pfault_interrupt);
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200637 if (rc)
638 goto out_extint;
639 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP;
640 if (rc)
641 goto out_pfault;
Heiko Carstensdf7997a2011-05-26 09:48:23 +0200642 service_subclass_irq_register();
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200643 hotcpu_notifier(pfault_cpu_notify, 0);
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100644 return 0;
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200645
646out_pfault:
647 unregister_external_interrupt(0x2603, pfault_interrupt);
648out_extint:
649 pfault_disable = 1;
650 return rc;
Heiko Carstens29b08d22006-12-04 15:40:40 +0100651}
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100652early_initcall(pfault_irq_init);
653
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200654#endif /* CONFIG_PFAULT */