blob: b57723aee848f634a03d51ff4bcbc140f5505424 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * arch/s390/mm/fault.c
3 *
4 * S390 version
5 * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation
6 * Author(s): Hartmut Penner (hp@de.ibm.com)
7 * Ulrich Weigand (uweigand@de.ibm.com)
8 *
9 * Derived from "arch/i386/mm/fault.c"
10 * Copyright (C) 1995 Linus Torvalds
11 */
12
Heiko Carstens052ff462011-01-05 12:47:28 +010013#include <linux/kernel_stat.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020014#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/signal.h>
16#include <linux/sched.h>
17#include <linux/kernel.h>
18#include <linux/errno.h>
19#include <linux/string.h>
20#include <linux/types.h>
21#include <linux/ptrace.h>
22#include <linux/mman.h>
23#include <linux/mm.h>
Heiko Carstens77575912009-06-12 10:26:25 +020024#include <linux/compat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/smp.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070026#include <linux/kdebug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/init.h>
28#include <linux/console.h>
29#include <linux/module.h>
30#include <linux/hardirq.h>
Michael Grundy4ba069b2006-09-20 15:58:39 +020031#include <linux/kprobes.h>
Martin Schwidefskybe5ec362007-04-27 16:01:44 +020032#include <linux/uaccess.h>
Gerald Schaefer53492b12008-04-30 13:38:46 +020033#include <linux/hugetlb.h>
Heiko Carstenscbb870c2010-02-26 22:37:43 +010034#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/system.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <asm/pgtable.h>
Heiko Carstensd7b250e2011-05-26 09:48:24 +020037#include <asm/irq.h>
Martin Schwidefsky6252d702008-02-09 18:24:37 +010038#include <asm/mmu_context.h>
Martin Schwidefsky50d72802009-12-07 12:51:45 +010039#include <asm/compat.h>
Heiko Carstensa8061702008-04-17 07:46:26 +020040#include "../kernel/entry.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080042#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#define __FAIL_ADDR_MASK 0x7ffff000
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#define __SUBCODE_MASK 0x0200
45#define __PF_RES_FIELD 0ULL
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080046#else /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#define __FAIL_ADDR_MASK -4096L
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#define __SUBCODE_MASK 0x0600
49#define __PF_RES_FIELD 0x8000000000000000ULL
Martin Schwidefsky347a8dc2006-01-06 00:19:28 -080050#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
Martin Schwidefsky50d72802009-12-07 12:51:45 +010052#define VM_FAULT_BADCONTEXT 0x010000
53#define VM_FAULT_BADMAP 0x020000
54#define VM_FAULT_BADACCESS 0x040000
55
Martin Schwidefsky92f842e2010-10-25 16:10:13 +020056static unsigned long store_indication;
57
58void fault_init(void)
59{
Martin Schwidefsky14375bc2010-10-25 16:10:51 +020060 if (test_facility(2) && test_facility(75))
Martin Schwidefsky92f842e2010-10-25 16:10:13 +020061 store_indication = 0xc00;
62}
63
Martin Schwidefsky7ecb3442009-12-07 12:51:44 +010064static inline int notify_page_fault(struct pt_regs *regs)
Martin Schwidefsky10c10312007-04-27 16:01:43 +020065{
Christoph Hellwig33464e32007-05-04 18:47:46 +020066 int ret = 0;
67
68 /* kprobe_running() needs smp_processor_id() */
Heiko Carstens22e0a042010-02-26 22:37:45 +010069 if (kprobes_built_in() && !user_mode(regs)) {
Christoph Hellwig33464e32007-05-04 18:47:46 +020070 preempt_disable();
71 if (kprobe_running() && kprobe_fault_handler(regs, 14))
72 ret = 1;
73 preempt_enable();
74 }
Christoph Hellwig33464e32007-05-04 18:47:46 +020075 return ret;
Michael Grundy4ba069b2006-09-20 15:58:39 +020076}
Michael Grundy4ba069b2006-09-20 15:58:39 +020077
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79/*
80 * Unlock any spinlocks which will prevent us from getting the
Kirill Korotaevcefc8be2007-02-10 01:46:18 -080081 * message out.
Linus Torvalds1da177e2005-04-16 15:20:36 -070082 */
83void bust_spinlocks(int yes)
84{
85 if (yes) {
86 oops_in_progress = 1;
87 } else {
88 int loglevel_save = console_loglevel;
89 console_unblank();
90 oops_in_progress = 0;
91 /*
92 * OK, the message is on the console. Now we call printk()
93 * without oops_in_progress set so that printk will give klogd
94 * a poke. Hold onto your hats...
95 */
96 console_loglevel = 15;
97 printk(" ");
98 console_loglevel = loglevel_save;
99 }
100}
101
102/*
Gerald Schaefer482b05d2007-03-05 23:35:54 +0100103 * Returns the address space associated with the fault.
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100104 * Returns 0 for kernel space and 1 for user space.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100106static inline int user_space_fault(unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107{
108 /*
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100109 * The lowest two bits of the translation exception
110 * identification indicate which paging table was used.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100112 trans_exc_code &= 3;
113 if (trans_exc_code == 2)
114 /* Access via secondary space, set_fs setting decides */
115 return current->thread.mm_segment.ar4;
Martin Schwidefskyb11b5332009-12-07 12:51:43 +0100116 if (user_mode == HOME_SPACE_MODE)
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100117 /* User space if the access has been done via home space. */
118 return trans_exc_code == 3;
119 /*
120 * If the user space is not the home space the kernel runs in home
121 * space. Access via secondary space has already been covered,
122 * access via primary space or access register is from user space
123 * and access via home space is from the kernel.
124 */
125 return trans_exc_code != 3;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126}
127
Heiko Carstensab3c68e2010-05-17 10:00:21 +0200128static inline void report_user_fault(struct pt_regs *regs, long int_code,
129 int signr, unsigned long address)
130{
131 if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
132 return;
133 if (!unhandled_signal(current, signr))
134 return;
135 if (!printk_ratelimit())
136 return;
137 printk("User process fault: interruption code 0x%lX ", int_code);
138 print_vma_addr(KERN_CONT "in ", regs->psw.addr & PSW_ADDR_INSN);
139 printk("\n");
140 printk("failing address: %lX\n", address);
141 show_regs(regs);
142}
143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
145 * Send SIGSEGV to task. This is an external routine
146 * to keep the stack usage of do_page_fault small.
147 */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100148static noinline void do_sigsegv(struct pt_regs *regs, long int_code,
149 int si_code, unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150{
151 struct siginfo si;
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100152 unsigned long address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100154 address = trans_exc_code & __FAIL_ADDR_MASK;
155 current->thread.prot_addr = address;
156 current->thread.trap_no = int_code;
Heiko Carstensab3c68e2010-05-17 10:00:21 +0200157 report_user_fault(regs, int_code, SIGSEGV, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 si.si_signo = SIGSEGV;
159 si.si_code = si_code;
Heiko Carstensd2c993d2006-07-12 16:41:55 +0200160 si.si_addr = (void __user *) address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 force_sig_info(SIGSEGV, &si, current);
162}
163
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100164static noinline void do_no_context(struct pt_regs *regs, long int_code,
165 unsigned long trans_exc_code)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200166{
167 const struct exception_table_entry *fixup;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100168 unsigned long address;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200169
170 /* Are we prepared to handle this kernel fault? */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100171 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200172 if (fixup) {
173 regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
174 return;
175 }
176
177 /*
178 * Oops. The kernel tried to access some bad page. We'll have to
179 * terminate things with extreme prejudice.
180 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100181 address = trans_exc_code & __FAIL_ADDR_MASK;
Martin Schwidefskyb11b5332009-12-07 12:51:43 +0100182 if (!user_space_fault(trans_exc_code))
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200183 printk(KERN_ALERT "Unable to handle kernel pointer dereference"
184 " at virtual kernel address %p\n", (void *)address);
185 else
186 printk(KERN_ALERT "Unable to handle kernel paging request"
187 " at virtual user address %p\n", (void *)address);
188
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100189 die("Oops", regs, int_code);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200190 do_exit(SIGKILL);
191}
192
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100193static noinline void do_low_address(struct pt_regs *regs, long int_code,
194 unsigned long trans_exc_code)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200195{
196 /* Low-address protection hit in kernel mode means
197 NULL pointer write access in kernel mode. */
198 if (regs->psw.mask & PSW_MASK_PSTATE) {
199 /* Low-address protection hit in user mode 'cannot happen'. */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100200 die ("Low-address protection", regs, int_code);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200201 do_exit(SIGKILL);
202 }
203
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100204 do_no_context(regs, int_code, trans_exc_code);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200205}
206
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100207static noinline void do_sigbus(struct pt_regs *regs, long int_code,
208 unsigned long trans_exc_code)
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200209{
210 struct task_struct *tsk = current;
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200211 unsigned long address;
212 struct siginfo si;
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200213
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200214 /*
215 * Send a sigbus, regardless of whether we were in kernel
216 * or user mode.
217 */
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200218 address = trans_exc_code & __FAIL_ADDR_MASK;
219 tsk->thread.prot_addr = address;
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100220 tsk->thread.trap_no = int_code;
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200221 si.si_signo = SIGBUS;
222 si.si_errno = 0;
223 si.si_code = BUS_ADRERR;
224 si.si_addr = (void __user *) address;
225 force_sig_info(SIGBUS, &si, tsk);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200226}
227
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100228static noinline void do_fault_error(struct pt_regs *regs, long int_code,
229 unsigned long trans_exc_code, int fault)
230{
231 int si_code;
232
233 switch (fault) {
234 case VM_FAULT_BADACCESS:
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100235 case VM_FAULT_BADMAP:
236 /* Bad memory access. Check if it is kernel or user space. */
237 if (regs->psw.mask & PSW_MASK_PSTATE) {
238 /* User mode accesses just cause a SIGSEGV */
239 si_code = (fault == VM_FAULT_BADMAP) ?
240 SEGV_MAPERR : SEGV_ACCERR;
241 do_sigsegv(regs, int_code, si_code, trans_exc_code);
242 return;
243 }
244 case VM_FAULT_BADCONTEXT:
245 do_no_context(regs, int_code, trans_exc_code);
246 break;
247 default: /* fault & VM_FAULT_ERROR */
Heiko Carstens99583182011-05-26 09:48:29 +0200248 if (fault & VM_FAULT_OOM) {
249 if (!(regs->psw.mask & PSW_MASK_PSTATE))
250 do_no_context(regs, int_code, trans_exc_code);
251 else
252 pagefault_out_of_memory();
253 } else if (fault & VM_FAULT_SIGBUS) {
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100254 /* Kernel mode? Handle exceptions or die */
255 if (!(regs->psw.mask & PSW_MASK_PSTATE))
256 do_no_context(regs, int_code, trans_exc_code);
Martin Schwidefsky36bf9682010-10-25 16:10:35 +0200257 else
258 do_sigbus(regs, int_code, trans_exc_code);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100259 } else
260 BUG();
261 break;
262 }
263}
264
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265/*
266 * This routine handles page faults. It determines the address,
267 * and the problem, and then passes it off to one of the appropriate
268 * routines.
269 *
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100270 * interruption code (int_code):
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 * 04 Protection -> Write-Protection (suprression)
272 * 10 Segment translation -> Not present (nullification)
273 * 11 Page translation -> Not present (nullification)
274 * 3b Region third trans. -> Not present (nullification)
275 */
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100276static inline int do_exception(struct pt_regs *regs, int access,
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100277 unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278{
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200279 struct task_struct *tsk;
280 struct mm_struct *mm;
281 struct vm_area_struct *vma;
282 unsigned long address;
Martin Schwidefsky92f842e2010-10-25 16:10:13 +0200283 int fault, write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Martin Schwidefsky7ecb3442009-12-07 12:51:44 +0100285 if (notify_page_fault(regs))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100286 return 0;
Michael Grundy4ba069b2006-09-20 15:58:39 +0200287
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200288 tsk = current;
289 mm = tsk->mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 /*
292 * Verify that the fault happened in user space, that
293 * we are not in an interrupt and that there is a
294 * user context.
295 */
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100296 fault = VM_FAULT_BADCONTEXT;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100297 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100298 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100300 address = trans_exc_code & __FAIL_ADDR_MASK;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200301 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200302 down_read(&mm->mmap_sem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100304 fault = VM_FAULT_BADMAP;
Gerald Schaefer482b05d2007-03-05 23:35:54 +0100305 vma = find_vma(mm, address);
306 if (!vma)
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100307 goto out_up;
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100308
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100309 if (unlikely(vma->vm_start > address)) {
310 if (!(vma->vm_flags & VM_GROWSDOWN))
311 goto out_up;
312 if (expand_stack(vma, address))
313 goto out_up;
314 }
315
316 /*
317 * Ok, we have a good vm_area for this memory access, so
318 * we can handle it..
319 */
320 fault = VM_FAULT_BADACCESS;
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100321 if (unlikely(!(vma->vm_flags & access)))
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100322 goto out_up;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Gerald Schaefer53492b12008-04-30 13:38:46 +0200324 if (is_vm_hugetlb_page(vma))
325 address &= HPAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 /*
327 * If for any reason at all we couldn't handle the fault,
328 * make sure we exit gracefully rather than endlessly redo
329 * the fault.
330 */
Martin Schwidefsky92f842e2010-10-25 16:10:13 +0200331 write = (access == VM_WRITE ||
332 (trans_exc_code & store_indication) == 0x400) ?
333 FAULT_FLAG_WRITE : 0;
334 fault = handle_mm_fault(mm, vma, address, write);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100335 if (unlikely(fault & VM_FAULT_ERROR))
336 goto out_up;
337
Heiko Carstensbde69af2009-09-11 10:29:06 +0200338 if (fault & VM_FAULT_MAJOR) {
Nick Piggin83c54072007-07-19 01:47:05 -0700339 tsk->maj_flt++;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200340 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
Heiko Carstensbde69af2009-09-11 10:29:06 +0200341 regs, address);
342 } else {
Nick Piggin83c54072007-07-19 01:47:05 -0700343 tsk->min_flt++;
Ingo Molnarcdd6c482009-09-21 12:02:48 +0200344 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
Heiko Carstensbde69af2009-09-11 10:29:06 +0200345 regs, address);
346 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 /*
348 * The instruction that caused the program check will
349 * be repeated. Don't signal single step via SIGTRAP.
350 */
Martin Schwidefsky5e9a2692011-01-05 12:48:10 +0100351 clear_tsk_thread_flag(tsk, TIF_PER_TRAP);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100352 fault = 0;
353out_up:
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200354 up_read(&mm->mmap_sem);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100355out:
356 return fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357}
358
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200359void __kprobes do_protection_exception(struct pt_regs *regs, long pgm_int_code,
360 unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361{
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100362 int fault;
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100363
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300364 /* Protection exception is suppressing, decrement psw address. */
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200365 regs->psw.addr -= (pgm_int_code >> 16);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200366 /*
367 * Check for low-address protection. This needs to be treated
368 * as a special case because the translation exception code
369 * field is not guaranteed to contain valid data in this case.
370 */
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100371 if (unlikely(!(trans_exc_code & 4))) {
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200372 do_low_address(regs, pgm_int_code, trans_exc_code);
Martin Schwidefsky10c10312007-04-27 16:01:43 +0200373 return;
374 }
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100375 fault = do_exception(regs, VM_WRITE, trans_exc_code);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100376 if (unlikely(fault))
377 do_fault_error(regs, 4, trans_exc_code, fault);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378}
379
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200380void __kprobes do_dat_exception(struct pt_regs *regs, long pgm_int_code,
381 unsigned long trans_exc_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382{
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100383 int access, fault;
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100384
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100385 access = VM_READ | VM_EXEC | VM_WRITE;
Martin Schwidefsky1ab947d2009-12-07 12:51:46 +0100386 fault = do_exception(regs, access, trans_exc_code);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100387 if (unlikely(fault))
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200388 do_fault_error(regs, pgm_int_code & 255, trans_exc_code, fault);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100391#ifdef CONFIG_64BIT
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200392void __kprobes do_asce_exception(struct pt_regs *regs, long pgm_int_code,
393 unsigned long trans_exc_code)
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100394{
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100395 struct mm_struct *mm = current->mm;
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100396 struct vm_area_struct *vma;
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100397
Martin Schwidefsky61365e12009-12-07 12:51:42 +0100398 if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100399 goto no_context;
400
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100401 down_read(&mm->mmap_sem);
Martin Schwidefsky50d72802009-12-07 12:51:45 +0100402 vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100403 up_read(&mm->mmap_sem);
404
405 if (vma) {
406 update_mm(mm, current);
407 return;
408 }
409
410 /* User mode accesses just cause a SIGSEGV */
411 if (regs->psw.mask & PSW_MASK_PSTATE) {
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200412 do_sigsegv(regs, pgm_int_code, SEGV_MAPERR, trans_exc_code);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100413 return;
414 }
415
416no_context:
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200417 do_no_context(regs, pgm_int_code, trans_exc_code);
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100418}
419#endif
420
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200421int __handle_fault(unsigned long uaddr, unsigned long pgm_int_code, int write)
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100422{
423 struct pt_regs regs;
424 int access, fault;
425
426 regs.psw.mask = psw_kernel_bits;
427 if (!irqs_disabled())
428 regs.psw.mask |= PSW_MASK_IO | PSW_MASK_EXT;
429 regs.psw.addr = (unsigned long) __builtin_return_address(0);
430 regs.psw.addr |= PSW_ADDR_AMODE;
431 uaddr &= PAGE_MASK;
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200432 access = write ? VM_WRITE : VM_READ;
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100433 fault = do_exception(&regs, access, uaddr | 2);
434 if (unlikely(fault)) {
Heiko Carstens99583182011-05-26 09:48:29 +0200435 if (fault & VM_FAULT_OOM)
436 return -EFAULT;
437 else if (fault & VM_FAULT_SIGBUS)
Martin Schwidefsky1e546222010-10-25 16:10:37 +0200438 do_sigbus(&regs, pgm_int_code, uaddr);
Gerald Schaefer6c1e3e72009-12-07 12:51:47 +0100439 }
440 return fault ? -EFAULT : 0;
441}
442
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443#ifdef CONFIG_PFAULT
444/*
445 * 'pfault' pseudo page faults routines.
446 */
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100447static int pfault_disable;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
449static int __init nopfault(char *str)
450{
451 pfault_disable = 1;
452 return 1;
453}
454
455__setup("nopfault", nopfault);
456
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200457struct pfault_refbk {
458 u16 refdiagc;
459 u16 reffcode;
460 u16 refdwlen;
461 u16 refversn;
462 u64 refgaddr;
463 u64 refselmk;
464 u64 refcmpmk;
465 u64 reserved;
466} __attribute__ ((packed, aligned(8)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
468int pfault_init(void)
469{
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200470 struct pfault_refbk refbk = {
471 .refdiagc = 0x258,
472 .reffcode = 0,
473 .refdwlen = 5,
474 .refversn = 2,
475 .refgaddr = __LC_CURRENT_PID,
476 .refselmk = 1ULL << 48,
477 .refcmpmk = 1ULL << 48,
478 .reserved = __PF_RES_FIELD };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 int rc;
480
Heiko Carstens29b08d22006-12-04 15:40:40 +0100481 if (!MACHINE_IS_VM || pfault_disable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 return -1;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200483 asm volatile(
484 " diag %1,%0,0x258\n"
485 "0: j 2f\n"
486 "1: la %0,8\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 "2:\n"
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200488 EX_TABLE(0b,1b)
489 : "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 return rc;
491}
492
493void pfault_fini(void)
494{
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200495 struct pfault_refbk refbk = {
496 .refdiagc = 0x258,
497 .reffcode = 1,
498 .refdwlen = 5,
499 .refversn = 2,
500 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
Heiko Carstens29b08d22006-12-04 15:40:40 +0100502 if (!MACHINE_IS_VM || pfault_disable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 return;
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200504 asm volatile(
505 " diag %0,0,0x258\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 "0:\n"
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +0200507 EX_TABLE(0b,0b)
508 : : "a" (&refbk), "m" (refbk) : "cc");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509}
510
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200511static DEFINE_SPINLOCK(pfault_lock);
512static LIST_HEAD(pfault_list);
513
Martin Schwidefskyf6649a72010-10-25 16:10:38 +0200514static void pfault_interrupt(unsigned int ext_int_code,
515 unsigned int param32, unsigned long param64)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516{
517 struct task_struct *tsk;
518 __u16 subcode;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200519 pid_t pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520
521 /*
522 * Get the external interruption subcode & pfault
523 * initial/completion signal bit. VM stores this
524 * in the 'cpu address' field associated with the
525 * external interrupt.
526 */
Martin Schwidefskyf6649a72010-10-25 16:10:38 +0200527 subcode = ext_int_code >> 16;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 if ((subcode & 0xff00) != __SUBCODE_MASK)
529 return;
Heiko Carstensa9851832011-04-29 10:42:19 +0200530 kstat_cpu(smp_processor_id()).irqs[EXTINT_PFL]++;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200531 if (subcode & 0x0080) {
532 /* Get the token (= pid of the affected task). */
533 pid = sizeof(void *) == 4 ? param32 : param64;
534 rcu_read_lock();
535 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
536 if (tsk)
537 get_task_struct(tsk);
538 rcu_read_unlock();
539 if (!tsk)
540 return;
541 } else {
542 tsk = current;
543 }
544 spin_lock(&pfault_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 if (subcode & 0x0080) {
546 /* signal bit is set -> a page has been swapped in by VM */
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200547 if (tsk->thread.pfault_wait == 1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 /* Initial interrupt was faster than the completion
549 * interrupt. pfault_wait is valid. Set pfault_wait
550 * back to zero and wake up the process. This can
551 * safely be done because the task is still sleeping
Martin Schwidefskyb6d09442005-09-03 15:58:02 -0700552 * and can't produce new pfaults. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 tsk->thread.pfault_wait = 0;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200554 list_del(&tsk->thread.list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 wake_up_process(tsk);
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200556 } else {
557 /* Completion interrupt was faster than initial
558 * interrupt. Set pfault_wait to -1 so the initial
559 * interrupt doesn't put the task to sleep. */
560 tsk->thread.pfault_wait = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 }
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200562 put_task_struct(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 } else {
564 /* signal bit not set -> a real page is missing. */
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200565 if (tsk->thread.pfault_wait == -1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 /* Completion interrupt was faster than the initial
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200567 * interrupt (pfault_wait == -1). Set pfault_wait
568 * back to zero and exit. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 tsk->thread.pfault_wait = 0;
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200570 } else {
571 /* Initial interrupt arrived before completion
572 * interrupt. Let the task sleep. */
573 tsk->thread.pfault_wait = 1;
574 list_add(&tsk->thread.list, &pfault_list);
575 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 set_tsk_need_resched(tsk);
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200577 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 }
Heiko Carstensf2db2e62011-05-23 10:24:34 +0200579 spin_unlock(&pfault_lock);
580}
581
582static int __cpuinit pfault_cpu_notify(struct notifier_block *self,
583 unsigned long action, void *hcpu)
584{
585 struct thread_struct *thread, *next;
586 struct task_struct *tsk;
587
588 switch (action) {
589 case CPU_DEAD:
590 case CPU_DEAD_FROZEN:
591 spin_lock_irq(&pfault_lock);
592 list_for_each_entry_safe(thread, next, &pfault_list, list) {
593 thread->pfault_wait = 0;
594 list_del(&thread->list);
595 tsk = container_of(thread, struct task_struct, thread);
596 wake_up_process(tsk);
597 }
598 spin_unlock_irq(&pfault_lock);
599 break;
600 default:
601 break;
602 }
603 return NOTIFY_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100606static int __init pfault_irq_init(void)
Heiko Carstens29b08d22006-12-04 15:40:40 +0100607{
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100608 int rc;
Heiko Carstens29b08d22006-12-04 15:40:40 +0100609
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100610 if (!MACHINE_IS_VM)
611 return 0;
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100612 rc = register_external_interrupt(0x2603, pfault_interrupt);
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200613 if (rc)
614 goto out_extint;
615 rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP;
616 if (rc)
617 goto out_pfault;
Heiko Carstensdf7997a2011-05-26 09:48:23 +0200618 service_subclass_irq_register();
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200619 hotcpu_notifier(pfault_cpu_notify, 0);
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100620 return 0;
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200621
622out_pfault:
623 unregister_external_interrupt(0x2603, pfault_interrupt);
624out_extint:
625 pfault_disable = 1;
626 return rc;
Heiko Carstens29b08d22006-12-04 15:40:40 +0100627}
Heiko Carstensfb0a9d72011-01-05 12:47:39 +0100628early_initcall(pfault_irq_init);
629
Heiko Carstens7dd8fe12011-05-23 10:24:35 +0200630#endif /* CONFIG_PFAULT */