blob: 2b43ba051ac94f0512744707fe197d27a6d562b1 [file] [log] [blame]
Andi Kleen6a460792009-09-16 11:50:15 +02001/*
2 * Copyright (C) 2008, 2009 Intel Corporation
3 * Authors: Andi Kleen, Fengguang Wu
4 *
5 * This software may be redistributed and/or modified under the terms of
6 * the GNU General Public License ("GPL") version 2 only as published by the
7 * Free Software Foundation.
8 *
9 * High level machine check handler. Handles pages reported by the
Andi Kleen1c80b992010-09-27 23:09:51 +020010 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
Andi Kleen6a460792009-09-16 11:50:15 +020011 * failure.
Andi Kleen1c80b992010-09-27 23:09:51 +020012 *
13 * In addition there is a "soft offline" entry point that allows stop using
14 * not-yet-corrupted-by-suspicious pages without killing anything.
Andi Kleen6a460792009-09-16 11:50:15 +020015 *
16 * Handles page cache pages in various states. The tricky part
Andi Kleen1c80b992010-09-27 23:09:51 +020017 * here is that we can access any page asynchronously in respect to
18 * other VM users, because memory failures could happen anytime and
19 * anywhere. This could violate some of their assumptions. This is why
20 * this code has to be extremely careful. Generally it tries to use
21 * normal locking rules, as in get the standard locks, even if that means
22 * the error handling takes potentially a long time.
23 *
24 * There are several operations here with exponential complexity because
25 * of unsuitable VM data structures. For example the operation to map back
26 * from RMAP chains to processes has to walk the complete process list and
27 * has non linear complexity with the number. But since memory corruptions
28 * are rare we hope to get away with this. This avoids impacting the core
29 * VM.
Andi Kleen6a460792009-09-16 11:50:15 +020030 */
31
32/*
33 * Notebook:
34 * - hugetlb needs more code
35 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
36 * - pass bad pages to kdump next kernel
37 */
Andi Kleen6a460792009-09-16 11:50:15 +020038#include <linux/kernel.h>
39#include <linux/mm.h>
40#include <linux/page-flags.h>
Wu Fengguang478c5ff2009-12-16 12:19:59 +010041#include <linux/kernel-page-flags.h>
Andi Kleen6a460792009-09-16 11:50:15 +020042#include <linux/sched.h>
Hugh Dickins01e00f82009-10-13 15:02:11 +010043#include <linux/ksm.h>
Andi Kleen6a460792009-09-16 11:50:15 +020044#include <linux/rmap.h>
45#include <linux/pagemap.h>
46#include <linux/swap.h>
47#include <linux/backing-dev.h>
Andi Kleenfacb6012009-12-16 12:20:00 +010048#include <linux/migrate.h>
49#include <linux/page-isolation.h>
50#include <linux/suspend.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090051#include <linux/slab.h>
Huang Yingbf998152010-05-31 14:28:19 +080052#include <linux/swapops.h>
Naoya Horiguchi7af446a2010-05-28 09:29:17 +090053#include <linux/hugetlb.h>
KOSAKI Motohiro20d6c962010-12-02 14:31:19 -080054#include <linux/memory_hotplug.h>
Minchan Kim5db8a732011-06-15 15:08:48 -070055#include <linux/mm_inline.h>
Huang Yingea8f5fb2011-07-13 13:14:27 +080056#include <linux/kfifo.h>
Andi Kleen6a460792009-09-16 11:50:15 +020057#include "internal.h"
58
59int sysctl_memory_failure_early_kill __read_mostly = 0;
60
61int sysctl_memory_failure_recovery __read_mostly = 1;
62
63atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0);
64
Andi Kleen27df5062009-12-21 19:56:42 +010065#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
66
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010067u32 hwpoison_filter_enable = 0;
Wu Fengguang7c116f22009-12-16 12:19:59 +010068u32 hwpoison_filter_dev_major = ~0U;
69u32 hwpoison_filter_dev_minor = ~0U;
Wu Fengguang478c5ff2009-12-16 12:19:59 +010070u64 hwpoison_filter_flags_mask;
71u64 hwpoison_filter_flags_value;
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010072EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
Wu Fengguang7c116f22009-12-16 12:19:59 +010073EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
74EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
Wu Fengguang478c5ff2009-12-16 12:19:59 +010075EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
76EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
Wu Fengguang7c116f22009-12-16 12:19:59 +010077
78static int hwpoison_filter_dev(struct page *p)
79{
80 struct address_space *mapping;
81 dev_t dev;
82
83 if (hwpoison_filter_dev_major == ~0U &&
84 hwpoison_filter_dev_minor == ~0U)
85 return 0;
86
87 /*
Andi Kleen1c80b992010-09-27 23:09:51 +020088 * page_mapping() does not accept slab pages.
Wu Fengguang7c116f22009-12-16 12:19:59 +010089 */
90 if (PageSlab(p))
91 return -EINVAL;
92
93 mapping = page_mapping(p);
94 if (mapping == NULL || mapping->host == NULL)
95 return -EINVAL;
96
97 dev = mapping->host->i_sb->s_dev;
98 if (hwpoison_filter_dev_major != ~0U &&
99 hwpoison_filter_dev_major != MAJOR(dev))
100 return -EINVAL;
101 if (hwpoison_filter_dev_minor != ~0U &&
102 hwpoison_filter_dev_minor != MINOR(dev))
103 return -EINVAL;
104
105 return 0;
106}
107
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100108static int hwpoison_filter_flags(struct page *p)
109{
110 if (!hwpoison_filter_flags_mask)
111 return 0;
112
113 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
114 hwpoison_filter_flags_value)
115 return 0;
116 else
117 return -EINVAL;
118}
119
Andi Kleen4fd466e2009-12-16 12:19:59 +0100120/*
121 * This allows stress tests to limit test scope to a collection of tasks
122 * by putting them under some memcg. This prevents killing unrelated/important
123 * processes such as /sbin/init. Note that the target task may share clean
124 * pages with init (eg. libc text), which is harmless. If the target task
125 * share _dirty_ pages with another task B, the test scheme must make sure B
126 * is also included in the memcg. At last, due to race conditions this filter
127 * can only guarantee that the page either belongs to the memcg tasks, or is
128 * a freed page.
129 */
130#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
131u64 hwpoison_filter_memcg;
132EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
133static int hwpoison_filter_task(struct page *p)
134{
135 struct mem_cgroup *mem;
136 struct cgroup_subsys_state *css;
137 unsigned long ino;
138
139 if (!hwpoison_filter_memcg)
140 return 0;
141
142 mem = try_get_mem_cgroup_from_page(p);
143 if (!mem)
144 return -EINVAL;
145
146 css = mem_cgroup_css(mem);
147 /* root_mem_cgroup has NULL dentries */
148 if (!css->cgroup->dentry)
149 return -EINVAL;
150
151 ino = css->cgroup->dentry->d_inode->i_ino;
152 css_put(css);
153
154 if (ino != hwpoison_filter_memcg)
155 return -EINVAL;
156
157 return 0;
158}
159#else
160static int hwpoison_filter_task(struct page *p) { return 0; }
161#endif
162
Wu Fengguang7c116f22009-12-16 12:19:59 +0100163int hwpoison_filter(struct page *p)
164{
Haicheng Li1bfe5fe2009-12-16 12:19:59 +0100165 if (!hwpoison_filter_enable)
166 return 0;
167
Wu Fengguang7c116f22009-12-16 12:19:59 +0100168 if (hwpoison_filter_dev(p))
169 return -EINVAL;
170
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100171 if (hwpoison_filter_flags(p))
172 return -EINVAL;
173
Andi Kleen4fd466e2009-12-16 12:19:59 +0100174 if (hwpoison_filter_task(p))
175 return -EINVAL;
176
Wu Fengguang7c116f22009-12-16 12:19:59 +0100177 return 0;
178}
Andi Kleen27df5062009-12-21 19:56:42 +0100179#else
180int hwpoison_filter(struct page *p)
181{
182 return 0;
183}
184#endif
185
Wu Fengguang7c116f22009-12-16 12:19:59 +0100186EXPORT_SYMBOL_GPL(hwpoison_filter);
187
Andi Kleen6a460792009-09-16 11:50:15 +0200188/*
189 * Send all the processes who have the page mapped an ``action optional''
190 * signal.
191 */
192static int kill_proc_ao(struct task_struct *t, unsigned long addr, int trapno,
Andi Kleen0d9ee6a2010-09-27 22:03:33 +0200193 unsigned long pfn, struct page *page)
Andi Kleen6a460792009-09-16 11:50:15 +0200194{
195 struct siginfo si;
196 int ret;
197
198 printk(KERN_ERR
199 "MCE %#lx: Killing %s:%d early due to hardware memory corruption\n",
200 pfn, t->comm, t->pid);
201 si.si_signo = SIGBUS;
202 si.si_errno = 0;
203 si.si_code = BUS_MCEERR_AO;
204 si.si_addr = (void *)addr;
205#ifdef __ARCH_SI_TRAPNO
206 si.si_trapno = trapno;
207#endif
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -0800208 si.si_addr_lsb = compound_trans_order(compound_head(page)) + PAGE_SHIFT;
Andi Kleen6a460792009-09-16 11:50:15 +0200209 /*
210 * Don't use force here, it's convenient if the signal
211 * can be temporarily blocked.
212 * This could cause a loop when the user sets SIGBUS
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300213 * to SIG_IGN, but hopefully no one will do that?
Andi Kleen6a460792009-09-16 11:50:15 +0200214 */
215 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
216 if (ret < 0)
217 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
218 t->comm, t->pid, ret);
219 return ret;
220}
221
222/*
Andi Kleen588f9ce2009-12-16 12:19:57 +0100223 * When a unknown page type is encountered drain as many buffers as possible
224 * in the hope to turn the page into a LRU or free page, which we can handle.
225 */
Andi Kleenfacb6012009-12-16 12:20:00 +0100226void shake_page(struct page *p, int access)
Andi Kleen588f9ce2009-12-16 12:19:57 +0100227{
228 if (!PageSlab(p)) {
229 lru_add_drain_all();
230 if (PageLRU(p))
231 return;
232 drain_all_pages();
233 if (PageLRU(p) || is_free_buddy_page(p))
234 return;
235 }
Andi Kleenfacb6012009-12-16 12:20:00 +0100236
Andi Kleen588f9ce2009-12-16 12:19:57 +0100237 /*
Jin Dongmingaf241a02011-02-01 15:52:41 -0800238 * Only call shrink_slab here (which would also shrink other caches) if
239 * access is not potentially fatal.
Andi Kleen588f9ce2009-12-16 12:19:57 +0100240 */
Andi Kleenfacb6012009-12-16 12:20:00 +0100241 if (access) {
242 int nr;
243 do {
Ying Hana09ed5e2011-05-24 17:12:26 -0700244 struct shrink_control shrink = {
245 .gfp_mask = GFP_KERNEL,
Ying Hana09ed5e2011-05-24 17:12:26 -0700246 };
247
Ying Han1495f232011-05-24 17:12:27 -0700248 nr = shrink_slab(&shrink, 1000, 1000);
Andi Kleen47f43e72010-09-28 07:37:55 +0200249 if (page_count(p) == 1)
Andi Kleenfacb6012009-12-16 12:20:00 +0100250 break;
251 } while (nr > 10);
252 }
Andi Kleen588f9ce2009-12-16 12:19:57 +0100253}
254EXPORT_SYMBOL_GPL(shake_page);
255
256/*
Andi Kleen6a460792009-09-16 11:50:15 +0200257 * Kill all processes that have a poisoned page mapped and then isolate
258 * the page.
259 *
260 * General strategy:
261 * Find all processes having the page mapped and kill them.
262 * But we keep a page reference around so that the page is not
263 * actually freed yet.
264 * Then stash the page away
265 *
266 * There's no convenient way to get back to mapped processes
267 * from the VMAs. So do a brute-force search over all
268 * running processes.
269 *
270 * Remember that machine checks are not common (or rather
271 * if they are common you have other problems), so this shouldn't
272 * be a performance issue.
273 *
274 * Also there are some races possible while we get from the
275 * error detection to actually handle it.
276 */
277
278struct to_kill {
279 struct list_head nd;
280 struct task_struct *tsk;
281 unsigned long addr;
Andi Kleen9033ae12010-09-27 23:36:05 +0200282 char addr_valid;
Andi Kleen6a460792009-09-16 11:50:15 +0200283};
284
285/*
286 * Failure handling: if we can't find or can't kill a process there's
287 * not much we can do. We just print a message and ignore otherwise.
288 */
289
290/*
291 * Schedule a process for later kill.
292 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
293 * TBD would GFP_NOIO be enough?
294 */
295static void add_to_kill(struct task_struct *tsk, struct page *p,
296 struct vm_area_struct *vma,
297 struct list_head *to_kill,
298 struct to_kill **tkc)
299{
300 struct to_kill *tk;
301
302 if (*tkc) {
303 tk = *tkc;
304 *tkc = NULL;
305 } else {
306 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
307 if (!tk) {
308 printk(KERN_ERR
309 "MCE: Out of memory while machine check handling\n");
310 return;
311 }
312 }
313 tk->addr = page_address_in_vma(p, vma);
314 tk->addr_valid = 1;
315
316 /*
317 * In theory we don't have to kill when the page was
318 * munmaped. But it could be also a mremap. Since that's
319 * likely very rare kill anyways just out of paranoia, but use
320 * a SIGKILL because the error is not contained anymore.
321 */
322 if (tk->addr == -EFAULT) {
Andi Kleenfb46e732010-09-27 23:31:30 +0200323 pr_info("MCE: Unable to find user space address %lx in %s\n",
Andi Kleen6a460792009-09-16 11:50:15 +0200324 page_to_pfn(p), tsk->comm);
325 tk->addr_valid = 0;
326 }
327 get_task_struct(tsk);
328 tk->tsk = tsk;
329 list_add_tail(&tk->nd, to_kill);
330}
331
332/*
333 * Kill the processes that have been collected earlier.
334 *
335 * Only do anything when DOIT is set, otherwise just free the list
336 * (this is used for clean pages which do not need killing)
337 * Also when FAIL is set do a force kill because something went
338 * wrong earlier.
339 */
340static void kill_procs_ao(struct list_head *to_kill, int doit, int trapno,
Andi Kleen0d9ee6a2010-09-27 22:03:33 +0200341 int fail, struct page *page, unsigned long pfn)
Andi Kleen6a460792009-09-16 11:50:15 +0200342{
343 struct to_kill *tk, *next;
344
345 list_for_each_entry_safe (tk, next, to_kill, nd) {
346 if (doit) {
347 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200348 * In case something went wrong with munmapping
Andi Kleen6a460792009-09-16 11:50:15 +0200349 * make sure the process doesn't catch the
350 * signal and then access the memory. Just kill it.
Andi Kleen6a460792009-09-16 11:50:15 +0200351 */
352 if (fail || tk->addr_valid == 0) {
353 printk(KERN_ERR
354 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
355 pfn, tk->tsk->comm, tk->tsk->pid);
356 force_sig(SIGKILL, tk->tsk);
357 }
358
359 /*
360 * In theory the process could have mapped
361 * something else on the address in-between. We could
362 * check for that, but we need to tell the
363 * process anyways.
364 */
365 else if (kill_proc_ao(tk->tsk, tk->addr, trapno,
Andi Kleen0d9ee6a2010-09-27 22:03:33 +0200366 pfn, page) < 0)
Andi Kleen6a460792009-09-16 11:50:15 +0200367 printk(KERN_ERR
368 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
369 pfn, tk->tsk->comm, tk->tsk->pid);
370 }
371 put_task_struct(tk->tsk);
372 kfree(tk);
373 }
374}
375
376static int task_early_kill(struct task_struct *tsk)
377{
378 if (!tsk->mm)
379 return 0;
380 if (tsk->flags & PF_MCE_PROCESS)
381 return !!(tsk->flags & PF_MCE_EARLY);
382 return sysctl_memory_failure_early_kill;
383}
384
385/*
386 * Collect processes when the error hit an anonymous page.
387 */
388static void collect_procs_anon(struct page *page, struct list_head *to_kill,
389 struct to_kill **tkc)
390{
391 struct vm_area_struct *vma;
392 struct task_struct *tsk;
393 struct anon_vma *av;
394
Andi Kleen6a460792009-09-16 11:50:15 +0200395 av = page_lock_anon_vma(page);
396 if (av == NULL) /* Not actually mapped anymore */
Peter Zijlstra9b679322011-06-27 16:18:09 -0700397 return;
398
399 read_lock(&tasklist_lock);
Andi Kleen6a460792009-09-16 11:50:15 +0200400 for_each_process (tsk) {
Rik van Riel5beb4932010-03-05 13:42:07 -0800401 struct anon_vma_chain *vmac;
402
Andi Kleen6a460792009-09-16 11:50:15 +0200403 if (!task_early_kill(tsk))
404 continue;
Rik van Riel5beb4932010-03-05 13:42:07 -0800405 list_for_each_entry(vmac, &av->head, same_anon_vma) {
406 vma = vmac->vma;
Andi Kleen6a460792009-09-16 11:50:15 +0200407 if (!page_mapped_in_vma(page, vma))
408 continue;
409 if (vma->vm_mm == tsk->mm)
410 add_to_kill(tsk, page, vma, to_kill, tkc);
411 }
412 }
Andi Kleen6a460792009-09-16 11:50:15 +0200413 read_unlock(&tasklist_lock);
Peter Zijlstra9b679322011-06-27 16:18:09 -0700414 page_unlock_anon_vma(av);
Andi Kleen6a460792009-09-16 11:50:15 +0200415}
416
417/*
418 * Collect processes when the error hit a file mapped page.
419 */
420static void collect_procs_file(struct page *page, struct list_head *to_kill,
421 struct to_kill **tkc)
422{
423 struct vm_area_struct *vma;
424 struct task_struct *tsk;
425 struct prio_tree_iter iter;
426 struct address_space *mapping = page->mapping;
427
Peter Zijlstra3d48ae42011-05-24 17:12:06 -0700428 mutex_lock(&mapping->i_mmap_mutex);
Peter Zijlstra9b679322011-06-27 16:18:09 -0700429 read_lock(&tasklist_lock);
Andi Kleen6a460792009-09-16 11:50:15 +0200430 for_each_process(tsk) {
431 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
432
433 if (!task_early_kill(tsk))
434 continue;
435
436 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff,
437 pgoff) {
438 /*
439 * Send early kill signal to tasks where a vma covers
440 * the page but the corrupted page is not necessarily
441 * mapped it in its pte.
442 * Assume applications who requested early kill want
443 * to be informed of all such data corruptions.
444 */
445 if (vma->vm_mm == tsk->mm)
446 add_to_kill(tsk, page, vma, to_kill, tkc);
447 }
448 }
Andi Kleen6a460792009-09-16 11:50:15 +0200449 read_unlock(&tasklist_lock);
Peter Zijlstra9b679322011-06-27 16:18:09 -0700450 mutex_unlock(&mapping->i_mmap_mutex);
Andi Kleen6a460792009-09-16 11:50:15 +0200451}
452
453/*
454 * Collect the processes who have the corrupted page mapped to kill.
455 * This is done in two steps for locking reasons.
456 * First preallocate one tokill structure outside the spin locks,
457 * so that we can kill at least one process reasonably reliable.
458 */
459static void collect_procs(struct page *page, struct list_head *tokill)
460{
461 struct to_kill *tk;
462
463 if (!page->mapping)
464 return;
465
466 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
467 if (!tk)
468 return;
469 if (PageAnon(page))
470 collect_procs_anon(page, tokill, &tk);
471 else
472 collect_procs_file(page, tokill, &tk);
473 kfree(tk);
474}
475
476/*
477 * Error handlers for various types of pages.
478 */
479
480enum outcome {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100481 IGNORED, /* Error: cannot be handled */
482 FAILED, /* Error: handling failed */
Andi Kleen6a460792009-09-16 11:50:15 +0200483 DELAYED, /* Will be handled later */
Andi Kleen6a460792009-09-16 11:50:15 +0200484 RECOVERED, /* Successfully recovered */
485};
486
487static const char *action_name[] = {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100488 [IGNORED] = "Ignored",
Andi Kleen6a460792009-09-16 11:50:15 +0200489 [FAILED] = "Failed",
490 [DELAYED] = "Delayed",
Andi Kleen6a460792009-09-16 11:50:15 +0200491 [RECOVERED] = "Recovered",
492};
493
494/*
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100495 * XXX: It is possible that a page is isolated from LRU cache,
496 * and then kept in swap cache or failed to remove from page cache.
497 * The page count will stop it from being freed by unpoison.
498 * Stress tests should be aware of this memory leak problem.
499 */
500static int delete_from_lru_cache(struct page *p)
501{
502 if (!isolate_lru_page(p)) {
503 /*
504 * Clear sensible page flags, so that the buddy system won't
505 * complain when the page is unpoison-and-freed.
506 */
507 ClearPageActive(p);
508 ClearPageUnevictable(p);
509 /*
510 * drop the page count elevated by isolate_lru_page()
511 */
512 page_cache_release(p);
513 return 0;
514 }
515 return -EIO;
516}
517
518/*
Andi Kleen6a460792009-09-16 11:50:15 +0200519 * Error hit kernel page.
520 * Do nothing, try to be lucky and not touch this instead. For a few cases we
521 * could be more sophisticated.
522 */
523static int me_kernel(struct page *p, unsigned long pfn)
524{
Andi Kleen6a460792009-09-16 11:50:15 +0200525 return IGNORED;
526}
527
528/*
529 * Page in unknown state. Do nothing.
530 */
531static int me_unknown(struct page *p, unsigned long pfn)
532{
533 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
534 return FAILED;
535}
536
537/*
Andi Kleen6a460792009-09-16 11:50:15 +0200538 * Clean (or cleaned) page cache page.
539 */
540static int me_pagecache_clean(struct page *p, unsigned long pfn)
541{
542 int err;
543 int ret = FAILED;
544 struct address_space *mapping;
545
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100546 delete_from_lru_cache(p);
547
Andi Kleen6a460792009-09-16 11:50:15 +0200548 /*
549 * For anonymous pages we're done the only reference left
550 * should be the one m_f() holds.
551 */
552 if (PageAnon(p))
553 return RECOVERED;
554
555 /*
556 * Now truncate the page in the page cache. This is really
557 * more like a "temporary hole punch"
558 * Don't do this for block devices when someone else
559 * has a reference, because it could be file system metadata
560 * and that's not safe to truncate.
561 */
562 mapping = page_mapping(p);
563 if (!mapping) {
564 /*
565 * Page has been teared down in the meanwhile
566 */
567 return FAILED;
568 }
569
570 /*
571 * Truncation is a bit tricky. Enable it per file system for now.
572 *
573 * Open: to take i_mutex or not for this? Right now we don't.
574 */
575 if (mapping->a_ops->error_remove_page) {
576 err = mapping->a_ops->error_remove_page(mapping, p);
577 if (err != 0) {
578 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
579 pfn, err);
580 } else if (page_has_private(p) &&
581 !try_to_release_page(p, GFP_NOIO)) {
Andi Kleenfb46e732010-09-27 23:31:30 +0200582 pr_info("MCE %#lx: failed to release buffers\n", pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200583 } else {
584 ret = RECOVERED;
585 }
586 } else {
587 /*
588 * If the file system doesn't support it just invalidate
589 * This fails on dirty or anything with private pages
590 */
591 if (invalidate_inode_page(p))
592 ret = RECOVERED;
593 else
594 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
595 pfn);
596 }
597 return ret;
598}
599
600/*
601 * Dirty cache page page
602 * Issues: when the error hit a hole page the error is not properly
603 * propagated.
604 */
605static int me_pagecache_dirty(struct page *p, unsigned long pfn)
606{
607 struct address_space *mapping = page_mapping(p);
608
609 SetPageError(p);
610 /* TBD: print more information about the file. */
611 if (mapping) {
612 /*
613 * IO error will be reported by write(), fsync(), etc.
614 * who check the mapping.
615 * This way the application knows that something went
616 * wrong with its dirty file data.
617 *
618 * There's one open issue:
619 *
620 * The EIO will be only reported on the next IO
621 * operation and then cleared through the IO map.
622 * Normally Linux has two mechanisms to pass IO error
623 * first through the AS_EIO flag in the address space
624 * and then through the PageError flag in the page.
625 * Since we drop pages on memory failure handling the
626 * only mechanism open to use is through AS_AIO.
627 *
628 * This has the disadvantage that it gets cleared on
629 * the first operation that returns an error, while
630 * the PageError bit is more sticky and only cleared
631 * when the page is reread or dropped. If an
632 * application assumes it will always get error on
633 * fsync, but does other operations on the fd before
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300634 * and the page is dropped between then the error
Andi Kleen6a460792009-09-16 11:50:15 +0200635 * will not be properly reported.
636 *
637 * This can already happen even without hwpoisoned
638 * pages: first on metadata IO errors (which only
639 * report through AS_EIO) or when the page is dropped
640 * at the wrong time.
641 *
642 * So right now we assume that the application DTRT on
643 * the first EIO, but we're not worse than other parts
644 * of the kernel.
645 */
646 mapping_set_error(mapping, EIO);
647 }
648
649 return me_pagecache_clean(p, pfn);
650}
651
652/*
653 * Clean and dirty swap cache.
654 *
655 * Dirty swap cache page is tricky to handle. The page could live both in page
656 * cache and swap cache(ie. page is freshly swapped in). So it could be
657 * referenced concurrently by 2 types of PTEs:
658 * normal PTEs and swap PTEs. We try to handle them consistently by calling
659 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
660 * and then
661 * - clear dirty bit to prevent IO
662 * - remove from LRU
663 * - but keep in the swap cache, so that when we return to it on
664 * a later page fault, we know the application is accessing
665 * corrupted data and shall be killed (we installed simple
666 * interception code in do_swap_page to catch it).
667 *
668 * Clean swap cache pages can be directly isolated. A later page fault will
669 * bring in the known good data from disk.
670 */
671static int me_swapcache_dirty(struct page *p, unsigned long pfn)
672{
Andi Kleen6a460792009-09-16 11:50:15 +0200673 ClearPageDirty(p);
674 /* Trigger EIO in shmem: */
675 ClearPageUptodate(p);
676
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100677 if (!delete_from_lru_cache(p))
678 return DELAYED;
679 else
680 return FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200681}
682
683static int me_swapcache_clean(struct page *p, unsigned long pfn)
684{
Andi Kleen6a460792009-09-16 11:50:15 +0200685 delete_from_swap_cache(p);
Wu Fengguange43c3af2009-09-29 13:16:20 +0800686
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100687 if (!delete_from_lru_cache(p))
688 return RECOVERED;
689 else
690 return FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200691}
692
693/*
694 * Huge pages. Needs work.
695 * Issues:
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900696 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
697 * To narrow down kill region to one page, we need to break up pmd.
Andi Kleen6a460792009-09-16 11:50:15 +0200698 */
699static int me_huge_page(struct page *p, unsigned long pfn)
700{
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +0900701 int res = 0;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900702 struct page *hpage = compound_head(p);
703 /*
704 * We can safely recover from error on free or reserved (i.e.
705 * not in-use) hugepage by dequeuing it from freelist.
706 * To check whether a hugepage is in-use or not, we can't use
707 * page->lru because it can be used in other hugepage operations,
708 * such as __unmap_hugepage_range() and gather_surplus_pages().
709 * So instead we use page_mapping() and PageAnon().
710 * We assume that this function is called with page lock held,
711 * so there is no race between isolation and mapping/unmapping.
712 */
713 if (!(page_mapping(hpage) || PageAnon(hpage))) {
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +0900714 res = dequeue_hwpoisoned_huge_page(hpage);
715 if (!res)
716 return RECOVERED;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900717 }
718 return DELAYED;
Andi Kleen6a460792009-09-16 11:50:15 +0200719}
720
721/*
722 * Various page states we can handle.
723 *
724 * A page state is defined by its current page->flags bits.
725 * The table matches them in order and calls the right handler.
726 *
727 * This is quite tricky because we can access page at any time
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300728 * in its live cycle, so all accesses have to be extremely careful.
Andi Kleen6a460792009-09-16 11:50:15 +0200729 *
730 * This is not complete. More states could be added.
731 * For any missing state don't attempt recovery.
732 */
733
734#define dirty (1UL << PG_dirty)
735#define sc (1UL << PG_swapcache)
736#define unevict (1UL << PG_unevictable)
737#define mlock (1UL << PG_mlocked)
738#define writeback (1UL << PG_writeback)
739#define lru (1UL << PG_lru)
740#define swapbacked (1UL << PG_swapbacked)
741#define head (1UL << PG_head)
742#define tail (1UL << PG_tail)
743#define compound (1UL << PG_compound)
744#define slab (1UL << PG_slab)
Andi Kleen6a460792009-09-16 11:50:15 +0200745#define reserved (1UL << PG_reserved)
746
747static struct page_state {
748 unsigned long mask;
749 unsigned long res;
750 char *msg;
751 int (*action)(struct page *p, unsigned long pfn);
752} error_states[] = {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100753 { reserved, reserved, "reserved kernel", me_kernel },
Wu Fengguang95d01fc2009-12-16 12:19:58 +0100754 /*
755 * free pages are specially detected outside this table:
756 * PG_buddy pages only make a small fraction of all free pages.
757 */
Andi Kleen6a460792009-09-16 11:50:15 +0200758
759 /*
760 * Could in theory check if slab page is free or if we can drop
761 * currently unused objects without touching them. But just
762 * treat it as standard kernel for now.
763 */
764 { slab, slab, "kernel slab", me_kernel },
765
766#ifdef CONFIG_PAGEFLAGS_EXTENDED
767 { head, head, "huge", me_huge_page },
768 { tail, tail, "huge", me_huge_page },
769#else
770 { compound, compound, "huge", me_huge_page },
771#endif
772
773 { sc|dirty, sc|dirty, "swapcache", me_swapcache_dirty },
774 { sc|dirty, sc, "swapcache", me_swapcache_clean },
775
776 { unevict|dirty, unevict|dirty, "unevictable LRU", me_pagecache_dirty},
777 { unevict, unevict, "unevictable LRU", me_pagecache_clean},
778
Andi Kleen6a460792009-09-16 11:50:15 +0200779 { mlock|dirty, mlock|dirty, "mlocked LRU", me_pagecache_dirty },
780 { mlock, mlock, "mlocked LRU", me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200781
782 { lru|dirty, lru|dirty, "LRU", me_pagecache_dirty },
783 { lru|dirty, lru, "clean LRU", me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200784
785 /*
786 * Catchall entry: must be at end.
787 */
788 { 0, 0, "unknown page state", me_unknown },
789};
790
Andi Kleen2326c462009-12-16 12:20:00 +0100791#undef dirty
792#undef sc
793#undef unevict
794#undef mlock
795#undef writeback
796#undef lru
797#undef swapbacked
798#undef head
799#undef tail
800#undef compound
801#undef slab
802#undef reserved
803
Andi Kleen6a460792009-09-16 11:50:15 +0200804static void action_result(unsigned long pfn, char *msg, int result)
805{
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100806 struct page *page = pfn_to_page(pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200807
808 printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n",
809 pfn,
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100810 PageDirty(page) ? "dirty " : "",
Andi Kleen6a460792009-09-16 11:50:15 +0200811 msg, action_name[result]);
812}
813
814static int page_action(struct page_state *ps, struct page *p,
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100815 unsigned long pfn)
Andi Kleen6a460792009-09-16 11:50:15 +0200816{
817 int result;
Wu Fengguang7456b042009-10-19 08:15:01 +0200818 int count;
Andi Kleen6a460792009-09-16 11:50:15 +0200819
820 result = ps->action(p, pfn);
821 action_result(pfn, ps->msg, result);
Wu Fengguang7456b042009-10-19 08:15:01 +0200822
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100823 count = page_count(p) - 1;
Wu Fengguang138ce282009-12-16 12:19:58 +0100824 if (ps->action == me_swapcache_dirty && result == DELAYED)
825 count--;
826 if (count != 0) {
Andi Kleen6a460792009-09-16 11:50:15 +0200827 printk(KERN_ERR
828 "MCE %#lx: %s page still referenced by %d users\n",
Wu Fengguang7456b042009-10-19 08:15:01 +0200829 pfn, ps->msg, count);
Wu Fengguang138ce282009-12-16 12:19:58 +0100830 result = FAILED;
831 }
Andi Kleen6a460792009-09-16 11:50:15 +0200832
833 /* Could do more checks here if page looks ok */
834 /*
835 * Could adjust zone counters here to correct for the missing page.
836 */
837
Wu Fengguang138ce282009-12-16 12:19:58 +0100838 return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +0200839}
840
Andi Kleen6a460792009-09-16 11:50:15 +0200841/*
842 * Do all that is necessary to remove user space mappings. Unmap
843 * the pages and send SIGBUS to the processes if the data was dirty.
844 */
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100845static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
Andi Kleen6a460792009-09-16 11:50:15 +0200846 int trapno)
847{
848 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
849 struct address_space *mapping;
850 LIST_HEAD(tokill);
851 int ret;
Andi Kleen6a460792009-09-16 11:50:15 +0200852 int kill = 1;
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900853 struct page *hpage = compound_head(p);
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800854 struct page *ppage;
Andi Kleen6a460792009-09-16 11:50:15 +0200855
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100856 if (PageReserved(p) || PageSlab(p))
857 return SWAP_SUCCESS;
Andi Kleen6a460792009-09-16 11:50:15 +0200858
Andi Kleen6a460792009-09-16 11:50:15 +0200859 /*
860 * This check implies we don't kill processes if their pages
861 * are in the swap cache early. Those are always late kills.
862 */
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900863 if (!page_mapped(hpage))
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100864 return SWAP_SUCCESS;
865
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900866 if (PageKsm(p))
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100867 return SWAP_FAIL;
Andi Kleen6a460792009-09-16 11:50:15 +0200868
869 if (PageSwapCache(p)) {
870 printk(KERN_ERR
871 "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
872 ttu |= TTU_IGNORE_HWPOISON;
873 }
874
875 /*
876 * Propagate the dirty bit from PTEs to struct page first, because we
877 * need this to decide if we should kill or just drop the page.
Wu Fengguangdb0480b2009-12-16 12:19:58 +0100878 * XXX: the dirty test could be racy: set_page_dirty() may not always
879 * be called inside page lock (it's recommended but not enforced).
Andi Kleen6a460792009-09-16 11:50:15 +0200880 */
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900881 mapping = page_mapping(hpage);
882 if (!PageDirty(hpage) && mapping &&
883 mapping_cap_writeback_dirty(mapping)) {
884 if (page_mkclean(hpage)) {
885 SetPageDirty(hpage);
Andi Kleen6a460792009-09-16 11:50:15 +0200886 } else {
887 kill = 0;
888 ttu |= TTU_IGNORE_HWPOISON;
889 printk(KERN_INFO
890 "MCE %#lx: corrupted page was clean: dropped without side effects\n",
891 pfn);
892 }
893 }
894
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800895 /*
896 * ppage: poisoned page
897 * if p is regular page(4k page)
898 * ppage == real poisoned page;
899 * else p is hugetlb or THP, ppage == head page.
900 */
901 ppage = hpage;
902
Jin Dongmingefeda7a2011-02-01 15:52:39 -0800903 if (PageTransHuge(hpage)) {
904 /*
905 * Verify that this isn't a hugetlbfs head page, the check for
906 * PageAnon is just for avoid tripping a split_huge_page
907 * internal debug check, as split_huge_page refuses to deal with
908 * anything that isn't an anon page. PageAnon can't go away fro
909 * under us because we hold a refcount on the hpage, without a
910 * refcount on the hpage. split_huge_page can't be safely called
911 * in the first place, having a refcount on the tail isn't
912 * enough * to be safe.
913 */
914 if (!PageHuge(hpage) && PageAnon(hpage)) {
915 if (unlikely(split_huge_page(hpage))) {
916 /*
917 * FIXME: if splitting THP is failed, it is
918 * better to stop the following operation rather
919 * than causing panic by unmapping. System might
920 * survive if the page is freed later.
921 */
922 printk(KERN_INFO
923 "MCE %#lx: failed to split THP\n", pfn);
924
925 BUG_ON(!PageHWPoison(p));
926 return SWAP_FAIL;
927 }
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800928 /* THP is split, so ppage should be the real poisoned page. */
929 ppage = p;
Jin Dongmingefeda7a2011-02-01 15:52:39 -0800930 }
931 }
932
Andi Kleen6a460792009-09-16 11:50:15 +0200933 /*
934 * First collect all the processes that have the page
935 * mapped in dirty form. This has to be done before try_to_unmap,
936 * because ttu takes the rmap data structures down.
937 *
938 * Error handling: We ignore errors here because
939 * there's nothing that can be done.
940 */
941 if (kill)
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800942 collect_procs(ppage, &tokill);
Andi Kleen6a460792009-09-16 11:50:15 +0200943
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800944 if (hpage != ppage)
Jens Axboe7eaceac2011-03-10 08:52:07 +0100945 lock_page(ppage);
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800946
947 ret = try_to_unmap(ppage, ttu);
Andi Kleen6a460792009-09-16 11:50:15 +0200948 if (ret != SWAP_SUCCESS)
949 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800950 pfn, page_mapcount(ppage));
951
952 if (hpage != ppage)
953 unlock_page(ppage);
Andi Kleen6a460792009-09-16 11:50:15 +0200954
955 /*
956 * Now that the dirty bit has been propagated to the
957 * struct page and all unmaps done we can decide if
958 * killing is needed or not. Only kill when the page
959 * was dirty, otherwise the tokill list is merely
960 * freed. When there was a problem unmapping earlier
961 * use a more force-full uncatchable kill to prevent
962 * any accesses to the poisoned memory.
963 */
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800964 kill_procs_ao(&tokill, !!PageDirty(ppage), trapno,
Andi Kleen0d9ee6a2010-09-27 22:03:33 +0200965 ret != SWAP_SUCCESS, p, pfn);
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100966
967 return ret;
Andi Kleen6a460792009-09-16 11:50:15 +0200968}
969
Naoya Horiguchi7013feb2010-05-28 09:29:18 +0900970static void set_page_hwpoison_huge_page(struct page *hpage)
971{
972 int i;
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -0800973 int nr_pages = 1 << compound_trans_order(hpage);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +0900974 for (i = 0; i < nr_pages; i++)
975 SetPageHWPoison(hpage + i);
976}
977
978static void clear_page_hwpoison_huge_page(struct page *hpage)
979{
980 int i;
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -0800981 int nr_pages = 1 << compound_trans_order(hpage);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +0900982 for (i = 0; i < nr_pages; i++)
983 ClearPageHWPoison(hpage + i);
984}
985
Andi Kleen82ba0112009-12-16 12:19:57 +0100986int __memory_failure(unsigned long pfn, int trapno, int flags)
Andi Kleen6a460792009-09-16 11:50:15 +0200987{
988 struct page_state *ps;
989 struct page *p;
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900990 struct page *hpage;
Andi Kleen6a460792009-09-16 11:50:15 +0200991 int res;
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +0900992 unsigned int nr_pages;
Andi Kleen6a460792009-09-16 11:50:15 +0200993
994 if (!sysctl_memory_failure_recovery)
995 panic("Memory failure from trap %d on page %lx", trapno, pfn);
996
997 if (!pfn_valid(pfn)) {
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100998 printk(KERN_ERR
999 "MCE %#lx: memory outside kernel control\n",
1000 pfn);
1001 return -ENXIO;
Andi Kleen6a460792009-09-16 11:50:15 +02001002 }
1003
1004 p = pfn_to_page(pfn);
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001005 hpage = compound_head(p);
Andi Kleen6a460792009-09-16 11:50:15 +02001006 if (TestSetPageHWPoison(p)) {
Wu Fengguangd95ea512009-12-16 12:19:58 +01001007 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001008 return 0;
1009 }
1010
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08001011 nr_pages = 1 << compound_trans_order(hpage);
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001012 atomic_long_add(nr_pages, &mce_bad_pages);
Andi Kleen6a460792009-09-16 11:50:15 +02001013
1014 /*
1015 * We need/can do nothing about count=0 pages.
1016 * 1) it's a free page, and therefore in safe hand:
1017 * prep_new_page() will be the gate keeper.
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001018 * 2) it's a free hugepage, which is also safe:
1019 * an affected hugepage will be dequeued from hugepage freelist,
1020 * so there's no concern about reusing it ever after.
1021 * 3) it's part of a non-compound high order page.
Andi Kleen6a460792009-09-16 11:50:15 +02001022 * Implies some kernel user: cannot stop them from
1023 * R/W the page; let's pray that the page has been
1024 * used and will be freed some time later.
1025 * In fact it's dangerous to directly bump up page count from 0,
1026 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1027 */
Andi Kleen82ba0112009-12-16 12:19:57 +01001028 if (!(flags & MF_COUNT_INCREASED) &&
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001029 !get_page_unless_zero(hpage)) {
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001030 if (is_free_buddy_page(p)) {
1031 action_result(pfn, "free buddy", DELAYED);
1032 return 0;
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001033 } else if (PageHuge(hpage)) {
1034 /*
1035 * Check "just unpoisoned", "filter hit", and
1036 * "race with other subpage."
1037 */
Jens Axboe7eaceac2011-03-10 08:52:07 +01001038 lock_page(hpage);
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001039 if (!PageHWPoison(hpage)
1040 || (hwpoison_filter(p) && TestClearPageHWPoison(p))
1041 || (p != hpage && TestSetPageHWPoison(hpage))) {
1042 atomic_long_sub(nr_pages, &mce_bad_pages);
1043 return 0;
1044 }
1045 set_page_hwpoison_huge_page(hpage);
1046 res = dequeue_hwpoisoned_huge_page(hpage);
1047 action_result(pfn, "free huge",
1048 res ? IGNORED : DELAYED);
1049 unlock_page(hpage);
1050 return res;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001051 } else {
1052 action_result(pfn, "high order kernel", IGNORED);
1053 return -EBUSY;
1054 }
Andi Kleen6a460792009-09-16 11:50:15 +02001055 }
1056
1057 /*
Wu Fengguange43c3af2009-09-29 13:16:20 +08001058 * We ignore non-LRU pages for good reasons.
1059 * - PG_locked is only well defined for LRU pages and a few others
1060 * - to avoid races with __set_page_locked()
1061 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1062 * The check (unnecessarily) ignores LRU pages being isolated and
1063 * walked by the page reclaim code, however that's not a big loss.
1064 */
Jin Dongmingaf241a02011-02-01 15:52:41 -08001065 if (!PageHuge(p) && !PageTransCompound(p)) {
1066 if (!PageLRU(p))
1067 shake_page(p, 0);
1068 if (!PageLRU(p)) {
1069 /*
1070 * shake_page could have turned it free.
1071 */
1072 if (is_free_buddy_page(p)) {
1073 action_result(pfn, "free buddy, 2nd try",
1074 DELAYED);
1075 return 0;
1076 }
1077 action_result(pfn, "non LRU", IGNORED);
1078 put_page(p);
1079 return -EBUSY;
Andi Kleen0474a602009-12-16 12:20:00 +01001080 }
Wu Fengguange43c3af2009-09-29 13:16:20 +08001081 }
Wu Fengguange43c3af2009-09-29 13:16:20 +08001082
1083 /*
Andi Kleen6a460792009-09-16 11:50:15 +02001084 * Lock the page and wait for writeback to finish.
1085 * It's very difficult to mess with pages currently under IO
1086 * and in many cases impossible, so we just avoid it here.
1087 */
Jens Axboe7eaceac2011-03-10 08:52:07 +01001088 lock_page(hpage);
Wu Fengguang847ce402009-12-16 12:19:58 +01001089
1090 /*
1091 * unpoison always clear PG_hwpoison inside page lock
1092 */
1093 if (!PageHWPoison(p)) {
Wu Fengguangd95ea512009-12-16 12:19:58 +01001094 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
Wu Fengguang847ce402009-12-16 12:19:58 +01001095 res = 0;
1096 goto out;
1097 }
Wu Fengguang7c116f22009-12-16 12:19:59 +01001098 if (hwpoison_filter(p)) {
1099 if (TestClearPageHWPoison(p))
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001100 atomic_long_sub(nr_pages, &mce_bad_pages);
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001101 unlock_page(hpage);
1102 put_page(hpage);
Wu Fengguang7c116f22009-12-16 12:19:59 +01001103 return 0;
1104 }
Wu Fengguang847ce402009-12-16 12:19:58 +01001105
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001106 /*
1107 * For error on the tail page, we should set PG_hwpoison
1108 * on the head page to show that the hugepage is hwpoisoned
1109 */
Jin Dongminga6d30dd2011-02-01 15:52:40 -08001110 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001111 action_result(pfn, "hugepage already hardware poisoned",
1112 IGNORED);
1113 unlock_page(hpage);
1114 put_page(hpage);
1115 return 0;
1116 }
1117 /*
1118 * Set PG_hwpoison on all pages in an error hugepage,
1119 * because containment is done in hugepage unit for now.
1120 * Since we have done TestSetPageHWPoison() for the head page with
1121 * page lock held, we can safely set PG_hwpoison bits on tail pages.
1122 */
1123 if (PageHuge(p))
1124 set_page_hwpoison_huge_page(hpage);
1125
Andi Kleen6a460792009-09-16 11:50:15 +02001126 wait_on_page_writeback(p);
1127
1128 /*
1129 * Now take care of user space mappings.
Minchan Kime64a7822011-03-22 16:32:44 -07001130 * Abort on fail: __delete_from_page_cache() assumes unmapped page.
Andi Kleen6a460792009-09-16 11:50:15 +02001131 */
Wu Fengguang1668bfd2009-12-16 12:19:58 +01001132 if (hwpoison_user_mappings(p, pfn, trapno) != SWAP_SUCCESS) {
1133 printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
1134 res = -EBUSY;
1135 goto out;
1136 }
Andi Kleen6a460792009-09-16 11:50:15 +02001137
1138 /*
1139 * Torn down by someone else?
1140 */
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +01001141 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
Andi Kleen6a460792009-09-16 11:50:15 +02001142 action_result(pfn, "already truncated LRU", IGNORED);
Wu Fengguangd95ea512009-12-16 12:19:58 +01001143 res = -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +02001144 goto out;
1145 }
1146
1147 res = -EBUSY;
1148 for (ps = error_states;; ps++) {
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +01001149 if ((p->flags & ps->mask) == ps->res) {
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +01001150 res = page_action(ps, p, pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001151 break;
1152 }
1153 }
1154out:
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001155 unlock_page(hpage);
Andi Kleen6a460792009-09-16 11:50:15 +02001156 return res;
1157}
1158EXPORT_SYMBOL_GPL(__memory_failure);
1159
1160/**
1161 * memory_failure - Handle memory failure of a page.
1162 * @pfn: Page Number of the corrupted page
1163 * @trapno: Trap number reported in the signal to user space.
1164 *
1165 * This function is called by the low level machine check code
1166 * of an architecture when it detects hardware memory corruption
1167 * of a page. It tries its best to recover, which includes
1168 * dropping pages, killing processes etc.
1169 *
1170 * The function is primarily of use for corruptions that
1171 * happen outside the current execution context (e.g. when
1172 * detected by a background scrubber)
1173 *
1174 * Must run in process context (e.g. a work queue) with interrupts
1175 * enabled and no spinlocks hold.
1176 */
1177void memory_failure(unsigned long pfn, int trapno)
1178{
1179 __memory_failure(pfn, trapno, 0);
1180}
Wu Fengguang847ce402009-12-16 12:19:58 +01001181
Huang Yingea8f5fb2011-07-13 13:14:27 +08001182#define MEMORY_FAILURE_FIFO_ORDER 4
1183#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
1184
1185struct memory_failure_entry {
1186 unsigned long pfn;
1187 int trapno;
1188 int flags;
1189};
1190
1191struct memory_failure_cpu {
1192 DECLARE_KFIFO(fifo, struct memory_failure_entry,
1193 MEMORY_FAILURE_FIFO_SIZE);
1194 spinlock_t lock;
1195 struct work_struct work;
1196};
1197
1198static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1199
1200/**
1201 * memory_failure_queue - Schedule handling memory failure of a page.
1202 * @pfn: Page Number of the corrupted page
1203 * @trapno: Trap number reported in the signal to user space.
1204 * @flags: Flags for memory failure handling
1205 *
1206 * This function is called by the low level hardware error handler
1207 * when it detects hardware memory corruption of a page. It schedules
1208 * the recovering of error page, including dropping pages, killing
1209 * processes etc.
1210 *
1211 * The function is primarily of use for corruptions that
1212 * happen outside the current execution context (e.g. when
1213 * detected by a background scrubber)
1214 *
1215 * Can run in IRQ context.
1216 */
1217void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1218{
1219 struct memory_failure_cpu *mf_cpu;
1220 unsigned long proc_flags;
1221 struct memory_failure_entry entry = {
1222 .pfn = pfn,
1223 .trapno = trapno,
1224 .flags = flags,
1225 };
1226
1227 mf_cpu = &get_cpu_var(memory_failure_cpu);
1228 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1229 if (kfifo_put(&mf_cpu->fifo, &entry))
1230 schedule_work_on(smp_processor_id(), &mf_cpu->work);
1231 else
1232 pr_err("Memory failure: buffer overflow when queuing memory failure at 0x%#lx\n",
1233 pfn);
1234 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1235 put_cpu_var(memory_failure_cpu);
1236}
1237EXPORT_SYMBOL_GPL(memory_failure_queue);
1238
1239static void memory_failure_work_func(struct work_struct *work)
1240{
1241 struct memory_failure_cpu *mf_cpu;
1242 struct memory_failure_entry entry = { 0, };
1243 unsigned long proc_flags;
1244 int gotten;
1245
1246 mf_cpu = &__get_cpu_var(memory_failure_cpu);
1247 for (;;) {
1248 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1249 gotten = kfifo_get(&mf_cpu->fifo, &entry);
1250 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1251 if (!gotten)
1252 break;
1253 __memory_failure(entry.pfn, entry.trapno, entry.flags);
1254 }
1255}
1256
1257static int __init memory_failure_init(void)
1258{
1259 struct memory_failure_cpu *mf_cpu;
1260 int cpu;
1261
1262 for_each_possible_cpu(cpu) {
1263 mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1264 spin_lock_init(&mf_cpu->lock);
1265 INIT_KFIFO(mf_cpu->fifo);
1266 INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1267 }
1268
1269 return 0;
1270}
1271core_initcall(memory_failure_init);
1272
Wu Fengguang847ce402009-12-16 12:19:58 +01001273/**
1274 * unpoison_memory - Unpoison a previously poisoned page
1275 * @pfn: Page number of the to be unpoisoned page
1276 *
1277 * Software-unpoison a page that has been poisoned by
1278 * memory_failure() earlier.
1279 *
1280 * This is only done on the software-level, so it only works
1281 * for linux injected failures, not real hardware failures
1282 *
1283 * Returns 0 for success, otherwise -errno.
1284 */
1285int unpoison_memory(unsigned long pfn)
1286{
1287 struct page *page;
1288 struct page *p;
1289 int freeit = 0;
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001290 unsigned int nr_pages;
Wu Fengguang847ce402009-12-16 12:19:58 +01001291
1292 if (!pfn_valid(pfn))
1293 return -ENXIO;
1294
1295 p = pfn_to_page(pfn);
1296 page = compound_head(p);
1297
1298 if (!PageHWPoison(p)) {
Andi Kleenfb46e732010-09-27 23:31:30 +02001299 pr_info("MCE: Page was already unpoisoned %#lx\n", pfn);
Wu Fengguang847ce402009-12-16 12:19:58 +01001300 return 0;
1301 }
1302
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08001303 nr_pages = 1 << compound_trans_order(page);
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001304
Wu Fengguang847ce402009-12-16 12:19:58 +01001305 if (!get_page_unless_zero(page)) {
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001306 /*
1307 * Since HWPoisoned hugepage should have non-zero refcount,
1308 * race between memory failure and unpoison seems to happen.
1309 * In such case unpoison fails and memory failure runs
1310 * to the end.
1311 */
1312 if (PageHuge(page)) {
1313 pr_debug("MCE: Memory failure is now running on free hugepage %#lx\n", pfn);
1314 return 0;
1315 }
Wu Fengguang847ce402009-12-16 12:19:58 +01001316 if (TestClearPageHWPoison(p))
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001317 atomic_long_sub(nr_pages, &mce_bad_pages);
Andi Kleenfb46e732010-09-27 23:31:30 +02001318 pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
Wu Fengguang847ce402009-12-16 12:19:58 +01001319 return 0;
1320 }
1321
Jens Axboe7eaceac2011-03-10 08:52:07 +01001322 lock_page(page);
Wu Fengguang847ce402009-12-16 12:19:58 +01001323 /*
1324 * This test is racy because PG_hwpoison is set outside of page lock.
1325 * That's acceptable because that won't trigger kernel panic. Instead,
1326 * the PG_hwpoison page will be caught and isolated on the entrance to
1327 * the free buddy page pool.
1328 */
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001329 if (TestClearPageHWPoison(page)) {
Andi Kleenfb46e732010-09-27 23:31:30 +02001330 pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001331 atomic_long_sub(nr_pages, &mce_bad_pages);
Wu Fengguang847ce402009-12-16 12:19:58 +01001332 freeit = 1;
Naoya Horiguchi6a901812010-09-08 10:19:40 +09001333 if (PageHuge(page))
1334 clear_page_hwpoison_huge_page(page);
Wu Fengguang847ce402009-12-16 12:19:58 +01001335 }
1336 unlock_page(page);
1337
1338 put_page(page);
1339 if (freeit)
1340 put_page(page);
1341
1342 return 0;
1343}
1344EXPORT_SYMBOL(unpoison_memory);
Andi Kleenfacb6012009-12-16 12:20:00 +01001345
1346static struct page *new_page(struct page *p, unsigned long private, int **x)
1347{
Andi Kleen12686d12009-12-16 12:20:01 +01001348 int nid = page_to_nid(p);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001349 if (PageHuge(p))
1350 return alloc_huge_page_node(page_hstate(compound_head(p)),
1351 nid);
1352 else
1353 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
Andi Kleenfacb6012009-12-16 12:20:00 +01001354}
1355
1356/*
1357 * Safely get reference count of an arbitrary page.
1358 * Returns 0 for a free page, -EIO for a zero refcount page
1359 * that is not free, and 1 for any other page type.
1360 * For 1 the page is returned with increased page count, otherwise not.
1361 */
1362static int get_any_page(struct page *p, unsigned long pfn, int flags)
1363{
1364 int ret;
1365
1366 if (flags & MF_COUNT_INCREASED)
1367 return 1;
1368
1369 /*
KOSAKI Motohiro20d6c962010-12-02 14:31:19 -08001370 * The lock_memory_hotplug prevents a race with memory hotplug.
Andi Kleenfacb6012009-12-16 12:20:00 +01001371 * This is a big hammer, a better would be nicer.
1372 */
KOSAKI Motohiro20d6c962010-12-02 14:31:19 -08001373 lock_memory_hotplug();
Andi Kleenfacb6012009-12-16 12:20:00 +01001374
1375 /*
1376 * Isolate the page, so that it doesn't get reallocated if it
1377 * was free.
1378 */
1379 set_migratetype_isolate(p);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001380 /*
1381 * When the target page is a free hugepage, just remove it
1382 * from free hugepage list.
1383 */
Andi Kleenfacb6012009-12-16 12:20:00 +01001384 if (!get_page_unless_zero(compound_head(p))) {
Naoya Horiguchid950b952010-09-08 10:19:39 +09001385 if (PageHuge(p)) {
Andi Kleen46e387b2010-10-22 17:40:48 +02001386 pr_info("get_any_page: %#lx free huge page\n", pfn);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001387 ret = dequeue_hwpoisoned_huge_page(compound_head(p));
1388 } else if (is_free_buddy_page(p)) {
Andi Kleenfb46e732010-09-27 23:31:30 +02001389 pr_info("get_any_page: %#lx free buddy page\n", pfn);
Andi Kleenfacb6012009-12-16 12:20:00 +01001390 /* Set hwpoison bit while page is still isolated */
1391 SetPageHWPoison(p);
1392 ret = 0;
1393 } else {
Andi Kleenfb46e732010-09-27 23:31:30 +02001394 pr_info("get_any_page: %#lx: unknown zero refcount page type %lx\n",
Andi Kleenfacb6012009-12-16 12:20:00 +01001395 pfn, p->flags);
1396 ret = -EIO;
1397 }
1398 } else {
1399 /* Not a free page */
1400 ret = 1;
1401 }
1402 unset_migratetype_isolate(p);
KOSAKI Motohiro20d6c962010-12-02 14:31:19 -08001403 unlock_memory_hotplug();
Andi Kleenfacb6012009-12-16 12:20:00 +01001404 return ret;
1405}
1406
Naoya Horiguchid950b952010-09-08 10:19:39 +09001407static int soft_offline_huge_page(struct page *page, int flags)
1408{
1409 int ret;
1410 unsigned long pfn = page_to_pfn(page);
1411 struct page *hpage = compound_head(page);
1412 LIST_HEAD(pagelist);
1413
1414 ret = get_any_page(page, pfn, flags);
1415 if (ret < 0)
1416 return ret;
1417 if (ret == 0)
1418 goto done;
1419
1420 if (PageHWPoison(hpage)) {
1421 put_page(hpage);
1422 pr_debug("soft offline: %#lx hugepage already poisoned\n", pfn);
1423 return -EBUSY;
1424 }
1425
1426 /* Keep page count to indicate a given hugepage is isolated. */
1427
1428 list_add(&hpage->lru, &pagelist);
Mel Gorman77f1fe62011-01-13 15:45:57 -08001429 ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0,
1430 true);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001431 if (ret) {
Minchan Kim48db54e2011-02-01 15:52:33 -08001432 struct page *page1, *page2;
1433 list_for_each_entry_safe(page1, page2, &pagelist, lru)
1434 put_page(page1);
1435
Naoya Horiguchid950b952010-09-08 10:19:39 +09001436 pr_debug("soft offline: %#lx: migration failed %d, type %lx\n",
1437 pfn, ret, page->flags);
1438 if (ret > 0)
1439 ret = -EIO;
1440 return ret;
1441 }
1442done:
1443 if (!PageHWPoison(hpage))
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08001444 atomic_long_add(1 << compound_trans_order(hpage), &mce_bad_pages);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001445 set_page_hwpoison_huge_page(hpage);
1446 dequeue_hwpoisoned_huge_page(hpage);
1447 /* keep elevated page count for bad page */
1448 return ret;
1449}
1450
Andi Kleenfacb6012009-12-16 12:20:00 +01001451/**
1452 * soft_offline_page - Soft offline a page.
1453 * @page: page to offline
1454 * @flags: flags. Same as memory_failure().
1455 *
1456 * Returns 0 on success, otherwise negated errno.
1457 *
1458 * Soft offline a page, by migration or invalidation,
1459 * without killing anything. This is for the case when
1460 * a page is not corrupted yet (so it's still valid to access),
1461 * but has had a number of corrected errors and is better taken
1462 * out.
1463 *
1464 * The actual policy on when to do that is maintained by
1465 * user space.
1466 *
1467 * This should never impact any application or cause data loss,
1468 * however it might take some time.
1469 *
1470 * This is not a 100% solution for all memory, but tries to be
1471 * ``good enough'' for the majority of memory.
1472 */
1473int soft_offline_page(struct page *page, int flags)
1474{
1475 int ret;
1476 unsigned long pfn = page_to_pfn(page);
1477
Naoya Horiguchid950b952010-09-08 10:19:39 +09001478 if (PageHuge(page))
1479 return soft_offline_huge_page(page, flags);
1480
Andi Kleenfacb6012009-12-16 12:20:00 +01001481 ret = get_any_page(page, pfn, flags);
1482 if (ret < 0)
1483 return ret;
1484 if (ret == 0)
1485 goto done;
1486
1487 /*
1488 * Page cache page we can handle?
1489 */
1490 if (!PageLRU(page)) {
1491 /*
1492 * Try to free it.
1493 */
1494 put_page(page);
1495 shake_page(page, 1);
1496
1497 /*
1498 * Did it turn free?
1499 */
1500 ret = get_any_page(page, pfn, 0);
1501 if (ret < 0)
1502 return ret;
1503 if (ret == 0)
1504 goto done;
1505 }
1506 if (!PageLRU(page)) {
Andi Kleenfb46e732010-09-27 23:31:30 +02001507 pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n",
Andi Kleenfacb6012009-12-16 12:20:00 +01001508 pfn, page->flags);
1509 return -EIO;
1510 }
1511
1512 lock_page(page);
1513 wait_on_page_writeback(page);
1514
1515 /*
1516 * Synchronized using the page lock with memory_failure()
1517 */
1518 if (PageHWPoison(page)) {
1519 unlock_page(page);
1520 put_page(page);
Andi Kleenfb46e732010-09-27 23:31:30 +02001521 pr_info("soft offline: %#lx page already poisoned\n", pfn);
Andi Kleenfacb6012009-12-16 12:20:00 +01001522 return -EBUSY;
1523 }
1524
1525 /*
1526 * Try to invalidate first. This should work for
1527 * non dirty unmapped page cache pages.
1528 */
1529 ret = invalidate_inode_page(page);
1530 unlock_page(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001531 /*
Andi Kleenfacb6012009-12-16 12:20:00 +01001532 * RED-PEN would be better to keep it isolated here, but we
1533 * would need to fix isolation locking first.
1534 */
Andi Kleenfacb6012009-12-16 12:20:00 +01001535 if (ret == 1) {
Konstantin Khlebnikovbd486282011-05-24 17:12:20 -07001536 put_page(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001537 ret = 0;
Andi Kleenfb46e732010-09-27 23:31:30 +02001538 pr_info("soft_offline: %#lx: invalidated\n", pfn);
Andi Kleenfacb6012009-12-16 12:20:00 +01001539 goto done;
1540 }
1541
1542 /*
1543 * Simple invalidation didn't work.
1544 * Try to migrate to a new page instead. migrate.c
1545 * handles a large number of cases for us.
1546 */
1547 ret = isolate_lru_page(page);
Konstantin Khlebnikovbd486282011-05-24 17:12:20 -07001548 /*
1549 * Drop page reference which is came from get_any_page()
1550 * successful isolate_lru_page() already took another one.
1551 */
1552 put_page(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001553 if (!ret) {
1554 LIST_HEAD(pagelist);
Minchan Kim5db8a732011-06-15 15:08:48 -07001555 inc_zone_page_state(page, NR_ISOLATED_ANON +
1556 page_is_file_cache(page));
Andi Kleenfacb6012009-12-16 12:20:00 +01001557 list_add(&page->lru, &pagelist);
Mel Gorman77f1fe62011-01-13 15:45:57 -08001558 ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
1559 0, true);
Andi Kleenfacb6012009-12-16 12:20:00 +01001560 if (ret) {
Andrea Arcangeli57fc4a52011-02-01 15:52:32 -08001561 putback_lru_pages(&pagelist);
Andi Kleenfb46e732010-09-27 23:31:30 +02001562 pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
Andi Kleenfacb6012009-12-16 12:20:00 +01001563 pfn, ret, page->flags);
1564 if (ret > 0)
1565 ret = -EIO;
1566 }
1567 } else {
Andi Kleenfb46e732010-09-27 23:31:30 +02001568 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
Andi Kleenfacb6012009-12-16 12:20:00 +01001569 pfn, ret, page_count(page), page->flags);
1570 }
1571 if (ret)
1572 return ret;
1573
1574done:
1575 atomic_long_add(1, &mce_bad_pages);
1576 SetPageHWPoison(page);
1577 /* keep elevated page count for bad page */
1578 return ret;
1579}