blob: d1f3351629766ffe112e3f904565ed3035412774 [file] [log] [blame]
Andi Kleen6a460792009-09-16 11:50:15 +02001/*
2 * Copyright (C) 2008, 2009 Intel Corporation
3 * Authors: Andi Kleen, Fengguang Wu
4 *
5 * This software may be redistributed and/or modified under the terms of
6 * the GNU General Public License ("GPL") version 2 only as published by the
7 * Free Software Foundation.
8 *
9 * High level machine check handler. Handles pages reported by the
10 * hardware as being corrupted usually due to a 2bit ECC memory or cache
11 * failure.
12 *
13 * Handles page cache pages in various states. The tricky part
14 * here is that we can access any page asynchronous to other VM
15 * users, because memory failures could happen anytime and anywhere,
16 * possibly violating some of their assumptions. This is why this code
17 * has to be extremely careful. Generally it tries to use normal locking
18 * rules, as in get the standard locks, even if that means the
19 * error handling takes potentially a long time.
20 *
21 * The operation to map back from RMAP chains to processes has to walk
22 * the complete process list and has non linear complexity with the number
23 * mappings. In short it can be quite slow. But since memory corruptions
24 * are rare we hope to get away with this.
25 */
26
27/*
28 * Notebook:
29 * - hugetlb needs more code
30 * - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
31 * - pass bad pages to kdump next kernel
32 */
33#define DEBUG 1 /* remove me in 2.6.34 */
34#include <linux/kernel.h>
35#include <linux/mm.h>
36#include <linux/page-flags.h>
Wu Fengguang478c5ff2009-12-16 12:19:59 +010037#include <linux/kernel-page-flags.h>
Andi Kleen6a460792009-09-16 11:50:15 +020038#include <linux/sched.h>
Hugh Dickins01e00f82009-10-13 15:02:11 +010039#include <linux/ksm.h>
Andi Kleen6a460792009-09-16 11:50:15 +020040#include <linux/rmap.h>
41#include <linux/pagemap.h>
42#include <linux/swap.h>
43#include <linux/backing-dev.h>
Andi Kleenfacb6012009-12-16 12:20:00 +010044#include <linux/migrate.h>
45#include <linux/page-isolation.h>
46#include <linux/suspend.h>
Andi Kleen6a460792009-09-16 11:50:15 +020047#include "internal.h"
48
49int sysctl_memory_failure_early_kill __read_mostly = 0;
50
51int sysctl_memory_failure_recovery __read_mostly = 1;
52
53atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0);
54
Andi Kleen27df5062009-12-21 19:56:42 +010055#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
56
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010057u32 hwpoison_filter_enable = 0;
Wu Fengguang7c116f22009-12-16 12:19:59 +010058u32 hwpoison_filter_dev_major = ~0U;
59u32 hwpoison_filter_dev_minor = ~0U;
Wu Fengguang478c5ff2009-12-16 12:19:59 +010060u64 hwpoison_filter_flags_mask;
61u64 hwpoison_filter_flags_value;
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010062EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
Wu Fengguang7c116f22009-12-16 12:19:59 +010063EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
64EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
Wu Fengguang478c5ff2009-12-16 12:19:59 +010065EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
66EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
Wu Fengguang7c116f22009-12-16 12:19:59 +010067
68static int hwpoison_filter_dev(struct page *p)
69{
70 struct address_space *mapping;
71 dev_t dev;
72
73 if (hwpoison_filter_dev_major == ~0U &&
74 hwpoison_filter_dev_minor == ~0U)
75 return 0;
76
77 /*
78 * page_mapping() does not accept slab page
79 */
80 if (PageSlab(p))
81 return -EINVAL;
82
83 mapping = page_mapping(p);
84 if (mapping == NULL || mapping->host == NULL)
85 return -EINVAL;
86
87 dev = mapping->host->i_sb->s_dev;
88 if (hwpoison_filter_dev_major != ~0U &&
89 hwpoison_filter_dev_major != MAJOR(dev))
90 return -EINVAL;
91 if (hwpoison_filter_dev_minor != ~0U &&
92 hwpoison_filter_dev_minor != MINOR(dev))
93 return -EINVAL;
94
95 return 0;
96}
97
Wu Fengguang478c5ff2009-12-16 12:19:59 +010098static int hwpoison_filter_flags(struct page *p)
99{
100 if (!hwpoison_filter_flags_mask)
101 return 0;
102
103 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
104 hwpoison_filter_flags_value)
105 return 0;
106 else
107 return -EINVAL;
108}
109
Andi Kleen4fd466e2009-12-16 12:19:59 +0100110/*
111 * This allows stress tests to limit test scope to a collection of tasks
112 * by putting them under some memcg. This prevents killing unrelated/important
113 * processes such as /sbin/init. Note that the target task may share clean
114 * pages with init (eg. libc text), which is harmless. If the target task
115 * share _dirty_ pages with another task B, the test scheme must make sure B
116 * is also included in the memcg. At last, due to race conditions this filter
117 * can only guarantee that the page either belongs to the memcg tasks, or is
118 * a freed page.
119 */
120#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
121u64 hwpoison_filter_memcg;
122EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
123static int hwpoison_filter_task(struct page *p)
124{
125 struct mem_cgroup *mem;
126 struct cgroup_subsys_state *css;
127 unsigned long ino;
128
129 if (!hwpoison_filter_memcg)
130 return 0;
131
132 mem = try_get_mem_cgroup_from_page(p);
133 if (!mem)
134 return -EINVAL;
135
136 css = mem_cgroup_css(mem);
137 /* root_mem_cgroup has NULL dentries */
138 if (!css->cgroup->dentry)
139 return -EINVAL;
140
141 ino = css->cgroup->dentry->d_inode->i_ino;
142 css_put(css);
143
144 if (ino != hwpoison_filter_memcg)
145 return -EINVAL;
146
147 return 0;
148}
149#else
150static int hwpoison_filter_task(struct page *p) { return 0; }
151#endif
152
Wu Fengguang7c116f22009-12-16 12:19:59 +0100153int hwpoison_filter(struct page *p)
154{
Haicheng Li1bfe5fe2009-12-16 12:19:59 +0100155 if (!hwpoison_filter_enable)
156 return 0;
157
Wu Fengguang7c116f22009-12-16 12:19:59 +0100158 if (hwpoison_filter_dev(p))
159 return -EINVAL;
160
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100161 if (hwpoison_filter_flags(p))
162 return -EINVAL;
163
Andi Kleen4fd466e2009-12-16 12:19:59 +0100164 if (hwpoison_filter_task(p))
165 return -EINVAL;
166
Wu Fengguang7c116f22009-12-16 12:19:59 +0100167 return 0;
168}
Andi Kleen27df5062009-12-21 19:56:42 +0100169#else
170int hwpoison_filter(struct page *p)
171{
172 return 0;
173}
174#endif
175
Wu Fengguang7c116f22009-12-16 12:19:59 +0100176EXPORT_SYMBOL_GPL(hwpoison_filter);
177
Andi Kleen6a460792009-09-16 11:50:15 +0200178/*
179 * Send all the processes who have the page mapped an ``action optional''
180 * signal.
181 */
182static int kill_proc_ao(struct task_struct *t, unsigned long addr, int trapno,
183 unsigned long pfn)
184{
185 struct siginfo si;
186 int ret;
187
188 printk(KERN_ERR
189 "MCE %#lx: Killing %s:%d early due to hardware memory corruption\n",
190 pfn, t->comm, t->pid);
191 si.si_signo = SIGBUS;
192 si.si_errno = 0;
193 si.si_code = BUS_MCEERR_AO;
194 si.si_addr = (void *)addr;
195#ifdef __ARCH_SI_TRAPNO
196 si.si_trapno = trapno;
197#endif
198 si.si_addr_lsb = PAGE_SHIFT;
199 /*
200 * Don't use force here, it's convenient if the signal
201 * can be temporarily blocked.
202 * This could cause a loop when the user sets SIGBUS
203 * to SIG_IGN, but hopefully noone will do that?
204 */
205 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
206 if (ret < 0)
207 printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
208 t->comm, t->pid, ret);
209 return ret;
210}
211
212/*
Andi Kleen588f9ce2009-12-16 12:19:57 +0100213 * When a unknown page type is encountered drain as many buffers as possible
214 * in the hope to turn the page into a LRU or free page, which we can handle.
215 */
Andi Kleenfacb6012009-12-16 12:20:00 +0100216void shake_page(struct page *p, int access)
Andi Kleen588f9ce2009-12-16 12:19:57 +0100217{
218 if (!PageSlab(p)) {
219 lru_add_drain_all();
220 if (PageLRU(p))
221 return;
222 drain_all_pages();
223 if (PageLRU(p) || is_free_buddy_page(p))
224 return;
225 }
Andi Kleenfacb6012009-12-16 12:20:00 +0100226
Andi Kleen588f9ce2009-12-16 12:19:57 +0100227 /*
Andi Kleenfacb6012009-12-16 12:20:00 +0100228 * Only all shrink_slab here (which would also
229 * shrink other caches) if access is not potentially fatal.
Andi Kleen588f9ce2009-12-16 12:19:57 +0100230 */
Andi Kleenfacb6012009-12-16 12:20:00 +0100231 if (access) {
232 int nr;
233 do {
234 nr = shrink_slab(1000, GFP_KERNEL, 1000);
235 if (page_count(p) == 0)
236 break;
237 } while (nr > 10);
238 }
Andi Kleen588f9ce2009-12-16 12:19:57 +0100239}
240EXPORT_SYMBOL_GPL(shake_page);
241
242/*
Andi Kleen6a460792009-09-16 11:50:15 +0200243 * Kill all processes that have a poisoned page mapped and then isolate
244 * the page.
245 *
246 * General strategy:
247 * Find all processes having the page mapped and kill them.
248 * But we keep a page reference around so that the page is not
249 * actually freed yet.
250 * Then stash the page away
251 *
252 * There's no convenient way to get back to mapped processes
253 * from the VMAs. So do a brute-force search over all
254 * running processes.
255 *
256 * Remember that machine checks are not common (or rather
257 * if they are common you have other problems), so this shouldn't
258 * be a performance issue.
259 *
260 * Also there are some races possible while we get from the
261 * error detection to actually handle it.
262 */
263
264struct to_kill {
265 struct list_head nd;
266 struct task_struct *tsk;
267 unsigned long addr;
268 unsigned addr_valid:1;
269};
270
271/*
272 * Failure handling: if we can't find or can't kill a process there's
273 * not much we can do. We just print a message and ignore otherwise.
274 */
275
276/*
277 * Schedule a process for later kill.
278 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
279 * TBD would GFP_NOIO be enough?
280 */
281static void add_to_kill(struct task_struct *tsk, struct page *p,
282 struct vm_area_struct *vma,
283 struct list_head *to_kill,
284 struct to_kill **tkc)
285{
286 struct to_kill *tk;
287
288 if (*tkc) {
289 tk = *tkc;
290 *tkc = NULL;
291 } else {
292 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
293 if (!tk) {
294 printk(KERN_ERR
295 "MCE: Out of memory while machine check handling\n");
296 return;
297 }
298 }
299 tk->addr = page_address_in_vma(p, vma);
300 tk->addr_valid = 1;
301
302 /*
303 * In theory we don't have to kill when the page was
304 * munmaped. But it could be also a mremap. Since that's
305 * likely very rare kill anyways just out of paranoia, but use
306 * a SIGKILL because the error is not contained anymore.
307 */
308 if (tk->addr == -EFAULT) {
309 pr_debug("MCE: Unable to find user space address %lx in %s\n",
310 page_to_pfn(p), tsk->comm);
311 tk->addr_valid = 0;
312 }
313 get_task_struct(tsk);
314 tk->tsk = tsk;
315 list_add_tail(&tk->nd, to_kill);
316}
317
318/*
319 * Kill the processes that have been collected earlier.
320 *
321 * Only do anything when DOIT is set, otherwise just free the list
322 * (this is used for clean pages which do not need killing)
323 * Also when FAIL is set do a force kill because something went
324 * wrong earlier.
325 */
326static void kill_procs_ao(struct list_head *to_kill, int doit, int trapno,
327 int fail, unsigned long pfn)
328{
329 struct to_kill *tk, *next;
330
331 list_for_each_entry_safe (tk, next, to_kill, nd) {
332 if (doit) {
333 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200334 * In case something went wrong with munmapping
Andi Kleen6a460792009-09-16 11:50:15 +0200335 * make sure the process doesn't catch the
336 * signal and then access the memory. Just kill it.
Andi Kleen6a460792009-09-16 11:50:15 +0200337 */
338 if (fail || tk->addr_valid == 0) {
339 printk(KERN_ERR
340 "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
341 pfn, tk->tsk->comm, tk->tsk->pid);
342 force_sig(SIGKILL, tk->tsk);
343 }
344
345 /*
346 * In theory the process could have mapped
347 * something else on the address in-between. We could
348 * check for that, but we need to tell the
349 * process anyways.
350 */
351 else if (kill_proc_ao(tk->tsk, tk->addr, trapno,
352 pfn) < 0)
353 printk(KERN_ERR
354 "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
355 pfn, tk->tsk->comm, tk->tsk->pid);
356 }
357 put_task_struct(tk->tsk);
358 kfree(tk);
359 }
360}
361
362static int task_early_kill(struct task_struct *tsk)
363{
364 if (!tsk->mm)
365 return 0;
366 if (tsk->flags & PF_MCE_PROCESS)
367 return !!(tsk->flags & PF_MCE_EARLY);
368 return sysctl_memory_failure_early_kill;
369}
370
371/*
372 * Collect processes when the error hit an anonymous page.
373 */
374static void collect_procs_anon(struct page *page, struct list_head *to_kill,
375 struct to_kill **tkc)
376{
377 struct vm_area_struct *vma;
378 struct task_struct *tsk;
379 struct anon_vma *av;
380
381 read_lock(&tasklist_lock);
382 av = page_lock_anon_vma(page);
383 if (av == NULL) /* Not actually mapped anymore */
384 goto out;
385 for_each_process (tsk) {
Rik van Riel5beb4932010-03-05 13:42:07 -0800386 struct anon_vma_chain *vmac;
387
Andi Kleen6a460792009-09-16 11:50:15 +0200388 if (!task_early_kill(tsk))
389 continue;
Rik van Riel5beb4932010-03-05 13:42:07 -0800390 list_for_each_entry(vmac, &av->head, same_anon_vma) {
391 vma = vmac->vma;
Andi Kleen6a460792009-09-16 11:50:15 +0200392 if (!page_mapped_in_vma(page, vma))
393 continue;
394 if (vma->vm_mm == tsk->mm)
395 add_to_kill(tsk, page, vma, to_kill, tkc);
396 }
397 }
398 page_unlock_anon_vma(av);
399out:
400 read_unlock(&tasklist_lock);
401}
402
403/*
404 * Collect processes when the error hit a file mapped page.
405 */
406static void collect_procs_file(struct page *page, struct list_head *to_kill,
407 struct to_kill **tkc)
408{
409 struct vm_area_struct *vma;
410 struct task_struct *tsk;
411 struct prio_tree_iter iter;
412 struct address_space *mapping = page->mapping;
413
414 /*
415 * A note on the locking order between the two locks.
416 * We don't rely on this particular order.
417 * If you have some other code that needs a different order
418 * feel free to switch them around. Or add a reverse link
419 * from mm_struct to task_struct, then this could be all
420 * done without taking tasklist_lock and looping over all tasks.
421 */
422
423 read_lock(&tasklist_lock);
424 spin_lock(&mapping->i_mmap_lock);
425 for_each_process(tsk) {
426 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
427
428 if (!task_early_kill(tsk))
429 continue;
430
431 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff,
432 pgoff) {
433 /*
434 * Send early kill signal to tasks where a vma covers
435 * the page but the corrupted page is not necessarily
436 * mapped it in its pte.
437 * Assume applications who requested early kill want
438 * to be informed of all such data corruptions.
439 */
440 if (vma->vm_mm == tsk->mm)
441 add_to_kill(tsk, page, vma, to_kill, tkc);
442 }
443 }
444 spin_unlock(&mapping->i_mmap_lock);
445 read_unlock(&tasklist_lock);
446}
447
448/*
449 * Collect the processes who have the corrupted page mapped to kill.
450 * This is done in two steps for locking reasons.
451 * First preallocate one tokill structure outside the spin locks,
452 * so that we can kill at least one process reasonably reliable.
453 */
454static void collect_procs(struct page *page, struct list_head *tokill)
455{
456 struct to_kill *tk;
457
458 if (!page->mapping)
459 return;
460
461 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
462 if (!tk)
463 return;
464 if (PageAnon(page))
465 collect_procs_anon(page, tokill, &tk);
466 else
467 collect_procs_file(page, tokill, &tk);
468 kfree(tk);
469}
470
471/*
472 * Error handlers for various types of pages.
473 */
474
475enum outcome {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100476 IGNORED, /* Error: cannot be handled */
477 FAILED, /* Error: handling failed */
Andi Kleen6a460792009-09-16 11:50:15 +0200478 DELAYED, /* Will be handled later */
Andi Kleen6a460792009-09-16 11:50:15 +0200479 RECOVERED, /* Successfully recovered */
480};
481
482static const char *action_name[] = {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100483 [IGNORED] = "Ignored",
Andi Kleen6a460792009-09-16 11:50:15 +0200484 [FAILED] = "Failed",
485 [DELAYED] = "Delayed",
Andi Kleen6a460792009-09-16 11:50:15 +0200486 [RECOVERED] = "Recovered",
487};
488
489/*
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100490 * XXX: It is possible that a page is isolated from LRU cache,
491 * and then kept in swap cache or failed to remove from page cache.
492 * The page count will stop it from being freed by unpoison.
493 * Stress tests should be aware of this memory leak problem.
494 */
495static int delete_from_lru_cache(struct page *p)
496{
497 if (!isolate_lru_page(p)) {
498 /*
499 * Clear sensible page flags, so that the buddy system won't
500 * complain when the page is unpoison-and-freed.
501 */
502 ClearPageActive(p);
503 ClearPageUnevictable(p);
504 /*
505 * drop the page count elevated by isolate_lru_page()
506 */
507 page_cache_release(p);
508 return 0;
509 }
510 return -EIO;
511}
512
513/*
Andi Kleen6a460792009-09-16 11:50:15 +0200514 * Error hit kernel page.
515 * Do nothing, try to be lucky and not touch this instead. For a few cases we
516 * could be more sophisticated.
517 */
518static int me_kernel(struct page *p, unsigned long pfn)
519{
Andi Kleen6a460792009-09-16 11:50:15 +0200520 return IGNORED;
521}
522
523/*
524 * Page in unknown state. Do nothing.
525 */
526static int me_unknown(struct page *p, unsigned long pfn)
527{
528 printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
529 return FAILED;
530}
531
532/*
Andi Kleen6a460792009-09-16 11:50:15 +0200533 * Clean (or cleaned) page cache page.
534 */
535static int me_pagecache_clean(struct page *p, unsigned long pfn)
536{
537 int err;
538 int ret = FAILED;
539 struct address_space *mapping;
540
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100541 delete_from_lru_cache(p);
542
Andi Kleen6a460792009-09-16 11:50:15 +0200543 /*
544 * For anonymous pages we're done the only reference left
545 * should be the one m_f() holds.
546 */
547 if (PageAnon(p))
548 return RECOVERED;
549
550 /*
551 * Now truncate the page in the page cache. This is really
552 * more like a "temporary hole punch"
553 * Don't do this for block devices when someone else
554 * has a reference, because it could be file system metadata
555 * and that's not safe to truncate.
556 */
557 mapping = page_mapping(p);
558 if (!mapping) {
559 /*
560 * Page has been teared down in the meanwhile
561 */
562 return FAILED;
563 }
564
565 /*
566 * Truncation is a bit tricky. Enable it per file system for now.
567 *
568 * Open: to take i_mutex or not for this? Right now we don't.
569 */
570 if (mapping->a_ops->error_remove_page) {
571 err = mapping->a_ops->error_remove_page(mapping, p);
572 if (err != 0) {
573 printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
574 pfn, err);
575 } else if (page_has_private(p) &&
576 !try_to_release_page(p, GFP_NOIO)) {
577 pr_debug("MCE %#lx: failed to release buffers\n", pfn);
578 } else {
579 ret = RECOVERED;
580 }
581 } else {
582 /*
583 * If the file system doesn't support it just invalidate
584 * This fails on dirty or anything with private pages
585 */
586 if (invalidate_inode_page(p))
587 ret = RECOVERED;
588 else
589 printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
590 pfn);
591 }
592 return ret;
593}
594
595/*
596 * Dirty cache page page
597 * Issues: when the error hit a hole page the error is not properly
598 * propagated.
599 */
600static int me_pagecache_dirty(struct page *p, unsigned long pfn)
601{
602 struct address_space *mapping = page_mapping(p);
603
604 SetPageError(p);
605 /* TBD: print more information about the file. */
606 if (mapping) {
607 /*
608 * IO error will be reported by write(), fsync(), etc.
609 * who check the mapping.
610 * This way the application knows that something went
611 * wrong with its dirty file data.
612 *
613 * There's one open issue:
614 *
615 * The EIO will be only reported on the next IO
616 * operation and then cleared through the IO map.
617 * Normally Linux has two mechanisms to pass IO error
618 * first through the AS_EIO flag in the address space
619 * and then through the PageError flag in the page.
620 * Since we drop pages on memory failure handling the
621 * only mechanism open to use is through AS_AIO.
622 *
623 * This has the disadvantage that it gets cleared on
624 * the first operation that returns an error, while
625 * the PageError bit is more sticky and only cleared
626 * when the page is reread or dropped. If an
627 * application assumes it will always get error on
628 * fsync, but does other operations on the fd before
629 * and the page is dropped inbetween then the error
630 * will not be properly reported.
631 *
632 * This can already happen even without hwpoisoned
633 * pages: first on metadata IO errors (which only
634 * report through AS_EIO) or when the page is dropped
635 * at the wrong time.
636 *
637 * So right now we assume that the application DTRT on
638 * the first EIO, but we're not worse than other parts
639 * of the kernel.
640 */
641 mapping_set_error(mapping, EIO);
642 }
643
644 return me_pagecache_clean(p, pfn);
645}
646
647/*
648 * Clean and dirty swap cache.
649 *
650 * Dirty swap cache page is tricky to handle. The page could live both in page
651 * cache and swap cache(ie. page is freshly swapped in). So it could be
652 * referenced concurrently by 2 types of PTEs:
653 * normal PTEs and swap PTEs. We try to handle them consistently by calling
654 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
655 * and then
656 * - clear dirty bit to prevent IO
657 * - remove from LRU
658 * - but keep in the swap cache, so that when we return to it on
659 * a later page fault, we know the application is accessing
660 * corrupted data and shall be killed (we installed simple
661 * interception code in do_swap_page to catch it).
662 *
663 * Clean swap cache pages can be directly isolated. A later page fault will
664 * bring in the known good data from disk.
665 */
666static int me_swapcache_dirty(struct page *p, unsigned long pfn)
667{
Andi Kleen6a460792009-09-16 11:50:15 +0200668 ClearPageDirty(p);
669 /* Trigger EIO in shmem: */
670 ClearPageUptodate(p);
671
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100672 if (!delete_from_lru_cache(p))
673 return DELAYED;
674 else
675 return FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200676}
677
678static int me_swapcache_clean(struct page *p, unsigned long pfn)
679{
Andi Kleen6a460792009-09-16 11:50:15 +0200680 delete_from_swap_cache(p);
Wu Fengguange43c3af2009-09-29 13:16:20 +0800681
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100682 if (!delete_from_lru_cache(p))
683 return RECOVERED;
684 else
685 return FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200686}
687
688/*
689 * Huge pages. Needs work.
690 * Issues:
691 * No rmap support so we cannot find the original mapper. In theory could walk
692 * all MMs and look for the mappings, but that would be non atomic and racy.
693 * Need rmap for hugepages for this. Alternatively we could employ a heuristic,
694 * like just walking the current process and hoping it has it mapped (that
695 * should be usually true for the common "shared database cache" case)
696 * Should handle free huge pages and dequeue them too, but this needs to
697 * handle huge page accounting correctly.
698 */
699static int me_huge_page(struct page *p, unsigned long pfn)
700{
701 return FAILED;
702}
703
704/*
705 * Various page states we can handle.
706 *
707 * A page state is defined by its current page->flags bits.
708 * The table matches them in order and calls the right handler.
709 *
710 * This is quite tricky because we can access page at any time
711 * in its live cycle, so all accesses have to be extremly careful.
712 *
713 * This is not complete. More states could be added.
714 * For any missing state don't attempt recovery.
715 */
716
717#define dirty (1UL << PG_dirty)
718#define sc (1UL << PG_swapcache)
719#define unevict (1UL << PG_unevictable)
720#define mlock (1UL << PG_mlocked)
721#define writeback (1UL << PG_writeback)
722#define lru (1UL << PG_lru)
723#define swapbacked (1UL << PG_swapbacked)
724#define head (1UL << PG_head)
725#define tail (1UL << PG_tail)
726#define compound (1UL << PG_compound)
727#define slab (1UL << PG_slab)
Andi Kleen6a460792009-09-16 11:50:15 +0200728#define reserved (1UL << PG_reserved)
729
730static struct page_state {
731 unsigned long mask;
732 unsigned long res;
733 char *msg;
734 int (*action)(struct page *p, unsigned long pfn);
735} error_states[] = {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100736 { reserved, reserved, "reserved kernel", me_kernel },
Wu Fengguang95d01fc2009-12-16 12:19:58 +0100737 /*
738 * free pages are specially detected outside this table:
739 * PG_buddy pages only make a small fraction of all free pages.
740 */
Andi Kleen6a460792009-09-16 11:50:15 +0200741
742 /*
743 * Could in theory check if slab page is free or if we can drop
744 * currently unused objects without touching them. But just
745 * treat it as standard kernel for now.
746 */
747 { slab, slab, "kernel slab", me_kernel },
748
749#ifdef CONFIG_PAGEFLAGS_EXTENDED
750 { head, head, "huge", me_huge_page },
751 { tail, tail, "huge", me_huge_page },
752#else
753 { compound, compound, "huge", me_huge_page },
754#endif
755
756 { sc|dirty, sc|dirty, "swapcache", me_swapcache_dirty },
757 { sc|dirty, sc, "swapcache", me_swapcache_clean },
758
759 { unevict|dirty, unevict|dirty, "unevictable LRU", me_pagecache_dirty},
760 { unevict, unevict, "unevictable LRU", me_pagecache_clean},
761
Andi Kleen6a460792009-09-16 11:50:15 +0200762 { mlock|dirty, mlock|dirty, "mlocked LRU", me_pagecache_dirty },
763 { mlock, mlock, "mlocked LRU", me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200764
765 { lru|dirty, lru|dirty, "LRU", me_pagecache_dirty },
766 { lru|dirty, lru, "clean LRU", me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200767
768 /*
769 * Catchall entry: must be at end.
770 */
771 { 0, 0, "unknown page state", me_unknown },
772};
773
Andi Kleen2326c462009-12-16 12:20:00 +0100774#undef dirty
775#undef sc
776#undef unevict
777#undef mlock
778#undef writeback
779#undef lru
780#undef swapbacked
781#undef head
782#undef tail
783#undef compound
784#undef slab
785#undef reserved
786
Andi Kleen6a460792009-09-16 11:50:15 +0200787static void action_result(unsigned long pfn, char *msg, int result)
788{
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100789 struct page *page = pfn_to_page(pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200790
791 printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n",
792 pfn,
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100793 PageDirty(page) ? "dirty " : "",
Andi Kleen6a460792009-09-16 11:50:15 +0200794 msg, action_name[result]);
795}
796
797static int page_action(struct page_state *ps, struct page *p,
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100798 unsigned long pfn)
Andi Kleen6a460792009-09-16 11:50:15 +0200799{
800 int result;
Wu Fengguang7456b042009-10-19 08:15:01 +0200801 int count;
Andi Kleen6a460792009-09-16 11:50:15 +0200802
803 result = ps->action(p, pfn);
804 action_result(pfn, ps->msg, result);
Wu Fengguang7456b042009-10-19 08:15:01 +0200805
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100806 count = page_count(p) - 1;
Wu Fengguang138ce282009-12-16 12:19:58 +0100807 if (ps->action == me_swapcache_dirty && result == DELAYED)
808 count--;
809 if (count != 0) {
Andi Kleen6a460792009-09-16 11:50:15 +0200810 printk(KERN_ERR
811 "MCE %#lx: %s page still referenced by %d users\n",
Wu Fengguang7456b042009-10-19 08:15:01 +0200812 pfn, ps->msg, count);
Wu Fengguang138ce282009-12-16 12:19:58 +0100813 result = FAILED;
814 }
Andi Kleen6a460792009-09-16 11:50:15 +0200815
816 /* Could do more checks here if page looks ok */
817 /*
818 * Could adjust zone counters here to correct for the missing page.
819 */
820
Wu Fengguang138ce282009-12-16 12:19:58 +0100821 return (result == RECOVERED || result == DELAYED) ? 0 : -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +0200822}
823
824#define N_UNMAP_TRIES 5
825
826/*
827 * Do all that is necessary to remove user space mappings. Unmap
828 * the pages and send SIGBUS to the processes if the data was dirty.
829 */
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100830static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
Andi Kleen6a460792009-09-16 11:50:15 +0200831 int trapno)
832{
833 enum ttu_flags ttu = TTU_UNMAP | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
834 struct address_space *mapping;
835 LIST_HEAD(tokill);
836 int ret;
837 int i;
838 int kill = 1;
839
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100840 if (PageReserved(p) || PageSlab(p))
841 return SWAP_SUCCESS;
Andi Kleen6a460792009-09-16 11:50:15 +0200842
Andi Kleen6a460792009-09-16 11:50:15 +0200843 /*
844 * This check implies we don't kill processes if their pages
845 * are in the swap cache early. Those are always late kills.
846 */
847 if (!page_mapped(p))
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100848 return SWAP_SUCCESS;
849
850 if (PageCompound(p) || PageKsm(p))
851 return SWAP_FAIL;
Andi Kleen6a460792009-09-16 11:50:15 +0200852
853 if (PageSwapCache(p)) {
854 printk(KERN_ERR
855 "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
856 ttu |= TTU_IGNORE_HWPOISON;
857 }
858
859 /*
860 * Propagate the dirty bit from PTEs to struct page first, because we
861 * need this to decide if we should kill or just drop the page.
Wu Fengguangdb0480b2009-12-16 12:19:58 +0100862 * XXX: the dirty test could be racy: set_page_dirty() may not always
863 * be called inside page lock (it's recommended but not enforced).
Andi Kleen6a460792009-09-16 11:50:15 +0200864 */
865 mapping = page_mapping(p);
866 if (!PageDirty(p) && mapping && mapping_cap_writeback_dirty(mapping)) {
867 if (page_mkclean(p)) {
868 SetPageDirty(p);
869 } else {
870 kill = 0;
871 ttu |= TTU_IGNORE_HWPOISON;
872 printk(KERN_INFO
873 "MCE %#lx: corrupted page was clean: dropped without side effects\n",
874 pfn);
875 }
876 }
877
878 /*
879 * First collect all the processes that have the page
880 * mapped in dirty form. This has to be done before try_to_unmap,
881 * because ttu takes the rmap data structures down.
882 *
883 * Error handling: We ignore errors here because
884 * there's nothing that can be done.
885 */
886 if (kill)
887 collect_procs(p, &tokill);
888
889 /*
890 * try_to_unmap can fail temporarily due to races.
891 * Try a few times (RED-PEN better strategy?)
892 */
893 for (i = 0; i < N_UNMAP_TRIES; i++) {
894 ret = try_to_unmap(p, ttu);
895 if (ret == SWAP_SUCCESS)
896 break;
897 pr_debug("MCE %#lx: try_to_unmap retry needed %d\n", pfn, ret);
898 }
899
900 if (ret != SWAP_SUCCESS)
901 printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
902 pfn, page_mapcount(p));
903
904 /*
905 * Now that the dirty bit has been propagated to the
906 * struct page and all unmaps done we can decide if
907 * killing is needed or not. Only kill when the page
908 * was dirty, otherwise the tokill list is merely
909 * freed. When there was a problem unmapping earlier
910 * use a more force-full uncatchable kill to prevent
911 * any accesses to the poisoned memory.
912 */
913 kill_procs_ao(&tokill, !!PageDirty(p), trapno,
914 ret != SWAP_SUCCESS, pfn);
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100915
916 return ret;
Andi Kleen6a460792009-09-16 11:50:15 +0200917}
918
Andi Kleen82ba0112009-12-16 12:19:57 +0100919int __memory_failure(unsigned long pfn, int trapno, int flags)
Andi Kleen6a460792009-09-16 11:50:15 +0200920{
921 struct page_state *ps;
922 struct page *p;
923 int res;
924
925 if (!sysctl_memory_failure_recovery)
926 panic("Memory failure from trap %d on page %lx", trapno, pfn);
927
928 if (!pfn_valid(pfn)) {
Wu Fengguanga7560fc2009-12-16 12:19:57 +0100929 printk(KERN_ERR
930 "MCE %#lx: memory outside kernel control\n",
931 pfn);
932 return -ENXIO;
Andi Kleen6a460792009-09-16 11:50:15 +0200933 }
934
935 p = pfn_to_page(pfn);
936 if (TestSetPageHWPoison(p)) {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100937 printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200938 return 0;
939 }
940
941 atomic_long_add(1, &mce_bad_pages);
942
943 /*
944 * We need/can do nothing about count=0 pages.
945 * 1) it's a free page, and therefore in safe hand:
946 * prep_new_page() will be the gate keeper.
947 * 2) it's part of a non-compound high order page.
948 * Implies some kernel user: cannot stop them from
949 * R/W the page; let's pray that the page has been
950 * used and will be freed some time later.
951 * In fact it's dangerous to directly bump up page count from 0,
952 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
953 */
Andi Kleen82ba0112009-12-16 12:19:57 +0100954 if (!(flags & MF_COUNT_INCREASED) &&
955 !get_page_unless_zero(compound_head(p))) {
Wu Fengguang8d22ba12009-12-16 12:19:58 +0100956 if (is_free_buddy_page(p)) {
957 action_result(pfn, "free buddy", DELAYED);
958 return 0;
959 } else {
960 action_result(pfn, "high order kernel", IGNORED);
961 return -EBUSY;
962 }
Andi Kleen6a460792009-09-16 11:50:15 +0200963 }
964
965 /*
Wu Fengguange43c3af2009-09-29 13:16:20 +0800966 * We ignore non-LRU pages for good reasons.
967 * - PG_locked is only well defined for LRU pages and a few others
968 * - to avoid races with __set_page_locked()
969 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
970 * The check (unnecessarily) ignores LRU pages being isolated and
971 * walked by the page reclaim code, however that's not a big loss.
972 */
973 if (!PageLRU(p))
Andi Kleenfacb6012009-12-16 12:20:00 +0100974 shake_page(p, 0);
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100975 if (!PageLRU(p)) {
Andi Kleen0474a602009-12-16 12:20:00 +0100976 /*
977 * shake_page could have turned it free.
978 */
979 if (is_free_buddy_page(p)) {
980 action_result(pfn, "free buddy, 2nd try", DELAYED);
981 return 0;
982 }
Wu Fengguange43c3af2009-09-29 13:16:20 +0800983 action_result(pfn, "non LRU", IGNORED);
984 put_page(p);
985 return -EBUSY;
986 }
Wu Fengguange43c3af2009-09-29 13:16:20 +0800987
988 /*
Andi Kleen6a460792009-09-16 11:50:15 +0200989 * Lock the page and wait for writeback to finish.
990 * It's very difficult to mess with pages currently under IO
991 * and in many cases impossible, so we just avoid it here.
992 */
993 lock_page_nosync(p);
Wu Fengguang847ce402009-12-16 12:19:58 +0100994
995 /*
996 * unpoison always clear PG_hwpoison inside page lock
997 */
998 if (!PageHWPoison(p)) {
Wu Fengguangd95ea512009-12-16 12:19:58 +0100999 printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
Wu Fengguang847ce402009-12-16 12:19:58 +01001000 res = 0;
1001 goto out;
1002 }
Wu Fengguang7c116f22009-12-16 12:19:59 +01001003 if (hwpoison_filter(p)) {
1004 if (TestClearPageHWPoison(p))
1005 atomic_long_dec(&mce_bad_pages);
1006 unlock_page(p);
1007 put_page(p);
1008 return 0;
1009 }
Wu Fengguang847ce402009-12-16 12:19:58 +01001010
Andi Kleen6a460792009-09-16 11:50:15 +02001011 wait_on_page_writeback(p);
1012
1013 /*
1014 * Now take care of user space mappings.
Wu Fengguang1668bfd2009-12-16 12:19:58 +01001015 * Abort on fail: __remove_from_page_cache() assumes unmapped page.
Andi Kleen6a460792009-09-16 11:50:15 +02001016 */
Wu Fengguang1668bfd2009-12-16 12:19:58 +01001017 if (hwpoison_user_mappings(p, pfn, trapno) != SWAP_SUCCESS) {
1018 printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
1019 res = -EBUSY;
1020 goto out;
1021 }
Andi Kleen6a460792009-09-16 11:50:15 +02001022
1023 /*
1024 * Torn down by someone else?
1025 */
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +01001026 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
Andi Kleen6a460792009-09-16 11:50:15 +02001027 action_result(pfn, "already truncated LRU", IGNORED);
Wu Fengguangd95ea512009-12-16 12:19:58 +01001028 res = -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +02001029 goto out;
1030 }
1031
1032 res = -EBUSY;
1033 for (ps = error_states;; ps++) {
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +01001034 if ((p->flags & ps->mask) == ps->res) {
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +01001035 res = page_action(ps, p, pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001036 break;
1037 }
1038 }
1039out:
1040 unlock_page(p);
1041 return res;
1042}
1043EXPORT_SYMBOL_GPL(__memory_failure);
1044
1045/**
1046 * memory_failure - Handle memory failure of a page.
1047 * @pfn: Page Number of the corrupted page
1048 * @trapno: Trap number reported in the signal to user space.
1049 *
1050 * This function is called by the low level machine check code
1051 * of an architecture when it detects hardware memory corruption
1052 * of a page. It tries its best to recover, which includes
1053 * dropping pages, killing processes etc.
1054 *
1055 * The function is primarily of use for corruptions that
1056 * happen outside the current execution context (e.g. when
1057 * detected by a background scrubber)
1058 *
1059 * Must run in process context (e.g. a work queue) with interrupts
1060 * enabled and no spinlocks hold.
1061 */
1062void memory_failure(unsigned long pfn, int trapno)
1063{
1064 __memory_failure(pfn, trapno, 0);
1065}
Wu Fengguang847ce402009-12-16 12:19:58 +01001066
1067/**
1068 * unpoison_memory - Unpoison a previously poisoned page
1069 * @pfn: Page number of the to be unpoisoned page
1070 *
1071 * Software-unpoison a page that has been poisoned by
1072 * memory_failure() earlier.
1073 *
1074 * This is only done on the software-level, so it only works
1075 * for linux injected failures, not real hardware failures
1076 *
1077 * Returns 0 for success, otherwise -errno.
1078 */
1079int unpoison_memory(unsigned long pfn)
1080{
1081 struct page *page;
1082 struct page *p;
1083 int freeit = 0;
1084
1085 if (!pfn_valid(pfn))
1086 return -ENXIO;
1087
1088 p = pfn_to_page(pfn);
1089 page = compound_head(p);
1090
1091 if (!PageHWPoison(p)) {
1092 pr_debug("MCE: Page was already unpoisoned %#lx\n", pfn);
1093 return 0;
1094 }
1095
1096 if (!get_page_unless_zero(page)) {
1097 if (TestClearPageHWPoison(p))
1098 atomic_long_dec(&mce_bad_pages);
1099 pr_debug("MCE: Software-unpoisoned free page %#lx\n", pfn);
1100 return 0;
1101 }
1102
1103 lock_page_nosync(page);
1104 /*
1105 * This test is racy because PG_hwpoison is set outside of page lock.
1106 * That's acceptable because that won't trigger kernel panic. Instead,
1107 * the PG_hwpoison page will be caught and isolated on the entrance to
1108 * the free buddy page pool.
1109 */
1110 if (TestClearPageHWPoison(p)) {
1111 pr_debug("MCE: Software-unpoisoned page %#lx\n", pfn);
1112 atomic_long_dec(&mce_bad_pages);
1113 freeit = 1;
1114 }
1115 unlock_page(page);
1116
1117 put_page(page);
1118 if (freeit)
1119 put_page(page);
1120
1121 return 0;
1122}
1123EXPORT_SYMBOL(unpoison_memory);
Andi Kleenfacb6012009-12-16 12:20:00 +01001124
1125static struct page *new_page(struct page *p, unsigned long private, int **x)
1126{
Andi Kleen12686d12009-12-16 12:20:01 +01001127 int nid = page_to_nid(p);
1128 return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
Andi Kleenfacb6012009-12-16 12:20:00 +01001129}
1130
1131/*
1132 * Safely get reference count of an arbitrary page.
1133 * Returns 0 for a free page, -EIO for a zero refcount page
1134 * that is not free, and 1 for any other page type.
1135 * For 1 the page is returned with increased page count, otherwise not.
1136 */
1137static int get_any_page(struct page *p, unsigned long pfn, int flags)
1138{
1139 int ret;
1140
1141 if (flags & MF_COUNT_INCREASED)
1142 return 1;
1143
1144 /*
1145 * The lock_system_sleep prevents a race with memory hotplug,
1146 * because the isolation assumes there's only a single user.
1147 * This is a big hammer, a better would be nicer.
1148 */
1149 lock_system_sleep();
1150
1151 /*
1152 * Isolate the page, so that it doesn't get reallocated if it
1153 * was free.
1154 */
1155 set_migratetype_isolate(p);
1156 if (!get_page_unless_zero(compound_head(p))) {
1157 if (is_free_buddy_page(p)) {
1158 pr_debug("get_any_page: %#lx free buddy page\n", pfn);
1159 /* Set hwpoison bit while page is still isolated */
1160 SetPageHWPoison(p);
1161 ret = 0;
1162 } else {
1163 pr_debug("get_any_page: %#lx: unknown zero refcount page type %lx\n",
1164 pfn, p->flags);
1165 ret = -EIO;
1166 }
1167 } else {
1168 /* Not a free page */
1169 ret = 1;
1170 }
1171 unset_migratetype_isolate(p);
1172 unlock_system_sleep();
1173 return ret;
1174}
1175
1176/**
1177 * soft_offline_page - Soft offline a page.
1178 * @page: page to offline
1179 * @flags: flags. Same as memory_failure().
1180 *
1181 * Returns 0 on success, otherwise negated errno.
1182 *
1183 * Soft offline a page, by migration or invalidation,
1184 * without killing anything. This is for the case when
1185 * a page is not corrupted yet (so it's still valid to access),
1186 * but has had a number of corrected errors and is better taken
1187 * out.
1188 *
1189 * The actual policy on when to do that is maintained by
1190 * user space.
1191 *
1192 * This should never impact any application or cause data loss,
1193 * however it might take some time.
1194 *
1195 * This is not a 100% solution for all memory, but tries to be
1196 * ``good enough'' for the majority of memory.
1197 */
1198int soft_offline_page(struct page *page, int flags)
1199{
1200 int ret;
1201 unsigned long pfn = page_to_pfn(page);
1202
1203 ret = get_any_page(page, pfn, flags);
1204 if (ret < 0)
1205 return ret;
1206 if (ret == 0)
1207 goto done;
1208
1209 /*
1210 * Page cache page we can handle?
1211 */
1212 if (!PageLRU(page)) {
1213 /*
1214 * Try to free it.
1215 */
1216 put_page(page);
1217 shake_page(page, 1);
1218
1219 /*
1220 * Did it turn free?
1221 */
1222 ret = get_any_page(page, pfn, 0);
1223 if (ret < 0)
1224 return ret;
1225 if (ret == 0)
1226 goto done;
1227 }
1228 if (!PageLRU(page)) {
1229 pr_debug("soft_offline: %#lx: unknown non LRU page type %lx\n",
1230 pfn, page->flags);
1231 return -EIO;
1232 }
1233
1234 lock_page(page);
1235 wait_on_page_writeback(page);
1236
1237 /*
1238 * Synchronized using the page lock with memory_failure()
1239 */
1240 if (PageHWPoison(page)) {
1241 unlock_page(page);
1242 put_page(page);
1243 pr_debug("soft offline: %#lx page already poisoned\n", pfn);
1244 return -EBUSY;
1245 }
1246
1247 /*
1248 * Try to invalidate first. This should work for
1249 * non dirty unmapped page cache pages.
1250 */
1251 ret = invalidate_inode_page(page);
1252 unlock_page(page);
1253
1254 /*
1255 * Drop count because page migration doesn't like raised
1256 * counts. The page could get re-allocated, but if it becomes
1257 * LRU the isolation will just fail.
1258 * RED-PEN would be better to keep it isolated here, but we
1259 * would need to fix isolation locking first.
1260 */
1261 put_page(page);
1262 if (ret == 1) {
1263 ret = 0;
1264 pr_debug("soft_offline: %#lx: invalidated\n", pfn);
1265 goto done;
1266 }
1267
1268 /*
1269 * Simple invalidation didn't work.
1270 * Try to migrate to a new page instead. migrate.c
1271 * handles a large number of cases for us.
1272 */
1273 ret = isolate_lru_page(page);
1274 if (!ret) {
1275 LIST_HEAD(pagelist);
1276
1277 list_add(&page->lru, &pagelist);
1278 ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0);
1279 if (ret) {
1280 pr_debug("soft offline: %#lx: migration failed %d, type %lx\n",
1281 pfn, ret, page->flags);
1282 if (ret > 0)
1283 ret = -EIO;
1284 }
1285 } else {
1286 pr_debug("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
1287 pfn, ret, page_count(page), page->flags);
1288 }
1289 if (ret)
1290 return ret;
1291
1292done:
1293 atomic_long_add(1, &mce_bad_pages);
1294 SetPageHWPoison(page);
1295 /* keep elevated page count for bad page */
1296 return ret;
1297}