blob: e83fd44867deff2bdcdf953329445441eff40f5f [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Akinobu Mita6a11f752009-03-31 15:23:17 -07002#include <linux/kernel.h>
Akinobu Mita8c5fb8e2011-10-31 17:08:10 -07003#include <linux/string.h>
Akinobu Mita6a11f752009-03-31 15:23:17 -07004#include <linux/mm.h>
Akinobu Mita64212ec2011-10-31 17:08:38 -07005#include <linux/highmem.h>
Joonsoo Kime30825f2014-12-12 16:55:49 -08006#include <linux/page_ext.h>
Akinobu Mita6a11f752009-03-31 15:23:17 -07007#include <linux/poison.h>
Akinobu Mita77311132011-10-31 17:08:05 -07008#include <linux/ratelimit.h>
Akinobu Mita6a11f752009-03-31 15:23:17 -07009
Laura Abbott8823b1d2016-03-15 14:56:27 -070010static bool want_page_poisoning __read_mostly;
11
12static int early_page_poison_param(char *buf)
13{
14 if (!buf)
15 return -EINVAL;
Minfei Huang2a138dc2016-05-20 16:58:13 -070016 return strtobool(buf, &want_page_poisoning);
Laura Abbott8823b1d2016-03-15 14:56:27 -070017}
18early_param("page_poison", early_page_poison_param);
19
20bool page_poisoning_enabled(void)
21{
Laura Abbott8823b1d2016-03-15 14:56:27 -070022 /*
Vinayak Menonbd33ef32017-05-03 14:54:42 -070023 * Assumes that debug_pagealloc_enabled is set before
24 * free_all_bootmem.
25 * Page poisoning is debug page alloc for some arches. If
26 * either of those options are enabled, enable poisoning.
Laura Abbott8823b1d2016-03-15 14:56:27 -070027 */
Vinayak Menonbd33ef32017-05-03 14:54:42 -070028 return (want_page_poisoning ||
29 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
30 debug_pagealloc_enabled()));
Akinobu Mita6a11f752009-03-31 15:23:17 -070031}
32
Akinobu Mita6a11f752009-03-31 15:23:17 -070033static void poison_page(struct page *page)
34{
Akinobu Mita64212ec2011-10-31 17:08:38 -070035 void *addr = kmap_atomic(page);
Akinobu Mita6a11f752009-03-31 15:23:17 -070036
Akinobu Mita6a11f752009-03-31 15:23:17 -070037 memset(addr, PAGE_POISON, PAGE_SIZE);
Akinobu Mita64212ec2011-10-31 17:08:38 -070038 kunmap_atomic(addr);
Akinobu Mita6a11f752009-03-31 15:23:17 -070039}
40
41static void poison_pages(struct page *page, int n)
42{
43 int i;
44
45 for (i = 0; i < n; i++)
46 poison_page(page + i);
47}
48
49static bool single_bit_flip(unsigned char a, unsigned char b)
50{
51 unsigned char error = a ^ b;
52
53 return error && !(error & (error - 1));
54}
55
56static void check_poison_mem(unsigned char *mem, size_t bytes)
57{
Akinobu Mita77311132011-10-31 17:08:05 -070058 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10);
Akinobu Mita6a11f752009-03-31 15:23:17 -070059 unsigned char *start;
60 unsigned char *end;
61
Laura Abbott8823b1d2016-03-15 14:56:27 -070062 if (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY))
63 return;
64
Akinobu Mita8c5fb8e2011-10-31 17:08:10 -070065 start = memchr_inv(mem, PAGE_POISON, bytes);
66 if (!start)
Akinobu Mita6a11f752009-03-31 15:23:17 -070067 return;
68
69 for (end = mem + bytes - 1; end > start; end--) {
70 if (*end != PAGE_POISON)
71 break;
72 }
73
Akinobu Mita77311132011-10-31 17:08:05 -070074 if (!__ratelimit(&ratelimit))
Akinobu Mita6a11f752009-03-31 15:23:17 -070075 return;
76 else if (start == end && single_bit_flip(*start, PAGE_POISON))
Laura Abbott8823b1d2016-03-15 14:56:27 -070077 pr_err("pagealloc: single bit error\n");
Akinobu Mita6a11f752009-03-31 15:23:17 -070078 else
Laura Abbott8823b1d2016-03-15 14:56:27 -070079 pr_err("pagealloc: memory corruption\n");
Akinobu Mita6a11f752009-03-31 15:23:17 -070080
81 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, start,
82 end - start + 1, 1);
83 dump_stack();
84}
85
Akinobu Mita6a11f752009-03-31 15:23:17 -070086static void unpoison_page(struct page *page)
87{
Akinobu Mita64212ec2011-10-31 17:08:38 -070088 void *addr;
Akinobu Mita6a11f752009-03-31 15:23:17 -070089
Akinobu Mita64212ec2011-10-31 17:08:38 -070090 addr = kmap_atomic(page);
Vinayak Menonbd33ef32017-05-03 14:54:42 -070091 /*
92 * Page poisoning when enabled poisons each and every page
93 * that is freed to buddy. Thus no extra check is done to
94 * see if a page was posioned.
95 */
Akinobu Mita64212ec2011-10-31 17:08:38 -070096 check_poison_mem(addr, PAGE_SIZE);
Akinobu Mita64212ec2011-10-31 17:08:38 -070097 kunmap_atomic(addr);
Akinobu Mita6a11f752009-03-31 15:23:17 -070098}
99
100static void unpoison_pages(struct page *page, int n)
101{
102 int i;
103
104 for (i = 0; i < n; i++)
105 unpoison_page(page + i);
106}
107
Laura Abbott8823b1d2016-03-15 14:56:27 -0700108void kernel_poison_pages(struct page *page, int numpages, int enable)
Akinobu Mita6a11f752009-03-31 15:23:17 -0700109{
Laura Abbott8823b1d2016-03-15 14:56:27 -0700110 if (!page_poisoning_enabled())
Joonsoo Kime30825f2014-12-12 16:55:49 -0800111 return;
112
Akinobu Mita6a11f752009-03-31 15:23:17 -0700113 if (enable)
114 unpoison_pages(page, numpages);
115 else
116 poison_pages(page, numpages);
117}
Laura Abbott8823b1d2016-03-15 14:56:27 -0700118
119#ifndef CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC
120void __kernel_map_pages(struct page *page, int numpages, int enable)
121{
122 /* This function does nothing, all work is done via poison pages */
123}
124#endif