blob: 3f9a41cf0ac6641b9a3fa4f0b1858df878660b7a [file] [log] [blame]
Andrey Ryabinin69786cdb2015-08-13 08:37:24 +03001/*
2 * This file contains some kasan initialization code.
3 *
4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 */
12
13#include <linux/bootmem.h>
14#include <linux/init.h>
15#include <linux/kasan.h>
16#include <linux/kernel.h>
17#include <linux/memblock.h>
18#include <linux/pfn.h>
19
20#include <asm/page.h>
21#include <asm/pgalloc.h>
22
23/*
24 * This page serves two purposes:
25 * - It used as early shadow memory. The entire shadow region populated
26 * with this page, before we will be able to setup normal shadow memory.
27 * - Latter it reused it as zero shadow to cover large ranges of memory
28 * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
29 */
30unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
31
32#if CONFIG_PGTABLE_LEVELS > 3
33pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
34#endif
35#if CONFIG_PGTABLE_LEVELS > 2
36pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
37#endif
38pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
39
40static __init void *early_alloc(size_t size, int node)
41{
42 return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
43 BOOTMEM_ALLOC_ACCESSIBLE, node);
44}
45
46static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
47 unsigned long end)
48{
49 pte_t *pte = pte_offset_kernel(pmd, addr);
50 pte_t zero_pte;
51
52 zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL);
53 zero_pte = pte_wrprotect(zero_pte);
54
55 while (addr + PAGE_SIZE <= end) {
56 set_pte_at(&init_mm, addr, pte, zero_pte);
57 addr += PAGE_SIZE;
58 pte = pte_offset_kernel(pmd, addr);
59 }
60}
61
62static void __init zero_pmd_populate(pud_t *pud, unsigned long addr,
63 unsigned long end)
64{
65 pmd_t *pmd = pmd_offset(pud, addr);
66 unsigned long next;
67
68 do {
69 next = pmd_addr_end(addr, end);
70
71 if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
72 pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
73 continue;
74 }
75
76 if (pmd_none(*pmd)) {
77 pmd_populate_kernel(&init_mm, pmd,
78 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
79 }
80 zero_pte_populate(pmd, addr, next);
81 } while (pmd++, addr = next, addr != end);
82}
83
84static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
85 unsigned long end)
86{
87 pud_t *pud = pud_offset(pgd, addr);
88 unsigned long next;
89
90 do {
91 next = pud_addr_end(addr, end);
92 if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
93 pmd_t *pmd;
94
95 pud_populate(&init_mm, pud, kasan_zero_pmd);
96 pmd = pmd_offset(pud, addr);
97 pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
98 continue;
99 }
100
101 if (pud_none(*pud)) {
102 pud_populate(&init_mm, pud,
103 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
104 }
105 zero_pmd_populate(pud, addr, next);
106 } while (pud++, addr = next, addr != end);
107}
108
109/**
110 * kasan_populate_zero_shadow - populate shadow memory region with
111 * kasan_zero_page
112 * @shadow_start - start of the memory range to populate
113 * @shadow_end - end of the memory range to populate
114 */
115void __init kasan_populate_zero_shadow(const void *shadow_start,
116 const void *shadow_end)
117{
118 unsigned long addr = (unsigned long)shadow_start;
119 unsigned long end = (unsigned long)shadow_end;
120 pgd_t *pgd = pgd_offset_k(addr);
121 unsigned long next;
122
123 do {
124 next = pgd_addr_end(addr, end);
125
126 if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
127 pud_t *pud;
128 pmd_t *pmd;
129
130 /*
131 * kasan_zero_pud should be populated with pmds
132 * at this moment.
133 * [pud,pmd]_populate*() below needed only for
134 * 3,2 - level page tables where we don't have
135 * puds,pmds, so pgd_populate(), pud_populate()
136 * is noops.
137 */
138 pgd_populate(&init_mm, pgd, kasan_zero_pud);
139 pud = pud_offset(pgd, addr);
140 pud_populate(&init_mm, pud, kasan_zero_pmd);
141 pmd = pmd_offset(pud, addr);
142 pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
143 continue;
144 }
145
146 if (pgd_none(*pgd)) {
147 pgd_populate(&init_mm, pgd,
148 early_alloc(PAGE_SIZE, NUMA_NO_NODE));
149 }
150 zero_pud_populate(pgd, addr, next);
151 } while (pgd++, addr = next, addr != end);
152}