Andrey Ryabinin | 8515522 | 2015-07-02 12:09:37 +0300 | [diff] [blame] | 1 | #define pr_fmt(fmt) "kasan: " fmt |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 2 | #include <linux/bootmem.h> |
| 3 | #include <linux/kasan.h> |
| 4 | #include <linux/kdebug.h> |
| 5 | #include <linux/mm.h> |
| 6 | #include <linux/sched.h> |
| 7 | #include <linux/vmalloc.h> |
| 8 | |
| 9 | #include <asm/tlbflush.h> |
| 10 | #include <asm/sections.h> |
| 11 | |
| 12 | extern pgd_t early_level4_pgt[PTRS_PER_PGD]; |
| 13 | extern struct range pfn_mapped[E820_X_MAX]; |
| 14 | |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 15 | static int __init map_range(struct range *range) |
| 16 | { |
| 17 | unsigned long start; |
| 18 | unsigned long end; |
| 19 | |
| 20 | start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start)); |
| 21 | end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end)); |
| 22 | |
| 23 | /* |
| 24 | * end + 1 here is intentional. We check several shadow bytes in advance |
| 25 | * to slightly speed up fastpath. In some rare cases we could cross |
| 26 | * boundary of mapped shadow, so we just map some more here. |
| 27 | */ |
| 28 | return vmemmap_populate(start, end + 1, NUMA_NO_NODE); |
| 29 | } |
| 30 | |
| 31 | static void __init clear_pgds(unsigned long start, |
| 32 | unsigned long end) |
| 33 | { |
| 34 | for (; start < end; start += PGDIR_SIZE) |
| 35 | pgd_clear(pgd_offset_k(start)); |
| 36 | } |
| 37 | |
Alexander Popov | 5d5aa3c | 2015-07-02 12:09:34 +0300 | [diff] [blame] | 38 | static void __init kasan_map_early_shadow(pgd_t *pgd) |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 39 | { |
| 40 | int i; |
| 41 | unsigned long start = KASAN_SHADOW_START; |
| 42 | unsigned long end = KASAN_SHADOW_END; |
| 43 | |
| 44 | for (i = pgd_index(start); start < end; i++) { |
| 45 | pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud) |
| 46 | | _KERNPG_TABLE); |
| 47 | start += PGDIR_SIZE; |
| 48 | } |
| 49 | } |
| 50 | |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 51 | #ifdef CONFIG_KASAN_INLINE |
| 52 | static int kasan_die_handler(struct notifier_block *self, |
| 53 | unsigned long val, |
| 54 | void *data) |
| 55 | { |
| 56 | if (val == DIE_GPF) { |
Dmitry Vyukov | 2ba7805 | 2016-07-14 12:06:53 -0700 | [diff] [blame^] | 57 | pr_emerg("CONFIG_KASAN_INLINE enabled\n"); |
| 58 | pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n"); |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 59 | } |
| 60 | return NOTIFY_OK; |
| 61 | } |
| 62 | |
| 63 | static struct notifier_block kasan_die_notifier = { |
| 64 | .notifier_call = kasan_die_handler, |
| 65 | }; |
| 66 | #endif |
| 67 | |
Alexander Popov | 5d5aa3c | 2015-07-02 12:09:34 +0300 | [diff] [blame] | 68 | void __init kasan_early_init(void) |
| 69 | { |
| 70 | int i; |
| 71 | pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL; |
| 72 | pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE; |
| 73 | pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE; |
| 74 | |
| 75 | for (i = 0; i < PTRS_PER_PTE; i++) |
| 76 | kasan_zero_pte[i] = __pte(pte_val); |
| 77 | |
| 78 | for (i = 0; i < PTRS_PER_PMD; i++) |
| 79 | kasan_zero_pmd[i] = __pmd(pmd_val); |
| 80 | |
| 81 | for (i = 0; i < PTRS_PER_PUD; i++) |
| 82 | kasan_zero_pud[i] = __pud(pud_val); |
| 83 | |
| 84 | kasan_map_early_shadow(early_level4_pgt); |
| 85 | kasan_map_early_shadow(init_level4_pgt); |
| 86 | } |
| 87 | |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 88 | void __init kasan_init(void) |
| 89 | { |
| 90 | int i; |
| 91 | |
| 92 | #ifdef CONFIG_KASAN_INLINE |
| 93 | register_die_notifier(&kasan_die_notifier); |
| 94 | #endif |
| 95 | |
| 96 | memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt)); |
| 97 | load_cr3(early_level4_pgt); |
Andrey Ryabinin | 241d2c5 | 2015-07-02 12:09:35 +0300 | [diff] [blame] | 98 | __flush_tlb_all(); |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 99 | |
| 100 | clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); |
| 101 | |
Andrey Ryabinin | 69786cdb | 2015-08-13 08:37:24 +0300 | [diff] [blame] | 102 | kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 103 | kasan_mem_to_shadow((void *)PAGE_OFFSET)); |
| 104 | |
| 105 | for (i = 0; i < E820_X_MAX; i++) { |
| 106 | if (pfn_mapped[i].end == 0) |
| 107 | break; |
| 108 | |
| 109 | if (map_range(&pfn_mapped[i])) |
| 110 | panic("kasan: unable to allocate shadow!"); |
| 111 | } |
Andrey Ryabinin | 69786cdb | 2015-08-13 08:37:24 +0300 | [diff] [blame] | 112 | kasan_populate_zero_shadow( |
| 113 | kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), |
| 114 | kasan_mem_to_shadow((void *)__START_KERNEL_map)); |
Andrey Ryabinin | c420f16 | 2015-02-13 14:39:59 -0800 | [diff] [blame] | 115 | |
| 116 | vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext), |
| 117 | (unsigned long)kasan_mem_to_shadow(_end), |
| 118 | NUMA_NO_NODE); |
| 119 | |
Andrey Ryabinin | 69786cdb | 2015-08-13 08:37:24 +0300 | [diff] [blame] | 120 | kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), |
Andrey Ryabinin | c420f16 | 2015-02-13 14:39:59 -0800 | [diff] [blame] | 121 | (void *)KASAN_SHADOW_END); |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 122 | |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 123 | load_cr3(init_level4_pgt); |
Andrey Ryabinin | 241d2c5 | 2015-07-02 12:09:35 +0300 | [diff] [blame] | 124 | __flush_tlb_all(); |
Andrey Ryabinin | 8515522 | 2015-07-02 12:09:37 +0300 | [diff] [blame] | 125 | |
Andrey Ryabinin | 69e0210 | 2016-01-11 15:51:18 +0300 | [diff] [blame] | 126 | /* |
| 127 | * kasan_zero_page has been used as early shadow memory, thus it may |
Andrey Ryabinin | 063fb3e | 2016-01-11 15:51:19 +0300 | [diff] [blame] | 128 | * contain some garbage. Now we can clear and write protect it, since |
| 129 | * after the TLB flush no one should write to it. |
Andrey Ryabinin | 69e0210 | 2016-01-11 15:51:18 +0300 | [diff] [blame] | 130 | */ |
| 131 | memset(kasan_zero_page, 0, PAGE_SIZE); |
Andrey Ryabinin | 063fb3e | 2016-01-11 15:51:19 +0300 | [diff] [blame] | 132 | for (i = 0; i < PTRS_PER_PTE; i++) { |
| 133 | pte_t pte = __pte(__pa(kasan_zero_page) | __PAGE_KERNEL_RO); |
| 134 | set_pte(&kasan_zero_pte[i], pte); |
| 135 | } |
| 136 | /* Flush TLBs again to be sure that write protection applied. */ |
| 137 | __flush_tlb_all(); |
Andrey Ryabinin | 69e0210 | 2016-01-11 15:51:18 +0300 | [diff] [blame] | 138 | |
| 139 | init_task.kasan_depth = 0; |
Andrey Konovalov | 25add7e | 2015-11-05 18:51:03 -0800 | [diff] [blame] | 140 | pr_info("KernelAddressSanitizer initialized\n"); |
Andrey Ryabinin | ef7f0d6 | 2015-02-13 14:39:25 -0800 | [diff] [blame] | 141 | } |