Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Based on arch/arm/mm/mmu.c |
| 3 | * |
| 4 | * Copyright (C) 1995-2005 Russell King |
| 5 | * Copyright (C) 2012 ARM Ltd. |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 18 | */ |
| 19 | |
Jisheng Zhang | 5a9e3e1 | 2016-08-15 14:45:46 +0800 | [diff] [blame] | 20 | #include <linux/cache.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 21 | #include <linux/export.h> |
| 22 | #include <linux/kernel.h> |
| 23 | #include <linux/errno.h> |
| 24 | #include <linux/init.h> |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 25 | #include <linux/libfdt.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 26 | #include <linux/mman.h> |
| 27 | #include <linux/nodemask.h> |
| 28 | #include <linux/memblock.h> |
| 29 | #include <linux/fs.h> |
Catalin Marinas | 2475ff9 | 2012-10-23 14:55:08 +0100 | [diff] [blame] | 30 | #include <linux/io.h> |
Catalin Marinas | 4108935 | 2015-01-29 17:33:35 +0000 | [diff] [blame] | 31 | #include <linux/slab.h> |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 32 | #include <linux/stop_machine.h> |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 33 | #include <linux/mm.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 34 | |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 35 | #include <asm/barrier.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 36 | #include <asm/cputype.h> |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 37 | #include <asm/fixmap.h> |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 38 | #include <asm/kasan.h> |
Suzuki K. Poulose | b433dce | 2015-10-19 14:19:28 +0100 | [diff] [blame] | 39 | #include <asm/kernel-pgtable.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 40 | #include <asm/sections.h> |
| 41 | #include <asm/setup.h> |
| 42 | #include <asm/sizes.h> |
| 43 | #include <asm/tlb.h> |
Jungseok Lee | c79b954 | 2014-05-12 18:40:51 +0900 | [diff] [blame] | 44 | #include <asm/memblock.h> |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 45 | #include <asm/mmu_context.h> |
| 46 | |
Ard Biesheuvel | dd006da | 2015-03-19 16:42:27 +0000 | [diff] [blame] | 47 | u64 idmap_t0sz = TCR_T0SZ(VA_BITS); |
| 48 | |
Jisheng Zhang | 5a9e3e1 | 2016-08-15 14:45:46 +0800 | [diff] [blame] | 49 | u64 kimage_voffset __ro_after_init; |
Ard Biesheuvel | a7f8de1 | 2016-02-16 13:52:42 +0100 | [diff] [blame] | 50 | EXPORT_SYMBOL(kimage_voffset); |
| 51 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 52 | /* |
| 53 | * Empty_zero_page is a special page that is used for zero-initialized data |
| 54 | * and COW. |
| 55 | */ |
Mark Rutland | 5227cfa | 2016-01-25 11:44:57 +0000 | [diff] [blame] | 56 | unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 57 | EXPORT_SYMBOL(empty_zero_page); |
| 58 | |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 59 | static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; |
| 60 | static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; |
| 61 | static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; |
| 62 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 63 | pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
| 64 | unsigned long size, pgprot_t vma_prot) |
| 65 | { |
| 66 | if (!pfn_valid(pfn)) |
| 67 | return pgprot_noncached(vma_prot); |
| 68 | else if (file->f_flags & O_SYNC) |
| 69 | return pgprot_writecombine(vma_prot); |
| 70 | return vma_prot; |
| 71 | } |
| 72 | EXPORT_SYMBOL(phys_mem_access_prot); |
| 73 | |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 74 | static phys_addr_t __init early_pgtable_alloc(void) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 75 | { |
Suzuki K. Poulose | 7142392 | 2015-11-20 17:45:40 +0000 | [diff] [blame] | 76 | phys_addr_t phys; |
| 77 | void *ptr; |
| 78 | |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 79 | phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 80 | |
| 81 | /* |
| 82 | * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE |
| 83 | * slot will be free, so we can (ab)use the FIX_PTE slot to initialise |
| 84 | * any level of table. |
| 85 | */ |
| 86 | ptr = pte_set_fixmap(phys); |
| 87 | |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 88 | memset(ptr, 0, PAGE_SIZE); |
| 89 | |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 90 | /* |
| 91 | * Implicit barriers also ensure the zeroed page is visible to the page |
| 92 | * table walker |
| 93 | */ |
| 94 | pte_clear_fixmap(); |
| 95 | |
| 96 | return phys; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 97 | } |
| 98 | |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 99 | static void alloc_init_pte(pmd_t *pmd, unsigned long addr, |
Catalin Marinas | 667c275 | 2015-11-26 15:42:41 +0000 | [diff] [blame] | 100 | unsigned long end, unsigned long pfn, |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 101 | pgprot_t prot, |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 102 | phys_addr_t (*pgtable_alloc)(void)) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 103 | { |
| 104 | pte_t *pte; |
| 105 | |
Catalin Marinas | 4133af6 | 2016-06-29 14:51:29 +0200 | [diff] [blame] | 106 | BUG_ON(pmd_sect(*pmd)); |
| 107 | if (pmd_none(*pmd)) { |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 108 | phys_addr_t pte_phys; |
| 109 | BUG_ON(!pgtable_alloc); |
| 110 | pte_phys = pgtable_alloc(); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 111 | pte = pte_set_fixmap(pte_phys); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 112 | __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 113 | pte_clear_fixmap(); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 114 | } |
Mark Rutland | a1c7657 | 2015-01-27 16:36:30 +0000 | [diff] [blame] | 115 | BUG_ON(pmd_bad(*pmd)); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 116 | |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 117 | pte = pte_set_fixmap_offset(pmd, addr); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 118 | do { |
Catalin Marinas | 667c275 | 2015-11-26 15:42:41 +0000 | [diff] [blame] | 119 | set_pte(pte, pfn_pte(pfn, prot)); |
| 120 | pfn++; |
| 121 | } while (pte++, addr += PAGE_SIZE, addr != end); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 122 | |
| 123 | pte_clear_fixmap(); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 124 | } |
| 125 | |
Mark Rutland | 11509a3 | 2016-01-25 11:45:10 +0000 | [diff] [blame] | 126 | static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 127 | phys_addr_t phys, pgprot_t prot, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 128 | phys_addr_t (*pgtable_alloc)(void), |
| 129 | bool allow_block_mappings) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 130 | { |
| 131 | pmd_t *pmd; |
| 132 | unsigned long next; |
| 133 | |
| 134 | /* |
| 135 | * Check for initial section mappings in the pgd/pud and remove them. |
| 136 | */ |
Catalin Marinas | 4133af6 | 2016-06-29 14:51:29 +0200 | [diff] [blame] | 137 | BUG_ON(pud_sect(*pud)); |
| 138 | if (pud_none(*pud)) { |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 139 | phys_addr_t pmd_phys; |
| 140 | BUG_ON(!pgtable_alloc); |
| 141 | pmd_phys = pgtable_alloc(); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 142 | pmd = pmd_set_fixmap(pmd_phys); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 143 | __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 144 | pmd_clear_fixmap(); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 145 | } |
Mark Rutland | a1c7657 | 2015-01-27 16:36:30 +0000 | [diff] [blame] | 146 | BUG_ON(pud_bad(*pud)); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 147 | |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 148 | pmd = pmd_set_fixmap_offset(pud, addr); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 149 | do { |
| 150 | next = pmd_addr_end(addr, end); |
| 151 | /* try section mapping first */ |
Laura Abbott | 83863f2 | 2016-02-05 16:24:47 -0800 | [diff] [blame] | 152 | if (((addr | next | phys) & ~SECTION_MASK) == 0 && |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 153 | allow_block_mappings) { |
Catalin Marinas | a55f992 | 2014-02-04 16:01:31 +0000 | [diff] [blame] | 154 | pmd_t old_pmd =*pmd; |
Mark Rutland | c661cb1 | 2016-03-22 10:11:45 +0000 | [diff] [blame] | 155 | pmd_set_huge(pmd, phys, prot); |
Catalin Marinas | a55f992 | 2014-02-04 16:01:31 +0000 | [diff] [blame] | 156 | /* |
| 157 | * Check for previous table entries created during |
| 158 | * boot (__create_page_tables) and flush them. |
| 159 | */ |
zhichang.yuan | 523d6e9 | 2014-12-09 07:26:47 +0000 | [diff] [blame] | 160 | if (!pmd_none(old_pmd)) { |
Catalin Marinas | a55f992 | 2014-02-04 16:01:31 +0000 | [diff] [blame] | 161 | flush_tlb_all(); |
zhichang.yuan | 523d6e9 | 2014-12-09 07:26:47 +0000 | [diff] [blame] | 162 | if (pmd_table(old_pmd)) { |
Mark Rutland | 316b39db | 2016-01-25 11:45:05 +0000 | [diff] [blame] | 163 | phys_addr_t table = pmd_page_paddr(old_pmd); |
Catalin Marinas | 4108935 | 2015-01-29 17:33:35 +0000 | [diff] [blame] | 164 | if (!WARN_ON_ONCE(slab_is_available())) |
| 165 | memblock_free(table, PAGE_SIZE); |
zhichang.yuan | 523d6e9 | 2014-12-09 07:26:47 +0000 | [diff] [blame] | 166 | } |
| 167 | } |
Catalin Marinas | a55f992 | 2014-02-04 16:01:31 +0000 | [diff] [blame] | 168 | } else { |
Catalin Marinas | 667c275 | 2015-11-26 15:42:41 +0000 | [diff] [blame] | 169 | alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 170 | prot, pgtable_alloc); |
Catalin Marinas | a55f992 | 2014-02-04 16:01:31 +0000 | [diff] [blame] | 171 | } |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 172 | phys += next - addr; |
| 173 | } while (pmd++, addr = next, addr != end); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 174 | |
| 175 | pmd_clear_fixmap(); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 176 | } |
| 177 | |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 178 | static inline bool use_1G_block(unsigned long addr, unsigned long next, |
| 179 | unsigned long phys) |
| 180 | { |
| 181 | if (PAGE_SHIFT != 12) |
| 182 | return false; |
| 183 | |
| 184 | if (((addr | next | phys) & ~PUD_MASK) != 0) |
| 185 | return false; |
| 186 | |
| 187 | return true; |
| 188 | } |
| 189 | |
Mark Rutland | 11509a3 | 2016-01-25 11:45:10 +0000 | [diff] [blame] | 190 | static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 191 | phys_addr_t phys, pgprot_t prot, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 192 | phys_addr_t (*pgtable_alloc)(void), |
| 193 | bool allow_block_mappings) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 194 | { |
Jungseok Lee | c79b954 | 2014-05-12 18:40:51 +0900 | [diff] [blame] | 195 | pud_t *pud; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 196 | unsigned long next; |
| 197 | |
Jungseok Lee | c79b954 | 2014-05-12 18:40:51 +0900 | [diff] [blame] | 198 | if (pgd_none(*pgd)) { |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 199 | phys_addr_t pud_phys; |
| 200 | BUG_ON(!pgtable_alloc); |
| 201 | pud_phys = pgtable_alloc(); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 202 | __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); |
Jungseok Lee | c79b954 | 2014-05-12 18:40:51 +0900 | [diff] [blame] | 203 | } |
| 204 | BUG_ON(pgd_bad(*pgd)); |
| 205 | |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 206 | pud = pud_set_fixmap_offset(pgd, addr); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 207 | do { |
| 208 | next = pud_addr_end(addr, end); |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 209 | |
| 210 | /* |
| 211 | * For 4K granule only, attempt to put down a 1GB block |
| 212 | */ |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 213 | if (use_1G_block(addr, next, phys) && allow_block_mappings) { |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 214 | pud_t old_pud = *pud; |
Mark Rutland | c661cb1 | 2016-03-22 10:11:45 +0000 | [diff] [blame] | 215 | pud_set_huge(pud, phys, prot); |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 216 | |
| 217 | /* |
| 218 | * If we have an old value for a pud, it will |
| 219 | * be pointing to a pmd table that we no longer |
| 220 | * need (from swapper_pg_dir). |
| 221 | * |
| 222 | * Look up the old pmd table and free it. |
| 223 | */ |
| 224 | if (!pud_none(old_pud)) { |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 225 | flush_tlb_all(); |
zhichang.yuan | 523d6e9 | 2014-12-09 07:26:47 +0000 | [diff] [blame] | 226 | if (pud_table(old_pud)) { |
Mark Rutland | 316b39db | 2016-01-25 11:45:05 +0000 | [diff] [blame] | 227 | phys_addr_t table = pud_page_paddr(old_pud); |
Catalin Marinas | 4108935 | 2015-01-29 17:33:35 +0000 | [diff] [blame] | 228 | if (!WARN_ON_ONCE(slab_is_available())) |
| 229 | memblock_free(table, PAGE_SIZE); |
zhichang.yuan | 523d6e9 | 2014-12-09 07:26:47 +0000 | [diff] [blame] | 230 | } |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 231 | } |
| 232 | } else { |
Mark Rutland | 11509a3 | 2016-01-25 11:45:10 +0000 | [diff] [blame] | 233 | alloc_init_pmd(pud, addr, next, phys, prot, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 234 | pgtable_alloc, allow_block_mappings); |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 235 | } |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 236 | phys += next - addr; |
| 237 | } while (pud++, addr = next, addr != end); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 238 | |
| 239 | pud_clear_fixmap(); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 240 | } |
| 241 | |
Ard Biesheuvel | 40f87d3 | 2016-06-29 14:51:30 +0200 | [diff] [blame] | 242 | static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, |
| 243 | unsigned long virt, phys_addr_t size, |
| 244 | pgprot_t prot, |
| 245 | phys_addr_t (*pgtable_alloc)(void), |
| 246 | bool allow_block_mappings) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 247 | { |
| 248 | unsigned long addr, length, end, next; |
Ard Biesheuvel | 40f87d3 | 2016-06-29 14:51:30 +0200 | [diff] [blame] | 249 | pgd_t *pgd = pgd_offset_raw(pgdir, virt); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 250 | |
Mark Rutland | cc5d2b3 | 2015-11-23 13:26:19 +0000 | [diff] [blame] | 251 | /* |
| 252 | * If the virtual and physical address don't have the same offset |
| 253 | * within a page, we cannot map the region as the caller expects. |
| 254 | */ |
| 255 | if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) |
| 256 | return; |
| 257 | |
Mark Rutland | 9c4e08a | 2015-11-23 13:26:20 +0000 | [diff] [blame] | 258 | phys &= PAGE_MASK; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 259 | addr = virt & PAGE_MASK; |
| 260 | length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); |
| 261 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 262 | end = addr + length; |
| 263 | do { |
| 264 | next = pgd_addr_end(addr, end); |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 265 | alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, |
| 266 | allow_block_mappings); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 267 | phys += next - addr; |
| 268 | } while (pgd++, addr = next, addr != end); |
| 269 | } |
| 270 | |
Ard Biesheuvel | 1378dc3 | 2016-07-22 19:32:25 +0200 | [diff] [blame] | 271 | static phys_addr_t pgd_pgtable_alloc(void) |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 272 | { |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 273 | void *ptr = (void *)__get_free_page(PGALLOC_GFP); |
Ard Biesheuvel | 1378dc3 | 2016-07-22 19:32:25 +0200 | [diff] [blame] | 274 | if (!ptr || !pgtable_page_ctor(virt_to_page(ptr))) |
| 275 | BUG(); |
Mark Rutland | 21ab99c | 2016-01-25 11:44:56 +0000 | [diff] [blame] | 276 | |
| 277 | /* Ensure the zeroed page is visible to the page table walker */ |
| 278 | dsb(ishst); |
Mark Rutland | f471044 | 2016-01-25 11:45:08 +0000 | [diff] [blame] | 279 | return __pa(ptr); |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 280 | } |
| 281 | |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 282 | /* |
| 283 | * This function can only be used to modify existing table entries, |
| 284 | * without allocating new levels of table. Note that this permits the |
| 285 | * creation of new section or page entries. |
| 286 | */ |
| 287 | static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 288 | phys_addr_t size, pgprot_t prot) |
Mark Salter | d7ecbdd | 2014-03-12 12:28:06 -0400 | [diff] [blame] | 289 | { |
| 290 | if (virt < VMALLOC_START) { |
| 291 | pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", |
| 292 | &phys, virt); |
| 293 | return; |
| 294 | } |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 295 | __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true); |
Mark Salter | d7ecbdd | 2014-03-12 12:28:06 -0400 | [diff] [blame] | 296 | } |
| 297 | |
Ard Biesheuvel | 8ce837c | 2014-10-20 15:42:07 +0200 | [diff] [blame] | 298 | void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, |
| 299 | unsigned long virt, phys_addr_t size, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 300 | pgprot_t prot, bool allow_block_mappings) |
Ard Biesheuvel | 8ce837c | 2014-10-20 15:42:07 +0200 | [diff] [blame] | 301 | { |
Ard Biesheuvel | 1378dc3 | 2016-07-22 19:32:25 +0200 | [diff] [blame] | 302 | BUG_ON(mm == &init_mm); |
| 303 | |
Mark Rutland | 11509a3 | 2016-01-25 11:45:10 +0000 | [diff] [blame] | 304 | __create_pgd_mapping(mm->pgd, phys, virt, size, prot, |
Ard Biesheuvel | 1378dc3 | 2016-07-22 19:32:25 +0200 | [diff] [blame] | 305 | pgd_pgtable_alloc, allow_block_mappings); |
Mark Salter | d7ecbdd | 2014-03-12 12:28:06 -0400 | [diff] [blame] | 306 | } |
| 307 | |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 308 | static void create_mapping_late(phys_addr_t phys, unsigned long virt, |
| 309 | phys_addr_t size, pgprot_t prot) |
| 310 | { |
| 311 | if (virt < VMALLOC_START) { |
| 312 | pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", |
| 313 | &phys, virt); |
| 314 | return; |
| 315 | } |
| 316 | |
Mark Rutland | 11509a3 | 2016-01-25 11:45:10 +0000 | [diff] [blame] | 317 | __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, |
Ard Biesheuvel | 258a160 | 2016-07-22 19:32:24 +0200 | [diff] [blame] | 318 | NULL, !debug_pagealloc_enabled()); |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 319 | } |
| 320 | |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 321 | static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 322 | { |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 323 | unsigned long kernel_start = __pa_symbol(_text); |
| 324 | unsigned long kernel_end = __pa_symbol(__init_begin); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 325 | |
| 326 | /* |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 327 | * Take care not to create a writable alias for the |
| 328 | * read-only text and rodata sections of the kernel image. |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 329 | */ |
| 330 | |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 331 | /* No overlap with the kernel text/rodata */ |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 332 | if (end < kernel_start || start >= kernel_end) { |
| 333 | __create_pgd_mapping(pgd, start, __phys_to_virt(start), |
| 334 | end - start, PAGE_KERNEL, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 335 | early_pgtable_alloc, |
| 336 | !debug_pagealloc_enabled()); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 337 | return; |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 338 | } |
| 339 | |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 340 | /* |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 341 | * This block overlaps the kernel text/rodata mappings. |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 342 | * Map the portion(s) which don't overlap. |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 343 | */ |
| 344 | if (start < kernel_start) |
| 345 | __create_pgd_mapping(pgd, start, |
| 346 | __phys_to_virt(start), |
| 347 | kernel_start - start, PAGE_KERNEL, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 348 | early_pgtable_alloc, |
| 349 | !debug_pagealloc_enabled()); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 350 | if (kernel_end < end) |
| 351 | __create_pgd_mapping(pgd, kernel_end, |
| 352 | __phys_to_virt(kernel_end), |
| 353 | end - kernel_end, PAGE_KERNEL, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 354 | early_pgtable_alloc, |
| 355 | !debug_pagealloc_enabled()); |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 356 | |
| 357 | /* |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 358 | * Map the linear alias of the [_text, __init_begin) interval as |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 359 | * read-only/non-executable. This makes the contents of the |
| 360 | * region accessible to subsystems such as hibernate, but |
| 361 | * protects it from inadvertent modification or execution. |
| 362 | */ |
| 363 | __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), |
| 364 | kernel_end - kernel_start, PAGE_KERNEL_RO, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 365 | early_pgtable_alloc, !debug_pagealloc_enabled()); |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 366 | } |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 367 | |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 368 | static void __init map_mem(pgd_t *pgd) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 369 | { |
| 370 | struct memblock_region *reg; |
Steve Capper | f6bc87c | 2013-04-30 11:00:33 +0100 | [diff] [blame] | 371 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 372 | /* map all the memory banks */ |
| 373 | for_each_memblock(memory, reg) { |
| 374 | phys_addr_t start = reg->base; |
| 375 | phys_addr_t end = start + reg->size; |
| 376 | |
| 377 | if (start >= end) |
| 378 | break; |
Ard Biesheuvel | 68709f4 | 2015-11-30 13:28:16 +0100 | [diff] [blame] | 379 | if (memblock_is_nomap(reg)) |
| 380 | continue; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 381 | |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 382 | __map_memblock(pgd, start, end); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 383 | } |
| 384 | } |
| 385 | |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 386 | void mark_rodata_ro(void) |
| 387 | { |
Jeremy Linton | 2f39b5f | 2016-02-19 11:50:32 -0600 | [diff] [blame] | 388 | unsigned long section_size; |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 389 | |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 390 | section_size = (unsigned long)_etext - (unsigned long)_text; |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 391 | create_mapping_late(__pa_symbol(_text), (unsigned long)_text, |
Jeremy Linton | 2f39b5f | 2016-02-19 11:50:32 -0600 | [diff] [blame] | 392 | section_size, PAGE_KERNEL_ROX); |
| 393 | /* |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 394 | * mark .rodata as read only. Use __init_begin rather than __end_rodata |
| 395 | * to cover NOTES and EXCEPTION_TABLE. |
Jeremy Linton | 2f39b5f | 2016-02-19 11:50:32 -0600 | [diff] [blame] | 396 | */ |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 397 | section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 398 | create_mapping_late(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, |
Jeremy Linton | 2f39b5f | 2016-02-19 11:50:32 -0600 | [diff] [blame] | 399 | section_size, PAGE_KERNEL_RO); |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 400 | } |
Laura Abbott | da14170 | 2015-01-21 17:36:06 -0800 | [diff] [blame] | 401 | |
Ard Biesheuvel | 2c09ec0 | 2016-03-30 17:43:06 +0200 | [diff] [blame] | 402 | static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, |
| 403 | pgprot_t prot, struct vm_struct *vma) |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 404 | { |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 405 | phys_addr_t pa_start = __pa_symbol(va_start); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 406 | unsigned long size = va_end - va_start; |
| 407 | |
| 408 | BUG_ON(!PAGE_ALIGNED(pa_start)); |
| 409 | BUG_ON(!PAGE_ALIGNED(size)); |
| 410 | |
| 411 | __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, |
Ard Biesheuvel | 53e1b32 | 2016-06-29 14:51:26 +0200 | [diff] [blame] | 412 | early_pgtable_alloc, !debug_pagealloc_enabled()); |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 413 | |
| 414 | vma->addr = va_start; |
| 415 | vma->phys_addr = pa_start; |
| 416 | vma->size = size; |
| 417 | vma->flags = VM_MAP; |
| 418 | vma->caller = __builtin_return_address(0); |
| 419 | |
| 420 | vm_area_add_early(vma); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 421 | } |
| 422 | |
Will Deacon | 5e54c4b | 2017-11-14 14:14:17 +0000 | [diff] [blame] | 423 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
| 424 | static int __init map_entry_trampoline(void) |
| 425 | { |
| 426 | extern char __entry_tramp_text_start[]; |
| 427 | |
Will Deacon | 1809afd | 2018-04-03 12:09:05 +0100 | [diff] [blame] | 428 | pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; |
Will Deacon | 5e54c4b | 2017-11-14 14:14:17 +0000 | [diff] [blame] | 429 | phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); |
| 430 | |
| 431 | /* The trampoline is always mapped and can therefore be global */ |
| 432 | pgprot_val(prot) &= ~PTE_NG; |
| 433 | |
| 434 | /* Map only the text into the trampoline page table */ |
| 435 | memset(tramp_pg_dir, 0, PGD_SIZE); |
| 436 | __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, |
| 437 | prot, pgd_pgtable_alloc, 0); |
| 438 | |
Will Deacon | 06fe41f | 2017-12-06 11:24:02 +0000 | [diff] [blame] | 439 | /* Map both the text and data into the kernel page table */ |
Will Deacon | 5e54c4b | 2017-11-14 14:14:17 +0000 | [diff] [blame] | 440 | __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); |
Will Deacon | 06fe41f | 2017-12-06 11:24:02 +0000 | [diff] [blame] | 441 | if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { |
| 442 | extern char __entry_tramp_data_start[]; |
| 443 | |
| 444 | __set_fixmap(FIX_ENTRY_TRAMP_DATA, |
| 445 | __pa_symbol(__entry_tramp_data_start), |
| 446 | PAGE_KERNEL_RO); |
| 447 | } |
| 448 | |
Will Deacon | 5e54c4b | 2017-11-14 14:14:17 +0000 | [diff] [blame] | 449 | return 0; |
| 450 | } |
| 451 | core_initcall(map_entry_trampoline); |
| 452 | #endif |
| 453 | |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 454 | /* |
| 455 | * Create fine-grained mappings for the kernel. |
| 456 | */ |
| 457 | static void __init map_kernel(pgd_t *pgd) |
| 458 | { |
Jeremy Linton | 2f39b5f | 2016-02-19 11:50:32 -0600 | [diff] [blame] | 459 | static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 460 | |
Ard Biesheuvel | 9fdc14c5 | 2016-06-23 15:53:17 +0200 | [diff] [blame] | 461 | map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); |
| 462 | map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); |
Ard Biesheuvel | 2c09ec0 | 2016-03-30 17:43:06 +0200 | [diff] [blame] | 463 | map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, |
| 464 | &vmlinux_init); |
| 465 | map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 466 | |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 467 | if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { |
| 468 | /* |
| 469 | * The fixmap falls in a separate pgd to the kernel, and doesn't |
| 470 | * live in the carveout for the swapper_pg_dir. We can simply |
| 471 | * re-use the existing dir for the fixmap. |
| 472 | */ |
| 473 | set_pgd(pgd_offset_raw(pgd, FIXADDR_START), |
| 474 | *pgd_offset_k(FIXADDR_START)); |
| 475 | } else if (CONFIG_PGTABLE_LEVELS > 3) { |
| 476 | /* |
| 477 | * The fixmap shares its top level pgd entry with the kernel |
| 478 | * mapping. This can really only occur when we are running |
| 479 | * with 16k/4 levels, so we can simply reuse the pud level |
| 480 | * entry instead. |
| 481 | */ |
| 482 | BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); |
| 483 | set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START), |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 484 | __pud(__pa_symbol(bm_pmd) | PUD_TYPE_TABLE)); |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 485 | pud_clear_fixmap(); |
| 486 | } else { |
| 487 | BUG(); |
| 488 | } |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 489 | |
| 490 | kasan_copy_shadow(pgd); |
| 491 | } |
| 492 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 493 | /* |
| 494 | * paging_init() sets up the page tables, initialises the zone memory |
| 495 | * maps and sets up the zero page. |
| 496 | */ |
| 497 | void __init paging_init(void) |
| 498 | { |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 499 | phys_addr_t pgd_phys = early_pgtable_alloc(); |
| 500 | pgd_t *pgd = pgd_set_fixmap(pgd_phys); |
| 501 | |
| 502 | map_kernel(pgd); |
| 503 | map_mem(pgd); |
| 504 | |
| 505 | /* |
| 506 | * We want to reuse the original swapper_pg_dir so we don't have to |
| 507 | * communicate the new address to non-coherent secondaries in |
| 508 | * secondary_entry, and so cpu_switch_mm can generate the address with |
| 509 | * adrp+add rather than a load from some global variable. |
| 510 | * |
| 511 | * To do this we need to go via a temporary pgd. |
| 512 | */ |
| 513 | cpu_replace_ttbr1(__va(pgd_phys)); |
Arnd Bergmann | fd2e662 | 2018-02-19 11:13:26 +0100 | [diff] [blame] | 514 | memcpy(swapper_pg_dir, pgd, PGD_SIZE); |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 515 | cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 516 | |
| 517 | pgd_clear_fixmap(); |
| 518 | memblock_free(pgd_phys, PAGE_SIZE); |
| 519 | |
| 520 | /* |
| 521 | * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd |
| 522 | * allocated with it. |
| 523 | */ |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 524 | memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE, |
Mark Rutland | 068a17a | 2016-01-25 11:45:12 +0000 | [diff] [blame] | 525 | SWAPPER_DIR_SIZE - PAGE_SIZE); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 526 | } |
| 527 | |
| 528 | /* |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 529 | * Check whether a kernel address is valid (derived from arch/x86/). |
| 530 | */ |
| 531 | int kern_addr_valid(unsigned long addr) |
| 532 | { |
| 533 | pgd_t *pgd; |
| 534 | pud_t *pud; |
| 535 | pmd_t *pmd; |
| 536 | pte_t *pte; |
| 537 | |
| 538 | if ((((long)addr) >> VA_BITS) != -1UL) |
| 539 | return 0; |
| 540 | |
| 541 | pgd = pgd_offset_k(addr); |
| 542 | if (pgd_none(*pgd)) |
| 543 | return 0; |
| 544 | |
| 545 | pud = pud_offset(pgd, addr); |
| 546 | if (pud_none(*pud)) |
| 547 | return 0; |
| 548 | |
Steve Capper | 206a2a7 | 2014-05-06 14:02:27 +0100 | [diff] [blame] | 549 | if (pud_sect(*pud)) |
| 550 | return pfn_valid(pud_pfn(*pud)); |
| 551 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 552 | pmd = pmd_offset(pud, addr); |
| 553 | if (pmd_none(*pmd)) |
| 554 | return 0; |
| 555 | |
Dave Anderson | da6e4cb | 2014-04-15 18:53:24 +0100 | [diff] [blame] | 556 | if (pmd_sect(*pmd)) |
| 557 | return pfn_valid(pmd_pfn(*pmd)); |
| 558 | |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 559 | pte = pte_offset_kernel(pmd, addr); |
| 560 | if (pte_none(*pte)) |
| 561 | return 0; |
| 562 | |
| 563 | return pfn_valid(pte_pfn(*pte)); |
| 564 | } |
| 565 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
Suzuki K. Poulose | b433dce | 2015-10-19 14:19:28 +0100 | [diff] [blame] | 566 | #if !ARM64_SWAPPER_USES_SECTION_MAPS |
Johannes Weiner | 0aad818 | 2013-04-29 15:07:50 -0700 | [diff] [blame] | 567 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 568 | { |
Johannes Weiner | 0aad818 | 2013-04-29 15:07:50 -0700 | [diff] [blame] | 569 | return vmemmap_populate_basepages(start, end, node); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 570 | } |
Suzuki K. Poulose | b433dce | 2015-10-19 14:19:28 +0100 | [diff] [blame] | 571 | #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ |
Johannes Weiner | 0aad818 | 2013-04-29 15:07:50 -0700 | [diff] [blame] | 572 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 573 | { |
Johannes Weiner | 0aad818 | 2013-04-29 15:07:50 -0700 | [diff] [blame] | 574 | unsigned long addr = start; |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 575 | unsigned long next; |
| 576 | pgd_t *pgd; |
| 577 | pud_t *pud; |
| 578 | pmd_t *pmd; |
| 579 | |
| 580 | do { |
| 581 | next = pmd_addr_end(addr, end); |
| 582 | |
| 583 | pgd = vmemmap_pgd_populate(addr, node); |
| 584 | if (!pgd) |
| 585 | return -ENOMEM; |
| 586 | |
| 587 | pud = vmemmap_pud_populate(pgd, addr, node); |
| 588 | if (!pud) |
| 589 | return -ENOMEM; |
| 590 | |
| 591 | pmd = pmd_offset(pud, addr); |
| 592 | if (pmd_none(*pmd)) { |
| 593 | void *p = NULL; |
| 594 | |
| 595 | p = vmemmap_alloc_block_buf(PMD_SIZE, node); |
| 596 | if (!p) |
| 597 | return -ENOMEM; |
| 598 | |
Catalin Marinas | a501e32 | 2014-04-03 15:57:15 +0100 | [diff] [blame] | 599 | set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 600 | } else |
| 601 | vmemmap_verify((pte_t *)pmd, node, addr, next); |
| 602 | } while (addr = next, addr != end); |
| 603 | |
| 604 | return 0; |
| 605 | } |
| 606 | #endif /* CONFIG_ARM64_64K_PAGES */ |
Johannes Weiner | 0aad818 | 2013-04-29 15:07:50 -0700 | [diff] [blame] | 607 | void vmemmap_free(unsigned long start, unsigned long end) |
Tang Chen | 0197518 | 2013-02-22 16:33:08 -0800 | [diff] [blame] | 608 | { |
| 609 | } |
Catalin Marinas | c1cc155 | 2012-03-05 11:49:27 +0000 | [diff] [blame] | 610 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 611 | |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 612 | static inline pud_t * fixmap_pud(unsigned long addr) |
| 613 | { |
| 614 | pgd_t *pgd = pgd_offset_k(addr); |
| 615 | |
| 616 | BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); |
| 617 | |
Ard Biesheuvel | 157962f | 2016-02-16 13:52:38 +0100 | [diff] [blame] | 618 | return pud_offset_kimg(pgd, addr); |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 619 | } |
| 620 | |
| 621 | static inline pmd_t * fixmap_pmd(unsigned long addr) |
| 622 | { |
| 623 | pud_t *pud = fixmap_pud(addr); |
| 624 | |
| 625 | BUG_ON(pud_none(*pud) || pud_bad(*pud)); |
| 626 | |
Ard Biesheuvel | 157962f | 2016-02-16 13:52:38 +0100 | [diff] [blame] | 627 | return pmd_offset_kimg(pud, addr); |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 628 | } |
| 629 | |
| 630 | static inline pte_t * fixmap_pte(unsigned long addr) |
| 631 | { |
Ard Biesheuvel | 157962f | 2016-02-16 13:52:38 +0100 | [diff] [blame] | 632 | return &bm_pte[pte_index(addr)]; |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 633 | } |
| 634 | |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 635 | /* |
| 636 | * The p*d_populate functions call virt_to_phys implicitly so they can't be used |
| 637 | * directly on kernel symbols (bm_p*d). This function is called too early to use |
| 638 | * lm_alias so __p*d_populate functions must be used to populate with the |
| 639 | * physical address from __pa_symbol. |
| 640 | */ |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 641 | void __init early_fixmap_init(void) |
| 642 | { |
| 643 | pgd_t *pgd; |
| 644 | pud_t *pud; |
| 645 | pmd_t *pmd; |
| 646 | unsigned long addr = FIXADDR_START; |
| 647 | |
| 648 | pgd = pgd_offset_k(addr); |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 649 | if (CONFIG_PGTABLE_LEVELS > 3 && |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 650 | !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) { |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 651 | /* |
| 652 | * We only end up here if the kernel mapping and the fixmap |
| 653 | * share the top level pgd entry, which should only happen on |
| 654 | * 16k/4 levels configurations. |
| 655 | */ |
| 656 | BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); |
| 657 | pud = pud_offset_kimg(pgd, addr); |
| 658 | } else { |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 659 | if (pgd_none(*pgd)) |
| 660 | __pgd_populate(pgd, __pa_symbol(bm_pud), PUD_TYPE_TABLE); |
Ard Biesheuvel | f904077 | 2016-02-16 13:52:40 +0100 | [diff] [blame] | 661 | pud = fixmap_pud(addr); |
| 662 | } |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 663 | if (pud_none(*pud)) |
| 664 | __pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE); |
Ard Biesheuvel | 157962f | 2016-02-16 13:52:38 +0100 | [diff] [blame] | 665 | pmd = fixmap_pmd(addr); |
Laura Abbott | f8fee94e | 2017-01-10 13:35:49 -0800 | [diff] [blame] | 666 | __pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE); |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 667 | |
| 668 | /* |
| 669 | * The boot-ioremap range spans multiple pmds, for which |
Ard Biesheuvel | 157962f | 2016-02-16 13:52:38 +0100 | [diff] [blame] | 670 | * we are not prepared: |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 671 | */ |
| 672 | BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) |
| 673 | != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); |
| 674 | |
| 675 | if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) |
| 676 | || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { |
| 677 | WARN_ON(1); |
| 678 | pr_warn("pmd %p != %p, %p\n", |
| 679 | pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), |
| 680 | fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); |
| 681 | pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", |
| 682 | fix_to_virt(FIX_BTMAP_BEGIN)); |
| 683 | pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", |
| 684 | fix_to_virt(FIX_BTMAP_END)); |
| 685 | |
| 686 | pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); |
| 687 | pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); |
| 688 | } |
| 689 | } |
| 690 | |
| 691 | void __set_fixmap(enum fixed_addresses idx, |
| 692 | phys_addr_t phys, pgprot_t flags) |
| 693 | { |
| 694 | unsigned long addr = __fix_to_virt(idx); |
| 695 | pte_t *pte; |
| 696 | |
Mark Rutland | b63dbef | 2015-03-04 13:27:35 +0000 | [diff] [blame] | 697 | BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); |
Laura Abbott | af86e59 | 2014-11-21 21:50:42 +0000 | [diff] [blame] | 698 | |
| 699 | pte = fixmap_pte(addr); |
| 700 | |
| 701 | if (pgprot_val(flags)) { |
| 702 | set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); |
| 703 | } else { |
| 704 | pte_clear(&init_mm, addr, pte); |
| 705 | flush_tlb_kernel_range(addr, addr+PAGE_SIZE); |
| 706 | } |
| 707 | } |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 708 | |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 709 | void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 710 | { |
| 711 | const u64 dt_virt_base = __fix_to_virt(FIX_FDT); |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 712 | int offset; |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 713 | void *dt_virt; |
| 714 | |
| 715 | /* |
| 716 | * Check whether the physical FDT address is set and meets the minimum |
| 717 | * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be |
Ard Biesheuvel | 04a8481 | 2016-08-01 13:29:31 +0200 | [diff] [blame] | 718 | * at least 8 bytes so that we can always access the magic and size |
| 719 | * fields of the FDT header after mapping the first chunk, double check |
| 720 | * here if that is indeed the case. |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 721 | */ |
| 722 | BUILD_BUG_ON(MIN_FDT_ALIGN < 8); |
| 723 | if (!dt_phys || dt_phys % MIN_FDT_ALIGN) |
| 724 | return NULL; |
| 725 | |
| 726 | /* |
| 727 | * Make sure that the FDT region can be mapped without the need to |
| 728 | * allocate additional translation table pages, so that it is safe |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 729 | * to call create_mapping_noalloc() this early. |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 730 | * |
| 731 | * On 64k pages, the FDT will be mapped using PTEs, so we need to |
| 732 | * be in the same PMD as the rest of the fixmap. |
| 733 | * On 4k pages, we'll use section mappings for the FDT so we only |
| 734 | * have to be in the same PUD. |
| 735 | */ |
| 736 | BUILD_BUG_ON(dt_virt_base % SZ_2M); |
| 737 | |
Suzuki K. Poulose | b433dce | 2015-10-19 14:19:28 +0100 | [diff] [blame] | 738 | BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != |
| 739 | __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 740 | |
Suzuki K. Poulose | b433dce | 2015-10-19 14:19:28 +0100 | [diff] [blame] | 741 | offset = dt_phys % SWAPPER_BLOCK_SIZE; |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 742 | dt_virt = (void *)dt_virt_base + offset; |
| 743 | |
| 744 | /* map the first chunk so we can read the size from the header */ |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 745 | create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), |
| 746 | dt_virt_base, SWAPPER_BLOCK_SIZE, prot); |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 747 | |
Ard Biesheuvel | 04a8481 | 2016-08-01 13:29:31 +0200 | [diff] [blame] | 748 | if (fdt_magic(dt_virt) != FDT_MAGIC) |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 749 | return NULL; |
| 750 | |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 751 | *size = fdt_totalsize(dt_virt); |
| 752 | if (*size > MAX_FDT_SIZE) |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 753 | return NULL; |
| 754 | |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 755 | if (offset + *size > SWAPPER_BLOCK_SIZE) |
Laura Abbott | 132233a | 2016-02-05 16:24:46 -0800 | [diff] [blame] | 756 | create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, |
Ard Biesheuvel | f80fb3a | 2016-01-26 14:12:01 +0100 | [diff] [blame] | 757 | round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); |
| 758 | |
| 759 | return dt_virt; |
| 760 | } |
| 761 | |
| 762 | void *__init fixmap_remap_fdt(phys_addr_t dt_phys) |
| 763 | { |
| 764 | void *dt_virt; |
| 765 | int size; |
| 766 | |
| 767 | dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO); |
| 768 | if (!dt_virt) |
| 769 | return NULL; |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 770 | |
| 771 | memblock_reserve(dt_phys, size); |
Ard Biesheuvel | 61bd93c | 2015-06-01 13:40:32 +0200 | [diff] [blame] | 772 | return dt_virt; |
| 773 | } |
Ard Biesheuvel | 324420b | 2016-02-16 13:52:35 +0100 | [diff] [blame] | 774 | |
| 775 | int __init arch_ioremap_pud_supported(void) |
| 776 | { |
| 777 | /* only 4k granule supports level 1 block mappings */ |
| 778 | return IS_ENABLED(CONFIG_ARM64_4K_PAGES); |
| 779 | } |
| 780 | |
| 781 | int __init arch_ioremap_pmd_supported(void) |
| 782 | { |
| 783 | return 1; |
| 784 | } |
| 785 | |
| 786 | int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) |
| 787 | { |
| 788 | BUG_ON(phys & ~PUD_MASK); |
| 789 | set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); |
| 790 | return 1; |
| 791 | } |
| 792 | |
| 793 | int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) |
| 794 | { |
| 795 | BUG_ON(phys & ~PMD_MASK); |
| 796 | set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); |
| 797 | return 1; |
| 798 | } |
| 799 | |
| 800 | int pud_clear_huge(pud_t *pud) |
| 801 | { |
| 802 | if (!pud_sect(*pud)) |
| 803 | return 0; |
| 804 | pud_clear(pud); |
| 805 | return 1; |
| 806 | } |
| 807 | |
| 808 | int pmd_clear_huge(pmd_t *pmd) |
| 809 | { |
| 810 | if (!pmd_sect(*pmd)) |
| 811 | return 0; |
| 812 | pmd_clear(pmd); |
| 813 | return 1; |
| 814 | } |
Toshi Kani | 9c7f7bd | 2018-03-22 16:17:20 -0700 | [diff] [blame] | 815 | |
Chintan Pandya | 6e6b637 | 2018-06-27 08:13:47 -0600 | [diff] [blame] | 816 | int pud_free_pmd_page(pud_t *pud, unsigned long addr) |
Toshi Kani | 9c7f7bd | 2018-03-22 16:17:20 -0700 | [diff] [blame] | 817 | { |
| 818 | return pud_none(*pud); |
| 819 | } |
| 820 | |
Chintan Pandya | 6e6b637 | 2018-06-27 08:13:47 -0600 | [diff] [blame] | 821 | int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) |
Toshi Kani | 9c7f7bd | 2018-03-22 16:17:20 -0700 | [diff] [blame] | 822 | { |
| 823 | return pmd_none(*pmd); |
| 824 | } |