blob: e3f563c81c4802c1f9c519fe2bfc7317f8ab5467 [file] [log] [blame]
Catalin Marinasc1cc1552012-03-05 11:49:27 +00001/*
2 * Based on arch/arm/mm/mmu.c
3 *
4 * Copyright (C) 1995-2005 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/export.h>
21#include <linux/kernel.h>
22#include <linux/errno.h>
23#include <linux/init.h>
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +020024#include <linux/libfdt.h>
Catalin Marinasc1cc1552012-03-05 11:49:27 +000025#include <linux/mman.h>
26#include <linux/nodemask.h>
27#include <linux/memblock.h>
28#include <linux/fs.h>
Catalin Marinas2475ff92012-10-23 14:55:08 +010029#include <linux/io.h>
Catalin Marinas41089352015-01-29 17:33:35 +000030#include <linux/slab.h>
Laura Abbottda141702015-01-21 17:36:06 -080031#include <linux/stop_machine.h>
Catalin Marinasc1cc1552012-03-05 11:49:27 +000032
33#include <asm/cputype.h>
Laura Abbottaf86e592014-11-21 21:50:42 +000034#include <asm/fixmap.h>
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +010035#include <asm/kernel-pgtable.h>
Catalin Marinasc1cc1552012-03-05 11:49:27 +000036#include <asm/sections.h>
37#include <asm/setup.h>
38#include <asm/sizes.h>
39#include <asm/tlb.h>
Jungseok Leec79b9542014-05-12 18:40:51 +090040#include <asm/memblock.h>
Catalin Marinasc1cc1552012-03-05 11:49:27 +000041#include <asm/mmu_context.h>
42
43#include "mm.h"
44
Ard Biesheuveldd006da2015-03-19 16:42:27 +000045u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
46
Catalin Marinasc1cc1552012-03-05 11:49:27 +000047/*
48 * Empty_zero_page is a special page that is used for zero-initialized data
49 * and COW.
50 */
51struct page *empty_zero_page;
52EXPORT_SYMBOL(empty_zero_page);
53
Catalin Marinasc1cc1552012-03-05 11:49:27 +000054pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
55 unsigned long size, pgprot_t vma_prot)
56{
57 if (!pfn_valid(pfn))
58 return pgprot_noncached(vma_prot);
59 else if (file->f_flags & O_SYNC)
60 return pgprot_writecombine(vma_prot);
61 return vma_prot;
62}
63EXPORT_SYMBOL(phys_mem_access_prot);
64
65static void __init *early_alloc(unsigned long sz)
66{
67 void *ptr = __va(memblock_alloc(sz, sz));
Laura Abbottda141702015-01-21 17:36:06 -080068 BUG_ON(!ptr);
Catalin Marinasc1cc1552012-03-05 11:49:27 +000069 memset(ptr, 0, sz);
70 return ptr;
71}
72
Laura Abbottda141702015-01-21 17:36:06 -080073/*
74 * remap a PMD into pages
75 */
76static void split_pmd(pmd_t *pmd, pte_t *pte)
77{
78 unsigned long pfn = pmd_pfn(*pmd);
79 int i = 0;
80
81 do {
82 /*
83 * Need to have the least restrictive permissions available
Jeremy Linton348a65c2015-10-07 12:00:25 -050084 * permissions will be fixed up later. Default the new page
85 * range as contiguous ptes.
Laura Abbottda141702015-01-21 17:36:06 -080086 */
Jeremy Linton348a65c2015-10-07 12:00:25 -050087 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC_CONT));
Laura Abbottda141702015-01-21 17:36:06 -080088 pfn++;
89 } while (pte++, i++, i < PTRS_PER_PTE);
90}
91
Jeremy Linton348a65c2015-10-07 12:00:25 -050092/*
93 * Given a PTE with the CONT bit set, determine where the CONT range
94 * starts, and clear the entire range of PTE CONT bits.
95 */
96static void clear_cont_pte_range(pte_t *pte, unsigned long addr)
97{
98 int i;
99
100 pte -= CONT_RANGE_OFFSET(addr);
101 for (i = 0; i < CONT_PTES; i++) {
102 set_pte(pte, pte_mknoncont(*pte));
103 pte++;
104 }
105 flush_tlb_all();
106}
107
108/*
109 * Given a range of PTEs set the pfn and provided page protection flags
110 */
111static void __populate_init_pte(pte_t *pte, unsigned long addr,
112 unsigned long end, phys_addr_t phys,
113 pgprot_t prot)
114{
115 unsigned long pfn = __phys_to_pfn(phys);
116
117 do {
118 /* clear all the bits except the pfn, then apply the prot */
119 set_pte(pte, pfn_pte(pfn, prot));
120 pte++;
121 pfn++;
122 addr += PAGE_SIZE;
123 } while (addr != end);
124}
125
Laura Abbottda141702015-01-21 17:36:06 -0800126static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
Jeremy Linton348a65c2015-10-07 12:00:25 -0500127 unsigned long end, phys_addr_t phys,
Laura Abbottda141702015-01-21 17:36:06 -0800128 pgprot_t prot,
129 void *(*alloc)(unsigned long size))
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000130{
131 pte_t *pte;
Jeremy Linton348a65c2015-10-07 12:00:25 -0500132 unsigned long next;
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000133
Mark Rutlanda1c76572015-01-27 16:36:30 +0000134 if (pmd_none(*pmd) || pmd_sect(*pmd)) {
Laura Abbottda141702015-01-21 17:36:06 -0800135 pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
136 if (pmd_sect(*pmd))
137 split_pmd(pmd, pte);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000138 __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
Laura Abbottda141702015-01-21 17:36:06 -0800139 flush_tlb_all();
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000140 }
Mark Rutlanda1c76572015-01-27 16:36:30 +0000141 BUG_ON(pmd_bad(*pmd));
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000142
143 pte = pte_offset_kernel(pmd, addr);
144 do {
Jeremy Linton348a65c2015-10-07 12:00:25 -0500145 next = min(end, (addr + CONT_SIZE) & CONT_MASK);
146 if (((addr | next | phys) & ~CONT_MASK) == 0) {
147 /* a block of CONT_PTES */
148 __populate_init_pte(pte, addr, next, phys,
Ard Biesheuvelb2195452015-11-09 09:55:45 +0100149 __pgprot(pgprot_val(prot) | PTE_CONT));
Jeremy Linton348a65c2015-10-07 12:00:25 -0500150 } else {
151 /*
152 * If the range being split is already inside of a
153 * contiguous range but this PTE isn't going to be
154 * contiguous, then we want to unmark the adjacent
155 * ranges, then update the portion of the range we
156 * are interrested in.
157 */
158 clear_cont_pte_range(pte, addr);
159 __populate_init_pte(pte, addr, next, phys, prot);
160 }
161
162 pte += (next - addr) >> PAGE_SHIFT;
163 phys += next - addr;
164 addr = next;
165 } while (addr != end);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000166}
167
Jisheng Zhang9a17a212015-11-12 20:04:43 +0800168static void split_pud(pud_t *old_pud, pmd_t *pmd)
Laura Abbottda141702015-01-21 17:36:06 -0800169{
170 unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
171 pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
172 int i = 0;
173
174 do {
Ard Biesheuvel1e43ba92015-06-30 18:04:49 +0200175 set_pmd(pmd, __pmd(addr | pgprot_val(prot)));
Laura Abbottda141702015-01-21 17:36:06 -0800176 addr += PMD_SIZE;
177 } while (pmd++, i++, i < PTRS_PER_PMD);
178}
179
180static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200181 unsigned long addr, unsigned long end,
Laura Abbottda141702015-01-21 17:36:06 -0800182 phys_addr_t phys, pgprot_t prot,
183 void *(*alloc)(unsigned long size))
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000184{
185 pmd_t *pmd;
186 unsigned long next;
187
188 /*
189 * Check for initial section mappings in the pgd/pud and remove them.
190 */
Mark Rutlanda1c76572015-01-27 16:36:30 +0000191 if (pud_none(*pud) || pud_sect(*pud)) {
Laura Abbottda141702015-01-21 17:36:06 -0800192 pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
193 if (pud_sect(*pud)) {
194 /*
195 * need to have the 1G of mappings continue to be
196 * present
197 */
198 split_pud(pud, pmd);
199 }
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200200 pud_populate(mm, pud, pmd);
Laura Abbottda141702015-01-21 17:36:06 -0800201 flush_tlb_all();
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000202 }
Mark Rutlanda1c76572015-01-27 16:36:30 +0000203 BUG_ON(pud_bad(*pud));
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000204
205 pmd = pmd_offset(pud, addr);
206 do {
207 next = pmd_addr_end(addr, end);
208 /* try section mapping first */
Catalin Marinasa55f9922014-02-04 16:01:31 +0000209 if (((addr | next | phys) & ~SECTION_MASK) == 0) {
210 pmd_t old_pmd =*pmd;
Ard Biesheuvel8ce837c2014-10-20 15:42:07 +0200211 set_pmd(pmd, __pmd(phys |
212 pgprot_val(mk_sect_prot(prot))));
Catalin Marinasa55f9922014-02-04 16:01:31 +0000213 /*
214 * Check for previous table entries created during
215 * boot (__create_page_tables) and flush them.
216 */
zhichang.yuan523d6e92014-12-09 07:26:47 +0000217 if (!pmd_none(old_pmd)) {
Catalin Marinasa55f9922014-02-04 16:01:31 +0000218 flush_tlb_all();
zhichang.yuan523d6e92014-12-09 07:26:47 +0000219 if (pmd_table(old_pmd)) {
220 phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
Catalin Marinas41089352015-01-29 17:33:35 +0000221 if (!WARN_ON_ONCE(slab_is_available()))
222 memblock_free(table, PAGE_SIZE);
zhichang.yuan523d6e92014-12-09 07:26:47 +0000223 }
224 }
Catalin Marinasa55f9922014-02-04 16:01:31 +0000225 } else {
Jeremy Linton348a65c2015-10-07 12:00:25 -0500226 alloc_init_pte(pmd, addr, next, phys, prot, alloc);
Catalin Marinasa55f9922014-02-04 16:01:31 +0000227 }
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000228 phys += next - addr;
229 } while (pmd++, addr = next, addr != end);
230}
231
Laura Abbottda141702015-01-21 17:36:06 -0800232static inline bool use_1G_block(unsigned long addr, unsigned long next,
233 unsigned long phys)
234{
235 if (PAGE_SHIFT != 12)
236 return false;
237
238 if (((addr | next | phys) & ~PUD_MASK) != 0)
239 return false;
240
241 return true;
242}
243
244static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200245 unsigned long addr, unsigned long end,
Laura Abbottda141702015-01-21 17:36:06 -0800246 phys_addr_t phys, pgprot_t prot,
247 void *(*alloc)(unsigned long size))
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000248{
Jungseok Leec79b9542014-05-12 18:40:51 +0900249 pud_t *pud;
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000250 unsigned long next;
251
Jungseok Leec79b9542014-05-12 18:40:51 +0900252 if (pgd_none(*pgd)) {
Laura Abbottda141702015-01-21 17:36:06 -0800253 pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200254 pgd_populate(mm, pgd, pud);
Jungseok Leec79b9542014-05-12 18:40:51 +0900255 }
256 BUG_ON(pgd_bad(*pgd));
257
258 pud = pud_offset(pgd, addr);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000259 do {
260 next = pud_addr_end(addr, end);
Steve Capper206a2a72014-05-06 14:02:27 +0100261
262 /*
263 * For 4K granule only, attempt to put down a 1GB block
264 */
Laura Abbottda141702015-01-21 17:36:06 -0800265 if (use_1G_block(addr, next, phys)) {
Steve Capper206a2a72014-05-06 14:02:27 +0100266 pud_t old_pud = *pud;
Ard Biesheuvel8ce837c2014-10-20 15:42:07 +0200267 set_pud(pud, __pud(phys |
268 pgprot_val(mk_sect_prot(prot))));
Steve Capper206a2a72014-05-06 14:02:27 +0100269
270 /*
271 * If we have an old value for a pud, it will
272 * be pointing to a pmd table that we no longer
273 * need (from swapper_pg_dir).
274 *
275 * Look up the old pmd table and free it.
276 */
277 if (!pud_none(old_pud)) {
Steve Capper206a2a72014-05-06 14:02:27 +0100278 flush_tlb_all();
zhichang.yuan523d6e92014-12-09 07:26:47 +0000279 if (pud_table(old_pud)) {
280 phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
Catalin Marinas41089352015-01-29 17:33:35 +0000281 if (!WARN_ON_ONCE(slab_is_available()))
282 memblock_free(table, PAGE_SIZE);
zhichang.yuan523d6e92014-12-09 07:26:47 +0000283 }
Steve Capper206a2a72014-05-06 14:02:27 +0100284 }
285 } else {
Laura Abbottda141702015-01-21 17:36:06 -0800286 alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
Steve Capper206a2a72014-05-06 14:02:27 +0100287 }
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000288 phys += next - addr;
289 } while (pud++, addr = next, addr != end);
290}
291
292/*
293 * Create the page directory entries and any necessary page tables for the
294 * mapping specified by 'md'.
295 */
Laura Abbottda141702015-01-21 17:36:06 -0800296static void __create_mapping(struct mm_struct *mm, pgd_t *pgd,
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200297 phys_addr_t phys, unsigned long virt,
Laura Abbottda141702015-01-21 17:36:06 -0800298 phys_addr_t size, pgprot_t prot,
299 void *(*alloc)(unsigned long size))
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000300{
301 unsigned long addr, length, end, next;
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000302
303 addr = virt & PAGE_MASK;
304 length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));
305
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000306 end = addr + length;
307 do {
308 next = pgd_addr_end(addr, end);
Laura Abbottda141702015-01-21 17:36:06 -0800309 alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000310 phys += next - addr;
311 } while (pgd++, addr = next, addr != end);
312}
313
Laura Abbottda141702015-01-21 17:36:06 -0800314static void *late_alloc(unsigned long size)
315{
316 void *ptr;
317
318 BUG_ON(size > PAGE_SIZE);
319 ptr = (void *)__get_free_page(PGALLOC_GFP);
320 BUG_ON(!ptr);
321 return ptr;
322}
323
Mark Rutlandc53e0ba2015-07-28 10:31:06 +0100324static void __init create_mapping(phys_addr_t phys, unsigned long virt,
Laura Abbottda141702015-01-21 17:36:06 -0800325 phys_addr_t size, pgprot_t prot)
Mark Salterd7ecbdd2014-03-12 12:28:06 -0400326{
327 if (virt < VMALLOC_START) {
328 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
329 &phys, virt);
330 return;
331 }
Ard Biesheuvele1e1fdd2014-10-20 14:02:15 +0200332 __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
Laura Abbottda141702015-01-21 17:36:06 -0800333 size, prot, early_alloc);
Mark Salterd7ecbdd2014-03-12 12:28:06 -0400334}
335
Ard Biesheuvel8ce837c2014-10-20 15:42:07 +0200336void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
337 unsigned long virt, phys_addr_t size,
338 pgprot_t prot)
339{
Laura Abbottda141702015-01-21 17:36:06 -0800340 __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
Ard Biesheuvel60305db2015-01-22 10:01:40 +0000341 late_alloc);
Mark Salterd7ecbdd2014-03-12 12:28:06 -0400342}
343
Laura Abbottda141702015-01-21 17:36:06 -0800344static void create_mapping_late(phys_addr_t phys, unsigned long virt,
345 phys_addr_t size, pgprot_t prot)
346{
347 if (virt < VMALLOC_START) {
348 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
349 &phys, virt);
350 return;
351 }
352
353 return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
354 phys, virt, size, prot, late_alloc);
355}
356
357#ifdef CONFIG_DEBUG_RODATA
358static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
359{
360 /*
361 * Set up the executable regions using the existing section mappings
362 * for now. This will get more fine grained later once all memory
363 * is mapped
364 */
365 unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
366 unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
367
368 if (end < kernel_x_start) {
369 create_mapping(start, __phys_to_virt(start),
370 end - start, PAGE_KERNEL);
371 } else if (start >= kernel_x_end) {
372 create_mapping(start, __phys_to_virt(start),
373 end - start, PAGE_KERNEL);
374 } else {
375 if (start < kernel_x_start)
376 create_mapping(start, __phys_to_virt(start),
377 kernel_x_start - start,
378 PAGE_KERNEL);
379 create_mapping(kernel_x_start,
380 __phys_to_virt(kernel_x_start),
381 kernel_x_end - kernel_x_start,
382 PAGE_KERNEL_EXEC);
383 if (kernel_x_end < end)
384 create_mapping(kernel_x_end,
385 __phys_to_virt(kernel_x_end),
386 end - kernel_x_end,
387 PAGE_KERNEL);
388 }
389
390}
391#else
392static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
393{
394 create_mapping(start, __phys_to_virt(start), end - start,
395 PAGE_KERNEL_EXEC);
396}
397#endif
398
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000399static void __init map_mem(void)
400{
401 struct memblock_region *reg;
Catalin Marinase25208f2013-08-23 18:04:44 +0100402 phys_addr_t limit;
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000403
Steve Capperf6bc87c2013-04-30 11:00:33 +0100404 /*
405 * Temporarily limit the memblock range. We need to do this as
406 * create_mapping requires puds, pmds and ptes to be allocated from
407 * memory addressable from the initial direct kernel mapping.
408 *
Catalin Marinas3dec0fe2014-10-24 18:16:47 +0100409 * The initial direct kernel mapping, located at swapper_pg_dir, gives
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100410 * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps,
411 * memory starting from PHYS_OFFSET (which must be aligned to 2MB as
412 * per Documentation/arm64/booting.txt).
Steve Capperf6bc87c2013-04-30 11:00:33 +0100413 */
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100414 limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE;
Catalin Marinase25208f2013-08-23 18:04:44 +0100415 memblock_set_current_limit(limit);
Steve Capperf6bc87c2013-04-30 11:00:33 +0100416
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000417 /* map all the memory banks */
418 for_each_memblock(memory, reg) {
419 phys_addr_t start = reg->base;
420 phys_addr_t end = start + reg->size;
421
422 if (start >= end)
423 break;
424
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100425 if (ARM64_SWAPPER_USES_SECTION_MAPS) {
426 /*
427 * For the first memory bank align the start address and
428 * current memblock limit to prevent create_mapping() from
429 * allocating pte page tables from unmapped memory. With
430 * the section maps, if the first block doesn't end on section
431 * size boundary, create_mapping() will try to allocate a pte
432 * page, which may be returned from an unmapped area.
433 * When section maps are not used, the pte page table for the
434 * current limit is already present in swapper_pg_dir.
435 */
436 if (start < limit)
437 start = ALIGN(start, SECTION_SIZE);
438 if (end < limit) {
439 limit = end & SECTION_MASK;
440 memblock_set_current_limit(limit);
441 }
Catalin Marinase25208f2013-08-23 18:04:44 +0100442 }
Laura Abbottda141702015-01-21 17:36:06 -0800443 __map_memblock(start, end);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000444 }
Steve Capperf6bc87c2013-04-30 11:00:33 +0100445
446 /* Limit no longer required. */
447 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000448}
449
Jisheng Zhang9a17a212015-11-12 20:04:43 +0800450static void __init fixup_executable(void)
Laura Abbottda141702015-01-21 17:36:06 -0800451{
452#ifdef CONFIG_DEBUG_RODATA
453 /* now that we are actually fully mapped, make the start/end more fine grained */
454 if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
455 unsigned long aligned_start = round_down(__pa(_stext),
456 SECTION_SIZE);
457
458 create_mapping(aligned_start, __phys_to_virt(aligned_start),
459 __pa(_stext) - aligned_start,
460 PAGE_KERNEL);
461 }
462
463 if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
464 unsigned long aligned_end = round_up(__pa(__init_end),
465 SECTION_SIZE);
466 create_mapping(__pa(__init_end), (unsigned long)__init_end,
467 aligned_end - __pa(__init_end),
468 PAGE_KERNEL);
469 }
470#endif
471}
472
473#ifdef CONFIG_DEBUG_RODATA
474void mark_rodata_ro(void)
475{
476 create_mapping_late(__pa(_stext), (unsigned long)_stext,
477 (unsigned long)_etext - (unsigned long)_stext,
478 PAGE_KERNEL_EXEC | PTE_RDONLY);
479
480}
481#endif
482
483void fixup_init(void)
484{
485 create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
486 (unsigned long)__init_end - (unsigned long)__init_begin,
487 PAGE_KERNEL);
488}
489
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000490/*
491 * paging_init() sets up the page tables, initialises the zone memory
492 * maps and sets up the zero page.
493 */
494void __init paging_init(void)
495{
496 void *zero_page;
497
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000498 map_mem();
Laura Abbottda141702015-01-21 17:36:06 -0800499 fixup_executable();
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000500
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000501 /* allocate the zero page. */
502 zero_page = early_alloc(PAGE_SIZE);
503
504 bootmem_init();
505
506 empty_zero_page = virt_to_page(zero_page);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000507
508 /*
509 * TTBR0 is only used for the identity mapping at this stage. Make it
510 * point to zero page to avoid speculatively fetching new entries.
511 */
512 cpu_set_reserved_ttbr0();
Will Deacon8e63d382015-10-06 18:46:23 +0100513 local_flush_tlb_all();
Ard Biesheuveldd006da2015-03-19 16:42:27 +0000514 cpu_set_default_tcr_t0sz();
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000515}
516
517/*
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000518 * Check whether a kernel address is valid (derived from arch/x86/).
519 */
520int kern_addr_valid(unsigned long addr)
521{
522 pgd_t *pgd;
523 pud_t *pud;
524 pmd_t *pmd;
525 pte_t *pte;
526
527 if ((((long)addr) >> VA_BITS) != -1UL)
528 return 0;
529
530 pgd = pgd_offset_k(addr);
531 if (pgd_none(*pgd))
532 return 0;
533
534 pud = pud_offset(pgd, addr);
535 if (pud_none(*pud))
536 return 0;
537
Steve Capper206a2a72014-05-06 14:02:27 +0100538 if (pud_sect(*pud))
539 return pfn_valid(pud_pfn(*pud));
540
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000541 pmd = pmd_offset(pud, addr);
542 if (pmd_none(*pmd))
543 return 0;
544
Dave Andersonda6e4cb2014-04-15 18:53:24 +0100545 if (pmd_sect(*pmd))
546 return pfn_valid(pmd_pfn(*pmd));
547
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000548 pte = pte_offset_kernel(pmd, addr);
549 if (pte_none(*pte))
550 return 0;
551
552 return pfn_valid(pte_pfn(*pte));
553}
554#ifdef CONFIG_SPARSEMEM_VMEMMAP
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100555#if !ARM64_SWAPPER_USES_SECTION_MAPS
Johannes Weiner0aad8182013-04-29 15:07:50 -0700556int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000557{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700558 return vmemmap_populate_basepages(start, end, node);
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000559}
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100560#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
Johannes Weiner0aad8182013-04-29 15:07:50 -0700561int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000562{
Johannes Weiner0aad8182013-04-29 15:07:50 -0700563 unsigned long addr = start;
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000564 unsigned long next;
565 pgd_t *pgd;
566 pud_t *pud;
567 pmd_t *pmd;
568
569 do {
570 next = pmd_addr_end(addr, end);
571
572 pgd = vmemmap_pgd_populate(addr, node);
573 if (!pgd)
574 return -ENOMEM;
575
576 pud = vmemmap_pud_populate(pgd, addr, node);
577 if (!pud)
578 return -ENOMEM;
579
580 pmd = pmd_offset(pud, addr);
581 if (pmd_none(*pmd)) {
582 void *p = NULL;
583
584 p = vmemmap_alloc_block_buf(PMD_SIZE, node);
585 if (!p)
586 return -ENOMEM;
587
Catalin Marinasa501e322014-04-03 15:57:15 +0100588 set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000589 } else
590 vmemmap_verify((pte_t *)pmd, node, addr, next);
591 } while (addr = next, addr != end);
592
593 return 0;
594}
595#endif /* CONFIG_ARM64_64K_PAGES */
Johannes Weiner0aad8182013-04-29 15:07:50 -0700596void vmemmap_free(unsigned long start, unsigned long end)
Tang Chen01975182013-02-22 16:33:08 -0800597{
598}
Catalin Marinasc1cc1552012-03-05 11:49:27 +0000599#endif /* CONFIG_SPARSEMEM_VMEMMAP */
Laura Abbottaf86e592014-11-21 21:50:42 +0000600
601static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
Kirill A. Shutemov9f25e6a2015-04-14 15:45:39 -0700602#if CONFIG_PGTABLE_LEVELS > 2
Laura Abbottaf86e592014-11-21 21:50:42 +0000603static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
604#endif
Kirill A. Shutemov9f25e6a2015-04-14 15:45:39 -0700605#if CONFIG_PGTABLE_LEVELS > 3
Laura Abbottaf86e592014-11-21 21:50:42 +0000606static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
607#endif
608
609static inline pud_t * fixmap_pud(unsigned long addr)
610{
611 pgd_t *pgd = pgd_offset_k(addr);
612
613 BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));
614
615 return pud_offset(pgd, addr);
616}
617
618static inline pmd_t * fixmap_pmd(unsigned long addr)
619{
620 pud_t *pud = fixmap_pud(addr);
621
622 BUG_ON(pud_none(*pud) || pud_bad(*pud));
623
624 return pmd_offset(pud, addr);
625}
626
627static inline pte_t * fixmap_pte(unsigned long addr)
628{
629 pmd_t *pmd = fixmap_pmd(addr);
630
631 BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd));
632
633 return pte_offset_kernel(pmd, addr);
634}
635
636void __init early_fixmap_init(void)
637{
638 pgd_t *pgd;
639 pud_t *pud;
640 pmd_t *pmd;
641 unsigned long addr = FIXADDR_START;
642
643 pgd = pgd_offset_k(addr);
644 pgd_populate(&init_mm, pgd, bm_pud);
645 pud = pud_offset(pgd, addr);
646 pud_populate(&init_mm, pud, bm_pmd);
647 pmd = pmd_offset(pud, addr);
648 pmd_populate_kernel(&init_mm, pmd, bm_pte);
649
650 /*
651 * The boot-ioremap range spans multiple pmds, for which
652 * we are not preparted:
653 */
654 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
655 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
656
657 if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
658 || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
659 WARN_ON(1);
660 pr_warn("pmd %p != %p, %p\n",
661 pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
662 fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
663 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
664 fix_to_virt(FIX_BTMAP_BEGIN));
665 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
666 fix_to_virt(FIX_BTMAP_END));
667
668 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
669 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN);
670 }
671}
672
673void __set_fixmap(enum fixed_addresses idx,
674 phys_addr_t phys, pgprot_t flags)
675{
676 unsigned long addr = __fix_to_virt(idx);
677 pte_t *pte;
678
Mark Rutlandb63dbef2015-03-04 13:27:35 +0000679 BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
Laura Abbottaf86e592014-11-21 21:50:42 +0000680
681 pte = fixmap_pte(addr);
682
683 if (pgprot_val(flags)) {
684 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
685 } else {
686 pte_clear(&init_mm, addr, pte);
687 flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
688 }
689}
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200690
691void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
692{
693 const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
Ard Biesheuvelfb226c32015-11-09 09:55:46 +0100694 pgprot_t prot = PAGE_KERNEL_RO;
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100695 int size, offset;
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200696 void *dt_virt;
697
698 /*
699 * Check whether the physical FDT address is set and meets the minimum
700 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
701 * at least 8 bytes so that we can always access the size field of the
702 * FDT header after mapping the first chunk, double check here if that
703 * is indeed the case.
704 */
705 BUILD_BUG_ON(MIN_FDT_ALIGN < 8);
706 if (!dt_phys || dt_phys % MIN_FDT_ALIGN)
707 return NULL;
708
709 /*
710 * Make sure that the FDT region can be mapped without the need to
711 * allocate additional translation table pages, so that it is safe
712 * to call create_mapping() this early.
713 *
714 * On 64k pages, the FDT will be mapped using PTEs, so we need to
715 * be in the same PMD as the rest of the fixmap.
716 * On 4k pages, we'll use section mappings for the FDT so we only
717 * have to be in the same PUD.
718 */
719 BUILD_BUG_ON(dt_virt_base % SZ_2M);
720
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100721 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
722 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200723
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100724 offset = dt_phys % SWAPPER_BLOCK_SIZE;
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200725 dt_virt = (void *)dt_virt_base + offset;
726
727 /* map the first chunk so we can read the size from the header */
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100728 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
729 SWAPPER_BLOCK_SIZE, prot);
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200730
731 if (fdt_check_header(dt_virt) != 0)
732 return NULL;
733
734 size = fdt_totalsize(dt_virt);
735 if (size > MAX_FDT_SIZE)
736 return NULL;
737
Suzuki K. Pouloseb433dce2015-10-19 14:19:28 +0100738 if (offset + size > SWAPPER_BLOCK_SIZE)
739 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
740 round_up(offset + size, SWAPPER_BLOCK_SIZE), prot);
Ard Biesheuvel61bd93c2015-06-01 13:40:32 +0200741
742 memblock_reserve(dt_phys, size);
743
744 return dt_virt;
745}