blob: 71f1415e2472399049a2273eded9c4fb8d173265 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
7 * Copyright (C) 1996 Paul Mackerras
Paul Mackerras14cf11a2005-09-26 16:04:21 +10008 *
9 * Derived from "arch/i386/mm/init.c"
10 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
11 *
12 * Dave Engebretsen <engebret@us.ibm.com>
13 * Rework for PPC64 port.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
21
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +100022#undef DEBUG
23
Paul Mackerras14cf11a2005-09-26 16:04:21 +100024#include <linux/signal.h>
25#include <linux/sched.h>
26#include <linux/kernel.h>
27#include <linux/errno.h>
28#include <linux/string.h>
29#include <linux/types.h>
30#include <linux/mman.h>
31#include <linux/mm.h>
32#include <linux/swap.h>
33#include <linux/stddef.h>
34#include <linux/vmalloc.h>
35#include <linux/init.h>
36#include <linux/delay.h>
37#include <linux/bootmem.h>
38#include <linux/highmem.h>
39#include <linux/idr.h>
40#include <linux/nodemask.h>
41#include <linux/module.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070042#include <linux/poison.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100043#include <linux/memblock.h>
David Gibsona4fe3ce2009-10-26 19:24:31 +000044#include <linux/hugetlb.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090045#include <linux/slab.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100046
47#include <asm/pgalloc.h>
48#include <asm/page.h>
49#include <asm/prom.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100050#include <asm/rtas.h>
51#include <asm/io.h>
52#include <asm/mmu_context.h>
53#include <asm/pgtable.h>
54#include <asm/mmu.h>
55#include <asm/uaccess.h>
56#include <asm/smp.h>
57#include <asm/machdep.h>
58#include <asm/tlb.h>
59#include <asm/eeh.h>
60#include <asm/processor.h>
61#include <asm/mmzone.h>
62#include <asm/cputable.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100063#include <asm/sections.h>
64#include <asm/system.h>
65#include <asm/iommu.h>
66#include <asm/abs_addr.h>
67#include <asm/vdso.h>
David Gibson800fc3e2005-11-16 15:43:48 +110068
69#include "mmu_decl.h"
Paul Mackerras14cf11a2005-09-26 16:04:21 +100070
Benjamin Herrenschmidt94491682009-06-02 21:17:45 +000071#ifdef CONFIG_PPC_STD_MMU_64
Paul Mackerras14cf11a2005-09-26 16:04:21 +100072#if PGTABLE_RANGE > USER_VSID_RANGE
73#warning Limited user VSID range means pagetable space is wasted
74#endif
75
76#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
77#warning TASK_SIZE is smaller than it needs to be.
78#endif
Benjamin Herrenschmidt94491682009-06-02 21:17:45 +000079#endif /* CONFIG_PPC_STD_MMU_64 */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100080
Kumar Gala37dd2ba2008-04-22 04:22:34 +100081phys_addr_t memstart_addr = ~0;
82phys_addr_t kernstart_addr;
Kumar Galad7917ba2008-04-16 05:52:22 +100083
Paul Mackerras14cf11a2005-09-26 16:04:21 +100084void free_initmem(void)
85{
86 unsigned long addr;
87
88 addr = (unsigned long)__init_begin;
89 for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {
Randy Dunlapc9cf5522006-06-27 02:53:52 -070090 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
Paul Mackerras14cf11a2005-09-26 16:04:21 +100091 ClearPageReserved(virt_to_page(addr));
Nick Piggin7835e982006-03-22 00:08:40 -080092 init_page_count(virt_to_page(addr));
Paul Mackerras14cf11a2005-09-26 16:04:21 +100093 free_page(addr);
94 totalram_pages++;
95 }
96 printk ("Freeing unused kernel memory: %luk freed\n",
97 ((unsigned long)__init_end - (unsigned long)__init_begin) >> 10);
98}
99
100#ifdef CONFIG_BLK_DEV_INITRD
101void free_initrd_mem(unsigned long start, unsigned long end)
102{
103 if (start < end)
104 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
105 for (; start < end; start += PAGE_SIZE) {
106 ClearPageReserved(virt_to_page(start));
Nick Piggin7835e982006-03-22 00:08:40 -0800107 init_page_count(virt_to_page(start));
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000108 free_page(start);
109 totalram_pages++;
110 }
111}
112#endif
113
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700114static void pgd_ctor(void *addr)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000115{
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700116 memset(addr, 0, PGD_TABLE_SIZE);
117}
118
119static void pmd_ctor(void *addr)
120{
121 memset(addr, 0, PMD_TABLE_SIZE);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000122}
123
David Gibsona0668cd2009-10-28 16:27:18 +0000124struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE];
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000125
David Gibsona0668cd2009-10-28 16:27:18 +0000126/*
127 * Create a kmem_cache() for pagetables. This is not used for PTE
128 * pages - they're linked to struct page, come from the normal free
129 * pages pool and have a different entry size (see real_pte_t) to
130 * everything else. Caches created by this function are used for all
131 * the higher level pagetables, and for hugepage pagetables.
132 */
133void pgtable_cache_add(unsigned shift, void (*ctor)(void *))
134{
135 char *name;
136 unsigned long table_size = sizeof(void *) << shift;
137 unsigned long align = table_size;
138
139 /* When batching pgtable pointers for RCU freeing, we store
140 * the index size in the low bits. Table alignment must be
David Gibsona4fe3ce2009-10-26 19:24:31 +0000141 * big enough to fit it.
142 *
143 * Likewise, hugeapge pagetable pointers contain a (different)
144 * shift value in the low bits. All tables must be aligned so
145 * as to leave enough 0 bits in the address to contain it. */
146 unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1,
147 HUGEPD_SHIFT_MASK + 1);
David Gibsona0668cd2009-10-28 16:27:18 +0000148 struct kmem_cache *new;
149
150 /* It would be nice if this was a BUILD_BUG_ON(), but at the
151 * moment, gcc doesn't seem to recognize is_power_of_2 as a
152 * constant expression, so so much for that. */
153 BUG_ON(!is_power_of_2(minalign));
154 BUG_ON((shift < 1) || (shift > MAX_PGTABLE_INDEX_SIZE));
155
156 if (PGT_CACHE(shift))
157 return; /* Already have a cache of this size */
158
159 align = max_t(unsigned long, align, minalign);
160 name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift);
161 new = kmem_cache_create(name, table_size, align, 0, ctor);
162 PGT_CACHE(shift) = new;
163
164 pr_debug("Allocated pgtable cache for order %d\n", shift);
165}
166
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000167
168void pgtable_cache_init(void)
169{
David Gibsona0668cd2009-10-28 16:27:18 +0000170 pgtable_cache_add(PGD_INDEX_SIZE, pgd_ctor);
171 pgtable_cache_add(PMD_INDEX_SIZE, pmd_ctor);
172 if (!PGT_CACHE(PGD_INDEX_SIZE) || !PGT_CACHE(PMD_INDEX_SIZE))
173 panic("Couldn't allocate pgtable caches");
174
175 /* In all current configs, when the PUD index exists it's the
176 * same size as either the pgd or pmd index. Verify that the
177 * initialization above has also created a PUD cache. This
178 * will need re-examiniation if we add new possibilities for
179 * the pagetable layout. */
180 BUG_ON(PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE));
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000181}
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700182
183#ifdef CONFIG_SPARSEMEM_VMEMMAP
184/*
185 * Given an address within the vmemmap, determine the pfn of the page that
186 * represents the start of the section it is within. Note that we have to
187 * do this by hand as the proffered address may not be correctly aligned.
188 * Subtraction of non-aligned pointers produces undefined results.
189 */
Michael Ellerman09de9ff2008-05-08 14:27:07 +1000190static unsigned long __meminit vmemmap_section_start(unsigned long page)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700191{
192 unsigned long offset = page - ((unsigned long)(vmemmap));
193
194 /* Return the pfn of the start of the section. */
195 return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;
196}
197
198/*
199 * Check if this vmemmap page is already initialised. If any section
200 * which overlaps this vmemmap page is initialised then this page is
201 * initialised already.
202 */
Michael Ellerman09de9ff2008-05-08 14:27:07 +1000203static int __meminit vmemmap_populated(unsigned long start, int page_size)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700204{
205 unsigned long end = start + page_size;
206
207 for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
208 if (pfn_valid(vmemmap_section_start(start)))
209 return 1;
210
211 return 0;
212}
213
Benjamin Herrenschmidt32a74942009-07-23 23:15:58 +0000214/* On hash-based CPUs, the vmemmap is bolted in the hash table.
215 *
216 * On Book3E CPUs, the vmemmap is currently mapped in the top half of
217 * the vmalloc space using normal page tables, though the size of
218 * pages encoded in the PTEs can be different
219 */
220
221#ifdef CONFIG_PPC_BOOK3E
222static void __meminit vmemmap_create_mapping(unsigned long start,
223 unsigned long page_size,
224 unsigned long phys)
225{
226 /* Create a PTE encoding without page size */
227 unsigned long i, flags = _PAGE_PRESENT | _PAGE_ACCESSED |
228 _PAGE_KERNEL_RW;
229
230 /* PTEs only contain page size encodings up to 32M */
231 BUG_ON(mmu_psize_defs[mmu_vmemmap_psize].enc > 0xf);
232
233 /* Encode the size in the PTE */
234 flags |= mmu_psize_defs[mmu_vmemmap_psize].enc << 8;
235
236 /* For each PTE for that area, map things. Note that we don't
237 * increment phys because all PTEs are of the large size and
238 * thus must have the low bits clear
239 */
240 for (i = 0; i < page_size; i += PAGE_SIZE)
241 BUG_ON(map_kernel_page(start + i, phys, flags));
242}
243#else /* CONFIG_PPC_BOOK3E */
244static void __meminit vmemmap_create_mapping(unsigned long start,
245 unsigned long page_size,
246 unsigned long phys)
247{
248 int mapped = htab_bolt_mapping(start, start + page_size, phys,
249 PAGE_KERNEL, mmu_vmemmap_psize,
250 mmu_kernel_ssize);
251 BUG_ON(mapped < 0);
252}
253#endif /* CONFIG_PPC_BOOK3E */
254
Mark Nelson91eea672010-04-21 16:21:03 +0000255struct vmemmap_backing *vmemmap_list;
256
257static __meminit struct vmemmap_backing * vmemmap_list_alloc(int node)
258{
259 static struct vmemmap_backing *next;
260 static int num_left;
261
262 /* allocate a page when required and hand out chunks */
263 if (!next || !num_left) {
264 next = vmemmap_alloc_block(PAGE_SIZE, node);
265 if (unlikely(!next)) {
266 WARN_ON(1);
267 return NULL;
268 }
269 num_left = PAGE_SIZE / sizeof(struct vmemmap_backing);
270 }
271
272 num_left--;
273
274 return next++;
275}
276
277static __meminit void vmemmap_list_populate(unsigned long phys,
278 unsigned long start,
279 int node)
280{
281 struct vmemmap_backing *vmem_back;
282
283 vmem_back = vmemmap_list_alloc(node);
284 if (unlikely(!vmem_back)) {
285 WARN_ON(1);
286 return;
287 }
288
289 vmem_back->phys = phys;
290 vmem_back->virt_addr = start;
291 vmem_back->list = vmemmap_list;
292
293 vmemmap_list = vmem_back;
294}
295
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700296int __meminit vmemmap_populate(struct page *start_page,
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000297 unsigned long nr_pages, int node)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700298{
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700299 unsigned long start = (unsigned long)start_page;
300 unsigned long end = (unsigned long)(start_page + nr_pages);
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000301 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700302
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700303 /* Align to the page size of the linear mapping. */
304 start = _ALIGN_DOWN(start, page_size);
305
Benjamin Herrenschmidt32a74942009-07-23 23:15:58 +0000306 pr_debug("vmemmap_populate page %p, %ld pages, node %d\n",
307 start_page, nr_pages, node);
308 pr_debug(" -> map %lx..%lx\n", start, end);
309
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700310 for (; start < end; start += page_size) {
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700311 void *p;
312
313 if (vmemmap_populated(start, page_size))
314 continue;
315
316 p = vmemmap_alloc_block(page_size, node);
317 if (!p)
318 return -ENOMEM;
319
Mark Nelson91eea672010-04-21 16:21:03 +0000320 vmemmap_list_populate(__pa(p), start, node);
321
Benjamin Herrenschmidt32a74942009-07-23 23:15:58 +0000322 pr_debug(" * %016lx..%016lx allocated at %p\n",
323 start, start + page_size, p);
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700324
Benjamin Herrenschmidt32a74942009-07-23 23:15:58 +0000325 vmemmap_create_mapping(start, page_size, __pa(p));
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700326 }
327
328 return 0;
329}
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000330#endif /* CONFIG_SPARSEMEM_VMEMMAP */