blob: 68a821add28df21c09c2716cc2cb8ef5c77dc796 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
7 * Copyright (C) 1996 Paul Mackerras
Paul Mackerras14cf11a2005-09-26 16:04:21 +10008 *
9 * Derived from "arch/i386/mm/init.c"
10 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
11 *
12 * Dave Engebretsen <engebret@us.ibm.com>
13 * Rework for PPC64 port.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 *
20 */
21
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +100022#undef DEBUG
23
Paul Mackerras14cf11a2005-09-26 16:04:21 +100024#include <linux/signal.h>
25#include <linux/sched.h>
26#include <linux/kernel.h>
27#include <linux/errno.h>
28#include <linux/string.h>
29#include <linux/types.h>
30#include <linux/mman.h>
31#include <linux/mm.h>
32#include <linux/swap.h>
33#include <linux/stddef.h>
34#include <linux/vmalloc.h>
35#include <linux/init.h>
36#include <linux/delay.h>
37#include <linux/bootmem.h>
38#include <linux/highmem.h>
39#include <linux/idr.h>
40#include <linux/nodemask.h>
41#include <linux/module.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070042#include <linux/poison.h>
David S. Millerd9b2b2a2008-02-13 16:56:49 -080043#include <linux/lmb.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100044
45#include <asm/pgalloc.h>
46#include <asm/page.h>
47#include <asm/prom.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100048#include <asm/rtas.h>
49#include <asm/io.h>
50#include <asm/mmu_context.h>
51#include <asm/pgtable.h>
52#include <asm/mmu.h>
53#include <asm/uaccess.h>
54#include <asm/smp.h>
55#include <asm/machdep.h>
56#include <asm/tlb.h>
57#include <asm/eeh.h>
58#include <asm/processor.h>
59#include <asm/mmzone.h>
60#include <asm/cputable.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100061#include <asm/sections.h>
62#include <asm/system.h>
63#include <asm/iommu.h>
64#include <asm/abs_addr.h>
65#include <asm/vdso.h>
David Gibson800fc3e2005-11-16 15:43:48 +110066
67#include "mmu_decl.h"
Paul Mackerras14cf11a2005-09-26 16:04:21 +100068
Benjamin Herrenschmidt94491682009-06-02 21:17:45 +000069#ifdef CONFIG_PPC_STD_MMU_64
Paul Mackerras14cf11a2005-09-26 16:04:21 +100070#if PGTABLE_RANGE > USER_VSID_RANGE
71#warning Limited user VSID range means pagetable space is wasted
72#endif
73
74#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
75#warning TASK_SIZE is smaller than it needs to be.
76#endif
Benjamin Herrenschmidt94491682009-06-02 21:17:45 +000077#endif /* CONFIG_PPC_STD_MMU_64 */
Paul Mackerras14cf11a2005-09-26 16:04:21 +100078
Kumar Gala37dd2ba2008-04-22 04:22:34 +100079phys_addr_t memstart_addr = ~0;
80phys_addr_t kernstart_addr;
Kumar Galad7917ba2008-04-16 05:52:22 +100081
Paul Mackerras14cf11a2005-09-26 16:04:21 +100082void free_initmem(void)
83{
84 unsigned long addr;
85
86 addr = (unsigned long)__init_begin;
87 for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {
Randy Dunlapc9cf5522006-06-27 02:53:52 -070088 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
Paul Mackerras14cf11a2005-09-26 16:04:21 +100089 ClearPageReserved(virt_to_page(addr));
Nick Piggin7835e982006-03-22 00:08:40 -080090 init_page_count(virt_to_page(addr));
Paul Mackerras14cf11a2005-09-26 16:04:21 +100091 free_page(addr);
92 totalram_pages++;
93 }
94 printk ("Freeing unused kernel memory: %luk freed\n",
95 ((unsigned long)__init_end - (unsigned long)__init_begin) >> 10);
96}
97
98#ifdef CONFIG_BLK_DEV_INITRD
99void free_initrd_mem(unsigned long start, unsigned long end)
100{
101 if (start < end)
102 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
103 for (; start < end; start += PAGE_SIZE) {
104 ClearPageReserved(virt_to_page(start));
Nick Piggin7835e982006-03-22 00:08:40 -0800105 init_page_count(virt_to_page(start));
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000106 free_page(start);
107 totalram_pages++;
108 }
109}
110#endif
111
Ed Swarthoutdf174e32007-09-21 12:53:02 +1000112#ifdef CONFIG_PROC_KCORE
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000113static struct kcore_list kcore_vmem;
114
115static int __init setup_kcore(void)
116{
117 int i;
118
119 for (i=0; i < lmb.memory.cnt; i++) {
120 unsigned long base, size;
121 struct kcore_list *kcore_mem;
122
123 base = lmb.memory.region[i].base;
124 size = lmb.memory.region[i].size;
125
126 /* GFP_ATOMIC to avoid might_sleep warnings during boot */
127 kcore_mem = kmalloc(sizeof(struct kcore_list), GFP_ATOMIC);
128 if (!kcore_mem)
Harvey Harrisone48b1b42008-03-29 08:21:07 +1100129 panic("%s: kmalloc failed\n", __func__);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000130
131 kclist_add(kcore_mem, __va(base), size);
132 }
133
134 kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START);
135
136 return 0;
137}
138module_init(setup_kcore);
Ed Swarthoutdf174e32007-09-21 12:53:02 +1000139#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000140
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700141static void pgd_ctor(void *addr)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000142{
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700143 memset(addr, 0, PGD_TABLE_SIZE);
144}
145
146static void pmd_ctor(void *addr)
147{
148 memset(addr, 0, PMD_TABLE_SIZE);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000149}
150
Benjamin Herrenschmidt87655ff2005-11-10 14:53:16 +1100151static const unsigned int pgtable_cache_size[2] = {
Hugh Dickins517e2262007-05-09 14:38:48 +1000152 PGD_TABLE_SIZE, PMD_TABLE_SIZE
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000153};
154static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {
Hugh Dickins517e2262007-05-09 14:38:48 +1000155#ifdef CONFIG_PPC_64K_PAGES
156 "pgd_cache", "pmd_cache",
157#else
158 "pgd_cache", "pud_pmd_cache",
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100159#endif /* CONFIG_PPC_64K_PAGES */
Hugh Dickins517e2262007-05-09 14:38:48 +1000160};
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000161
David Gibsonf10a04c2006-04-28 15:02:51 +1000162#ifdef CONFIG_HUGETLB_PAGE
Jon Tollefson0d9ea752008-07-23 21:27:56 -0700163/* Hugepages need an extra cache per hugepagesize, initialized in
164 * hugetlbpage.c. We can't put into the tables above, because HPAGE_SHIFT
165 * is not compile time constant. */
166struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)+MMU_PAGE_COUNT];
David Gibsonf10a04c2006-04-28 15:02:51 +1000167#else
Christoph Lametere18b8902006-12-06 20:33:20 -0800168struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];
David Gibsonf10a04c2006-04-28 15:02:51 +1000169#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000170
171void pgtable_cache_init(void)
172{
Alexey Dobriyan51cc5062008-07-25 19:45:34 -0700173 pgtable_cache[0] = kmem_cache_create(pgtable_cache_name[0], PGD_TABLE_SIZE, PGD_TABLE_SIZE, SLAB_PANIC, pgd_ctor);
174 pgtable_cache[1] = kmem_cache_create(pgtable_cache_name[1], PMD_TABLE_SIZE, PMD_TABLE_SIZE, SLAB_PANIC, pmd_ctor);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000175}
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700176
177#ifdef CONFIG_SPARSEMEM_VMEMMAP
178/*
179 * Given an address within the vmemmap, determine the pfn of the page that
180 * represents the start of the section it is within. Note that we have to
181 * do this by hand as the proffered address may not be correctly aligned.
182 * Subtraction of non-aligned pointers produces undefined results.
183 */
Michael Ellerman09de9ff2008-05-08 14:27:07 +1000184static unsigned long __meminit vmemmap_section_start(unsigned long page)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700185{
186 unsigned long offset = page - ((unsigned long)(vmemmap));
187
188 /* Return the pfn of the start of the section. */
189 return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;
190}
191
192/*
193 * Check if this vmemmap page is already initialised. If any section
194 * which overlaps this vmemmap page is initialised then this page is
195 * initialised already.
196 */
Michael Ellerman09de9ff2008-05-08 14:27:07 +1000197static int __meminit vmemmap_populated(unsigned long start, int page_size)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700198{
199 unsigned long end = start + page_size;
200
201 for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
202 if (pfn_valid(vmemmap_section_start(start)))
203 return 1;
204
205 return 0;
206}
207
208int __meminit vmemmap_populate(struct page *start_page,
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000209 unsigned long nr_pages, int node)
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700210{
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700211 unsigned long start = (unsigned long)start_page;
212 unsigned long end = (unsigned long)(start_page + nr_pages);
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000213 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700214
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700215 /* Align to the page size of the linear mapping. */
216 start = _ALIGN_DOWN(start, page_size);
217
218 for (; start < end; start += page_size) {
219 int mapped;
220 void *p;
221
222 if (vmemmap_populated(start, page_size))
223 continue;
224
225 p = vmemmap_alloc_block(page_size, node);
226 if (!p)
227 return -ENOMEM;
228
Stephen Rothwell6548d832007-11-13 15:41:49 +1100229 pr_debug("vmemmap %08lx allocated at %p, physical %08lx.\n",
230 start, p, __pa(p));
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700231
Benjamin Herrenschmidtbc033b62008-08-05 16:19:56 +1000232 mapped = htab_bolt_mapping(start, start + page_size, __pa(p),
David Gibsonf5ea64d2008-10-12 17:54:24 +0000233 pgprot_val(PAGE_KERNEL),
234 mmu_vmemmap_psize, mmu_kernel_ssize);
Andy Whitcroftd29eff72007-10-16 01:24:17 -0700235 BUG_ON(mapped < 0);
236 }
237
238 return 0;
239}
Benjamin Herrenschmidtcec08e72008-04-30 15:41:48 +1000240#endif /* CONFIG_SPARSEMEM_VMEMMAP */