blob: 8106bba41ecb4b7ce5b87a58545c50102cc562b7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/i386/mm/init.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
7 */
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/module.h>
10#include <linux/signal.h>
11#include <linux/sched.h>
12#include <linux/kernel.h>
13#include <linux/errno.h>
14#include <linux/string.h>
15#include <linux/types.h>
16#include <linux/ptrace.h>
17#include <linux/mman.h>
18#include <linux/mm.h>
19#include <linux/hugetlb.h>
20#include <linux/swap.h>
21#include <linux/smp.h>
22#include <linux/init.h>
23#include <linux/highmem.h>
24#include <linux/pagemap.h>
Jan Beulich6fb14752007-05-02 19:27:10 +020025#include <linux/pfn.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070026#include <linux/poison.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <linux/bootmem.h>
28#include <linux/slab.h>
29#include <linux/proc_fs.h>
Dave Hansen05039b92005-10-29 18:16:57 -070030#include <linux/memory_hotplug.h>
Adrian Bunk27d99f72005-11-13 16:06:51 -080031#include <linux/initrd.h>
Shaohua Li55b23552006-06-23 02:04:49 -070032#include <linux/cpumask.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
H. Peter Anvinf832ff12008-02-04 16:47:58 +010034#include <asm/asm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/processor.h>
36#include <asm/system.h>
37#include <asm/uaccess.h>
38#include <asm/pgtable.h>
39#include <asm/dma.h>
40#include <asm/fixmap.h>
41#include <asm/e820.h>
42#include <asm/apic.h>
Ingo Molnar8550eb92008-01-30 13:34:10 +010043#include <asm/bugs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <asm/tlb.h>
45#include <asm/tlbflush.h>
Jeremy Fitzhardingea5a19c62008-01-30 13:33:39 +010046#include <asm/pgalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#include <asm/sections.h>
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +020048#include <asm/paravirt.h>
Ian Campbell551889a2008-02-09 23:24:09 +010049#include <asm/setup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51unsigned int __VMALLOC_RESERVE = 128 << 20;
52
53DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
54unsigned long highstart_pfn, highend_pfn;
55
Ingo Molnar8550eb92008-01-30 13:34:10 +010056static noinline int do_test_wp_bit(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
58/*
59 * Creates a middle page table and puts a pointer to it in the
60 * given global directory entry. This only returns the gd entry
61 * in non-PAE compilation mode, since the middle layer is folded.
62 */
63static pmd_t * __init one_md_table_init(pgd_t *pgd)
64{
65 pud_t *pud;
66 pmd_t *pmd_table;
Ingo Molnar8550eb92008-01-30 13:34:10 +010067
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#ifdef CONFIG_X86_PAE
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +020069 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
70 pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
71
Jeremy Fitzhardinge6c435452008-01-30 13:33:39 +010072 paravirt_alloc_pd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +020073 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
74 pud = pud_offset(pgd, 0);
Ingo Molnar8550eb92008-01-30 13:34:10 +010075 BUG_ON(pmd_table != pmd_offset(pud, 0));
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +020076 }
77#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 pud = pud_offset(pgd, 0);
79 pmd_table = pmd_offset(pud, 0);
Ingo Molnar8550eb92008-01-30 13:34:10 +010080
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 return pmd_table;
82}
83
84/*
85 * Create a page table and place a pointer to it in a middle page
Ingo Molnar8550eb92008-01-30 13:34:10 +010086 * directory entry:
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 */
88static pte_t * __init one_page_table_init(pmd_t *pmd)
89{
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +020090 if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
Ingo Molnar509a80c2007-10-17 18:04:34 +020091 pte_t *page_table = NULL;
92
93#ifdef CONFIG_DEBUG_PAGEALLOC
94 page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
95#endif
Ingo Molnar8550eb92008-01-30 13:34:10 +010096 if (!page_table) {
Ingo Molnar509a80c2007-10-17 18:04:34 +020097 page_table =
98 (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
Ingo Molnar8550eb92008-01-30 13:34:10 +010099 }
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200100
Jeremy Fitzhardingefdb4c332007-07-17 18:37:03 -0700101 paravirt_alloc_pt(&init_mm, __pa(page_table) >> PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200103 BUG_ON(page_table != pte_offset_kernel(pmd, 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 }
Ingo Molnar509a80c2007-10-17 18:04:34 +0200105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 return pte_offset_kernel(pmd, 0);
107}
108
109/*
Ingo Molnar8550eb92008-01-30 13:34:10 +0100110 * This function initializes a certain range of kernel virtual memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 * with new bootmem page tables, everywhere page tables are missing in
112 * the given range.
Ingo Molnar8550eb92008-01-30 13:34:10 +0100113 *
114 * NOTE: The pagetables are allocated contiguous on the physical space
115 * so we can cache the place of the first one and move around without
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 * checking the pgd every time.
117 */
Ingo Molnar8550eb92008-01-30 13:34:10 +0100118static void __init
119page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121 int pgd_idx, pmd_idx;
122 unsigned long vaddr;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100123 pgd_t *pgd;
124 pmd_t *pmd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126 vaddr = start;
127 pgd_idx = pgd_index(vaddr);
128 pmd_idx = pmd_index(vaddr);
129 pgd = pgd_base + pgd_idx;
130
131 for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200132 pmd = one_md_table_init(pgd);
133 pmd = pmd + pmd_index(vaddr);
Ingo Molnar8550eb92008-01-30 13:34:10 +0100134 for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
135 pmd++, pmd_idx++) {
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200136 one_page_table_init(pmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138 vaddr += PMD_SIZE;
139 }
140 pmd_idx = 0;
141 }
142}
143
144static inline int is_kernel_text(unsigned long addr)
145{
146 if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
147 return 1;
148 return 0;
149}
150
151/*
Ingo Molnar8550eb92008-01-30 13:34:10 +0100152 * This maps the physical memory to kernel virtual address space, a total
153 * of max_low_pfn pages, by creating page tables starting from address
154 * PAGE_OFFSET:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 */
156static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
157{
Ingo Molnar8550eb92008-01-30 13:34:10 +0100158 int pgd_idx, pmd_idx, pte_ofs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 unsigned long pfn;
160 pgd_t *pgd;
161 pmd_t *pmd;
162 pte_t *pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
164 pgd_idx = pgd_index(PAGE_OFFSET);
165 pgd = pgd_base + pgd_idx;
166 pfn = 0;
167
168 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
169 pmd = one_md_table_init(pgd);
170 if (pfn >= max_low_pfn)
171 continue;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100172
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100173 for (pmd_idx = 0;
174 pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn;
175 pmd++, pmd_idx++) {
Ingo Molnar8550eb92008-01-30 13:34:10 +0100176 unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177
Ingo Molnar8550eb92008-01-30 13:34:10 +0100178 /*
179 * Map with big pages if possible, otherwise
180 * create normal page tables:
181 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 if (cpu_has_pse) {
Ingo Molnar8550eb92008-01-30 13:34:10 +0100183 unsigned int addr2;
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100184 pgprot_t prot = PAGE_KERNEL_LARGE;
185
Ingo Molnar8550eb92008-01-30 13:34:10 +0100186 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100187 PAGE_OFFSET + PAGE_SIZE-1;
188
Ingo Molnar8550eb92008-01-30 13:34:10 +0100189 if (is_kernel_text(addr) ||
190 is_kernel_text(addr2))
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100191 prot = PAGE_KERNEL_LARGE_EXEC;
192
193 set_pmd(pmd, pfn_pmd(pfn, prot));
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200194
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 pfn += PTRS_PER_PTE;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100196 continue;
197 }
198 pte = one_page_table_init(pmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199
Ingo Molnar8550eb92008-01-30 13:34:10 +0100200 for (pte_ofs = 0;
201 pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn;
202 pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) {
203 pgprot_t prot = PAGE_KERNEL;
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100204
Ingo Molnar8550eb92008-01-30 13:34:10 +0100205 if (is_kernel_text(addr))
206 prot = PAGE_KERNEL_EXEC;
Jeremy Fitzhardingef3f20de2008-01-30 13:31:09 +0100207
Ingo Molnar8550eb92008-01-30 13:34:10 +0100208 set_pte(pte, pfn_pte(pfn, prot));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 }
210 }
211 }
212}
213
214static inline int page_kills_ppro(unsigned long pagenr)
215{
216 if (pagenr >= 0x70000 && pagenr <= 0x7003F)
217 return 1;
218 return 0;
219}
220
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221#ifdef CONFIG_HIGHMEM
222pte_t *kmap_pte;
223pgprot_t kmap_prot;
224
Ingo Molnar8550eb92008-01-30 13:34:10 +0100225static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)
226{
227 return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
228 vaddr), vaddr), vaddr);
229}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
231static void __init kmap_init(void)
232{
233 unsigned long kmap_vstart;
234
Ingo Molnar8550eb92008-01-30 13:34:10 +0100235 /*
236 * Cache the first kmap pte:
237 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
239 kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
240
241 kmap_prot = PAGE_KERNEL;
242}
243
244static void __init permanent_kmaps_init(pgd_t *pgd_base)
245{
Ingo Molnar8550eb92008-01-30 13:34:10 +0100246 unsigned long vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 pgd_t *pgd;
248 pud_t *pud;
249 pmd_t *pmd;
250 pte_t *pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
252 vaddr = PKMAP_BASE;
253 page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
254
255 pgd = swapper_pg_dir + pgd_index(vaddr);
256 pud = pud_offset(pgd, vaddr);
257 pmd = pmd_offset(pud, vaddr);
258 pte = pte_offset_kernel(pmd, vaddr);
Ingo Molnar8550eb92008-01-30 13:34:10 +0100259 pkmap_page_table = pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260}
261
Matt Tolentinoc09b4242006-01-17 07:03:44 +0100262static void __meminit free_new_highpage(struct page *page)
Dave Hansen05039b92005-10-29 18:16:57 -0700263{
Nick Piggin7835e982006-03-22 00:08:40 -0800264 init_page_count(page);
Dave Hansen05039b92005-10-29 18:16:57 -0700265 __free_page(page);
266 totalhigh_pages++;
267}
268
269void __init add_one_highpage_init(struct page *page, int pfn, int bad_ppro)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270{
271 if (page_is_ram(pfn) && !(bad_ppro && page_kills_ppro(pfn))) {
272 ClearPageReserved(page);
Dave Hansen05039b92005-10-29 18:16:57 -0700273 free_new_highpage(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 } else
275 SetPageReserved(page);
276}
277
Ingo Molnar8550eb92008-01-30 13:34:10 +0100278static int __meminit
279add_one_highpage_hotplug(struct page *page, unsigned long pfn)
Dave Hansen05039b92005-10-29 18:16:57 -0700280{
281 free_new_highpage(page);
282 totalram_pages++;
283#ifdef CONFIG_FLATMEM
284 max_mapnr = max(pfn, max_mapnr);
285#endif
286 num_physpages++;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100287
Dave Hansen05039b92005-10-29 18:16:57 -0700288 return 0;
289}
290
291/*
292 * Not currently handling the NUMA case.
293 * Assuming single node and all memory that
294 * has been added dynamically that would be
Ingo Molnar8550eb92008-01-30 13:34:10 +0100295 * onlined here is in HIGHMEM.
Dave Hansen05039b92005-10-29 18:16:57 -0700296 */
Vivek Goyal0e0be252007-01-11 01:52:44 +0100297void __meminit online_page(struct page *page)
Dave Hansen05039b92005-10-29 18:16:57 -0700298{
299 ClearPageReserved(page);
300 add_one_highpage_hotplug(page, page_to_pfn(page));
301}
302
Ingo Molnar8550eb92008-01-30 13:34:10 +0100303#ifndef CONFIG_NUMA
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304static void __init set_highmem_pages_init(int bad_ppro)
305{
306 int pfn;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100307
Ingo Molnar23be8c72008-01-15 16:44:37 +0100308 for (pfn = highstart_pfn; pfn < highend_pfn; pfn++) {
309 /*
310 * Holes under sparsemem might not have no mem_map[]:
311 */
312 if (pfn_valid(pfn))
313 add_one_highpage_init(pfn_to_page(pfn), pfn, bad_ppro);
314 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315 totalram_pages += totalhigh_pages;
316}
Ingo Molnar8550eb92008-01-30 13:34:10 +0100317#endif /* !CONFIG_NUMA */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
319#else
Ingo Molnar8550eb92008-01-30 13:34:10 +0100320# define kmap_init() do { } while (0)
321# define permanent_kmaps_init(pgd_base) do { } while (0)
322# define set_highmem_pages_init(bad_ppro) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323#endif /* CONFIG_HIGHMEM */
324
Andi Kleenc93c82b2008-01-30 13:33:50 +0100325pteval_t __PAGE_KERNEL = _PAGE_KERNEL;
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700326EXPORT_SYMBOL(__PAGE_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Ingo Molnar8550eb92008-01-30 13:34:10 +0100328pteval_t __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200330void __init native_pagetable_setup_start(pgd_t *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331{
Ian Campbell551889a2008-02-09 23:24:09 +0100332 unsigned long pfn, va;
333 pgd_t *pgd;
334 pud_t *pud;
335 pmd_t *pmd;
336 pte_t *pte;
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200337
338 /*
Ian Campbell551889a2008-02-09 23:24:09 +0100339 * Remove any mappings which extend past the end of physical
340 * memory from the boot time page table:
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200341 */
Ian Campbell551889a2008-02-09 23:24:09 +0100342 for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
343 va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
344 pgd = base + pgd_index(va);
345 if (!pgd_present(*pgd))
346 break;
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200347
Ian Campbell551889a2008-02-09 23:24:09 +0100348 pud = pud_offset(pgd, va);
349 pmd = pmd_offset(pud, va);
350 if (!pmd_present(*pmd))
351 break;
352
353 pte = pte_offset_kernel(pmd, va);
354 if (!pte_present(*pte))
355 break;
356
357 pte_clear(NULL, va, pte);
358 }
Jeremy Fitzhardinge6c435452008-01-30 13:33:39 +0100359 paravirt_alloc_pd(&init_mm, __pa(base) >> PAGE_SHIFT);
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200360}
361
362void __init native_pagetable_setup_done(pgd_t *base)
363{
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200364}
365
366/*
367 * Build a proper pagetable for the kernel mappings. Up until this
368 * point, we've been running on some set of pagetables constructed by
369 * the boot process.
370 *
371 * If we're booting on native hardware, this will be a pagetable
Ian Campbell551889a2008-02-09 23:24:09 +0100372 * constructed in arch/x86/kernel/head_32.S. The root of the
373 * pagetable will be swapper_pg_dir.
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200374 *
375 * If we're booting paravirtualized under a hypervisor, then there are
376 * more options: we may already be running PAE, and the pagetable may
377 * or may not be based in swapper_pg_dir. In any case,
378 * paravirt_pagetable_setup_start() will set up swapper_pg_dir
379 * appropriately for the rest of the initialization to work.
380 *
381 * In general, pagetable_init() assumes that the pagetable may already
382 * be partially populated, and so it avoids stomping on any existing
383 * mappings.
384 */
Ingo Molnar8550eb92008-01-30 13:34:10 +0100385static void __init pagetable_init(void)
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200386{
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200387 pgd_t *pgd_base = swapper_pg_dir;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100388 unsigned long vaddr, end;
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200389
390 paravirt_pagetable_setup_start(pgd_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
392 /* Enable PSE if available */
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200393 if (cpu_has_pse)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 set_in_cr4(X86_CR4_PSE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
396 /* Enable PGE if available */
397 if (cpu_has_pge) {
398 set_in_cr4(X86_CR4_PGE);
399 __PAGE_KERNEL |= _PAGE_GLOBAL;
400 __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL;
401 }
402
403 kernel_physical_mapping_init(pgd_base);
404 remap_numa_kva();
405
406 /*
407 * Fixed mappings, only the page table structure has to be
408 * created - mappings will be set by set_fixmap():
409 */
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100410 early_ioremap_clear();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200412 end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
413 page_table_range_init(vaddr, end, pgd_base);
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100414 early_ioremap_reset();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
416 permanent_kmaps_init(pgd_base);
417
Jeremy Fitzhardingeb239fb22007-05-02 19:27:13 +0200418 paravirt_pagetable_setup_done(pgd_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419}
420
Rafael J. Wysockia6eb84b2008-02-01 15:28:16 +0100421#ifdef CONFIG_ACPI_SLEEP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422/*
Rafael J. Wysockia6eb84b2008-02-01 15:28:16 +0100423 * ACPI suspend needs this for resume, because things like the intel-agp
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 * driver might have split up a kernel 4MB mapping.
425 */
Rafael J. Wysockia6eb84b2008-02-01 15:28:16 +0100426char swsusp_pg_dir[PAGE_SIZE]
Ingo Molnar8550eb92008-01-30 13:34:10 +0100427 __attribute__ ((aligned(PAGE_SIZE)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428
429static inline void save_pg_dir(void)
430{
431 memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
432}
Rafael J. Wysockia6eb84b2008-02-01 15:28:16 +0100433#else /* !CONFIG_ACPI_SLEEP */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434static inline void save_pg_dir(void)
435{
436}
Rafael J. Wysockia6eb84b2008-02-01 15:28:16 +0100437#endif /* !CONFIG_ACPI_SLEEP */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
Ingo Molnar8550eb92008-01-30 13:34:10 +0100439void zap_low_mappings(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440{
441 int i;
442
443 save_pg_dir();
444
445 /*
446 * Zap initial low-memory mappings.
447 *
448 * Note that "pgd_clear()" doesn't do it for
449 * us, because pgd_clear() is a no-op on i386.
450 */
Ingo Molnar8550eb92008-01-30 13:34:10 +0100451 for (i = 0; i < USER_PTRS_PER_PGD; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452#ifdef CONFIG_X86_PAE
453 set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
454#else
455 set_pgd(swapper_pg_dir+i, __pgd(0));
456#endif
Ingo Molnar8550eb92008-01-30 13:34:10 +0100457 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 flush_tlb_all();
459}
460
Ingo Molnar8550eb92008-01-30 13:34:10 +0100461int nx_enabled;
Jan Beulichd5321ab2007-07-21 17:10:26 +0200462
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100463pteval_t __supported_pte_mask __read_mostly = ~_PAGE_NX;
464EXPORT_SYMBOL_GPL(__supported_pte_mask);
465
Jan Beulichd5321ab2007-07-21 17:10:26 +0200466#ifdef CONFIG_X86_PAE
467
Ingo Molnar8550eb92008-01-30 13:34:10 +0100468static int disable_nx __initdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
470/*
471 * noexec = on|off
472 *
473 * Control non executable mappings.
474 *
475 * on Enable
476 * off Disable
477 */
Rusty Russell1a3f2392006-09-26 10:52:32 +0200478static int __init noexec_setup(char *str)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479{
Rusty Russell1a3f2392006-09-26 10:52:32 +0200480 if (!str || !strcmp(str, "on")) {
481 if (cpu_has_nx) {
482 __supported_pte_mask |= _PAGE_NX;
483 disable_nx = 0;
484 }
Ingo Molnar8550eb92008-01-30 13:34:10 +0100485 } else {
486 if (!strcmp(str, "off")) {
487 disable_nx = 1;
488 __supported_pte_mask &= ~_PAGE_NX;
489 } else {
490 return -EINVAL;
491 }
492 }
Rusty Russell1a3f2392006-09-26 10:52:32 +0200493
494 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495}
Rusty Russell1a3f2392006-09-26 10:52:32 +0200496early_param("noexec", noexec_setup);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498static void __init set_nx(void)
499{
500 unsigned int v[4], l, h;
501
502 if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
503 cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
Ingo Molnar8550eb92008-01-30 13:34:10 +0100504
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 if ((v[3] & (1 << 20)) && !disable_nx) {
506 rdmsr(MSR_EFER, l, h);
507 l |= EFER_NX;
508 wrmsr(MSR_EFER, l, h);
509 nx_enabled = 1;
510 __supported_pte_mask |= _PAGE_NX;
511 }
512 }
513}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514#endif
515
516/*
517 * paging_init() sets up the page tables - note that the first 8MB are
518 * already mapped by head.S.
519 *
520 * This routines also unmaps the page at virtual kernel address 0, so
521 * that we can trap those pesky NULL-reference errors in the kernel.
522 */
523void __init paging_init(void)
524{
525#ifdef CONFIG_X86_PAE
526 set_nx();
527 if (nx_enabled)
Ingo Molnard7d119d2008-01-30 13:34:10 +0100528 printk(KERN_INFO "NX (Execute Disable) protection: active\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 pagetable_init();
531
532 load_cr3(swapper_pg_dir);
533
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 __flush_tlb_all();
535
536 kmap_init();
537}
538
539/*
540 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
541 * and also on some strange 486's (NexGen etc.). All 586+'s are OK. This
542 * used to involve black magic jumps to work around some nasty CPU bugs,
543 * but fortunately the switch to using exceptions got rid of all that.
544 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545static void __init test_wp_bit(void)
546{
Ingo Molnard7d119d2008-01-30 13:34:10 +0100547 printk(KERN_INFO
548 "Checking if this processor honours the WP bit even in supervisor mode...");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549
550 /* Any page-aligned address will do, the test is non-destructive */
551 __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
552 boot_cpu_data.wp_works_ok = do_test_wp_bit();
553 clear_fixmap(FIX_WP_TEST);
554
555 if (!boot_cpu_data.wp_works_ok) {
Ingo Molnard7d119d2008-01-30 13:34:10 +0100556 printk(KERN_CONT "No.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557#ifdef CONFIG_X86_WP_WORKS_OK
Ingo Molnard7d119d2008-01-30 13:34:10 +0100558 panic(
559 "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560#endif
561 } else {
Ingo Molnard7d119d2008-01-30 13:34:10 +0100562 printk(KERN_CONT "Ok.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 }
564}
565
Ingo Molnar8550eb92008-01-30 13:34:10 +0100566static struct kcore_list kcore_mem, kcore_vmalloc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567
568void __init mem_init(void)
569{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 int codesize, reservedpages, datasize, initsize;
Ingo Molnar8550eb92008-01-30 13:34:10 +0100571 int tmp, bad_ppro;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572
Andy Whitcroft05b79bd2005-06-23 00:07:57 -0700573#ifdef CONFIG_FLATMEM
Eric Sesterhenn8d8f3cb2006-10-03 23:34:58 +0200574 BUG_ON(!mem_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 bad_ppro = ppro_with_ram_bug();
577
578#ifdef CONFIG_HIGHMEM
579 /* check that fixmap and pkmap do not overlap */
Ingo Molnard7d119d2008-01-30 13:34:10 +0100580 if (PKMAP_BASE + LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
581 printk(KERN_ERR
582 "fixmap and kmap areas overlap - this will crash\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 printk(KERN_ERR "pkstart: %lxh pkend: %lxh fixstart %lxh\n",
Ingo Molnard7d119d2008-01-30 13:34:10 +0100584 PKMAP_BASE, PKMAP_BASE + LAST_PKMAP*PAGE_SIZE,
585 FIXADDR_START);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 BUG();
587 }
588#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589 /* this will put all low memory onto the freelists */
590 totalram_pages += free_all_bootmem();
591
592 reservedpages = 0;
593 for (tmp = 0; tmp < max_low_pfn; tmp++)
594 /*
Ingo Molnar8550eb92008-01-30 13:34:10 +0100595 * Only count reserved RAM pages:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 */
597 if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
598 reservedpages++;
599
600 set_highmem_pages_init(bad_ppro);
601
602 codesize = (unsigned long) &_etext - (unsigned long) &_text;
603 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
604 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
605
Ingo Molnar8550eb92008-01-30 13:34:10 +0100606 kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
607 kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 VMALLOC_END-VMALLOC_START);
609
Ingo Molnar8550eb92008-01-30 13:34:10 +0100610 printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, "
611 "%dk reserved, %dk data, %dk init, %ldk highmem)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
613 num_physpages << (PAGE_SHIFT-10),
614 codesize >> 10,
615 reservedpages << (PAGE_SHIFT-10),
616 datasize >> 10,
617 initsize >> 10,
618 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))
619 );
620
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700621#if 1 /* double-sanity-check paranoia */
Ingo Molnard7d119d2008-01-30 13:34:10 +0100622 printk(KERN_INFO "virtual kernel memory layout:\n"
Ingo Molnar8550eb92008-01-30 13:34:10 +0100623 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700624#ifdef CONFIG_HIGHMEM
Ingo Molnar8550eb92008-01-30 13:34:10 +0100625 " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700626#endif
Ingo Molnar8550eb92008-01-30 13:34:10 +0100627 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
628 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
629 " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
630 " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
631 " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
632 FIXADDR_START, FIXADDR_TOP,
633 (FIXADDR_TOP - FIXADDR_START) >> 10,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700634
635#ifdef CONFIG_HIGHMEM
Ingo Molnar8550eb92008-01-30 13:34:10 +0100636 PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
637 (LAST_PKMAP*PAGE_SIZE) >> 10,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700638#endif
639
Ingo Molnar8550eb92008-01-30 13:34:10 +0100640 VMALLOC_START, VMALLOC_END,
641 (VMALLOC_END - VMALLOC_START) >> 20,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700642
Ingo Molnar8550eb92008-01-30 13:34:10 +0100643 (unsigned long)__va(0), (unsigned long)high_memory,
644 ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700645
Ingo Molnar8550eb92008-01-30 13:34:10 +0100646 (unsigned long)&__init_begin, (unsigned long)&__init_end,
647 ((unsigned long)&__init_end -
648 (unsigned long)&__init_begin) >> 10,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700649
Ingo Molnar8550eb92008-01-30 13:34:10 +0100650 (unsigned long)&_etext, (unsigned long)&_edata,
651 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700652
Ingo Molnar8550eb92008-01-30 13:34:10 +0100653 (unsigned long)&_text, (unsigned long)&_etext,
654 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700655
656#ifdef CONFIG_HIGHMEM
Ingo Molnar8550eb92008-01-30 13:34:10 +0100657 BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START);
658 BUG_ON(VMALLOC_END > PKMAP_BASE);
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700659#endif
Ingo Molnar8550eb92008-01-30 13:34:10 +0100660 BUG_ON(VMALLOC_START > VMALLOC_END);
661 BUG_ON((unsigned long)high_memory > VMALLOC_START);
Jeremy Fitzhardinge052e7992006-09-25 23:32:25 -0700662#endif /* double-sanity-check paranoia */
663
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 if (boot_cpu_data.wp_works_ok < 0)
665 test_wp_bit();
666
Thomas Gleixner76ebd052008-02-09 23:24:09 +0100667 cpa_init();
668
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 /*
670 * Subtle. SMP is doing it's boot stuff late (because it has to
671 * fork idle threads) - but it also needs low mappings for the
672 * protected-mode entry to work. We zap these entries only after
673 * the WP-bit has been tested.
674 */
675#ifndef CONFIG_SMP
676 zap_low_mappings();
677#endif
678}
679
KAMEZAWA Hiroyukiad8f5792006-05-20 15:00:03 -0700680#ifdef CONFIG_MEMORY_HOTPLUG
Yasunori Gotobc02af92006-06-27 02:53:30 -0700681int arch_add_memory(int nid, u64 start, u64 size)
Dave Hansen05039b92005-10-29 18:16:57 -0700682{
Yasunori Goto7c7e9422006-12-22 01:11:13 -0800683 struct pglist_data *pgdata = NODE_DATA(nid);
Christoph Lameter776ed982006-09-25 23:31:09 -0700684 struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
Dave Hansen05039b92005-10-29 18:16:57 -0700685 unsigned long start_pfn = start >> PAGE_SHIFT;
686 unsigned long nr_pages = size >> PAGE_SHIFT;
687
688 return __add_pages(zone, start_pfn, nr_pages);
689}
Andi Kleen9d99aaa2006-04-07 19:49:15 +0200690#endif
Dave Hansen05039b92005-10-29 18:16:57 -0700691
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692/*
693 * This function cannot be __init, since exceptions don't work in that
694 * section. Put this after the callers, so that it cannot be inlined.
695 */
Ingo Molnar8550eb92008-01-30 13:34:10 +0100696static noinline int do_test_wp_bit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
698 char tmp_reg;
699 int flag;
700
701 __asm__ __volatile__(
Ingo Molnar8550eb92008-01-30 13:34:10 +0100702 " movb %0, %1 \n"
703 "1: movb %1, %0 \n"
704 " xorl %2, %2 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 "2: \n"
H. Peter Anvinf832ff12008-02-04 16:47:58 +0100706 _ASM_EXTABLE(1b,2b)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
708 "=q" (tmp_reg),
709 "=r" (flag)
710 :"2" (1)
711 :"memory");
Ingo Molnar8550eb92008-01-30 13:34:10 +0100712
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 return flag;
714}
715
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800716#ifdef CONFIG_DEBUG_RODATA
Arjan van de Venedeed302008-01-30 13:34:08 +0100717const int rodata_test_data = 0xC3;
718EXPORT_SYMBOL_GPL(rodata_test_data);
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800719
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800720void mark_rodata_ro(void)
721{
Jan Beulich6fb14752007-05-02 19:27:10 +0200722 unsigned long start = PFN_ALIGN(_text);
723 unsigned long size = PFN_ALIGN(_etext) - start;
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800724
Linus Torvalds602033e2007-07-26 12:07:21 -0700725#ifndef CONFIG_KPROBES
726#ifdef CONFIG_HOTPLUG_CPU
727 /* It must still be possible to apply SMP alternatives. */
728 if (num_possible_cpus() <= 1)
729#endif
730 {
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100731 set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
Ingo Molnard7d119d2008-01-30 13:34:10 +0100732 printk(KERN_INFO "Write protecting the kernel text: %luk\n",
733 size >> 10);
Andi Kleen0c42f392008-01-30 13:33:42 +0100734
735#ifdef CONFIG_CPA_DEBUG
Ingo Molnard7d119d2008-01-30 13:34:10 +0100736 printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n",
737 start, start+size);
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100738 set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT);
Andi Kleen0c42f392008-01-30 13:33:42 +0100739
Ingo Molnard7d119d2008-01-30 13:34:10 +0100740 printk(KERN_INFO "Testing CPA: write protecting again\n");
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100741 set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT);
Andi Kleen0c42f392008-01-30 13:33:42 +0100742#endif
Linus Torvalds602033e2007-07-26 12:07:21 -0700743 }
744#endif
Jan Beulich6fb14752007-05-02 19:27:10 +0200745 start += size;
746 size = (unsigned long)__end_rodata - start;
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100747 set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
Ingo Molnard7d119d2008-01-30 13:34:10 +0100748 printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
749 size >> 10);
Arjan van de Venedeed302008-01-30 13:34:08 +0100750 rodata_test();
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800751
Andi Kleen0c42f392008-01-30 13:33:42 +0100752#ifdef CONFIG_CPA_DEBUG
Ingo Molnard7d119d2008-01-30 13:34:10 +0100753 printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size);
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100754 set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT);
Andi Kleen0c42f392008-01-30 13:33:42 +0100755
Ingo Molnard7d119d2008-01-30 13:34:10 +0100756 printk(KERN_INFO "Testing CPA: write protecting again\n");
Arjan van de Ven6d238cc2008-01-30 13:34:06 +0100757 set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT);
Andi Kleen0c42f392008-01-30 13:33:42 +0100758#endif
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800759}
760#endif
761
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800762void free_init_pages(char *what, unsigned long begin, unsigned long end)
763{
Ingo Molnaree01f112008-01-30 13:34:09 +0100764#ifdef CONFIG_DEBUG_PAGEALLOC
765 /*
766 * If debugging page accesses then do not free this memory but
767 * mark them not present - any buggy init-section access will
768 * create a kernel page fault:
769 */
770 printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n",
771 begin, PAGE_ALIGN(end));
772 set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
773#else
Ingo Molnar86f03982008-01-30 13:34:09 +0100774 unsigned long addr;
775
Arjan van de Ven3c1df682008-01-30 13:34:07 +0100776 /*
777 * We just marked the kernel text read only above, now that
778 * we are going to free part of that, we need to make that
779 * writeable first.
780 */
781 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
782
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800783 for (addr = begin; addr < end; addr += PAGE_SIZE) {
Linus Torvaldse3ebadd2007-05-07 08:44:24 -0700784 ClearPageReserved(virt_to_page(addr));
785 init_page_count(virt_to_page(addr));
786 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
787 free_page(addr);
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800788 totalram_pages++;
789 }
Jan Beulich6fb14752007-05-02 19:27:10 +0200790 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
Ingo Molnaree01f112008-01-30 13:34:09 +0100791#endif
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800792}
793
794void free_initmem(void)
795{
796 free_init_pages("unused kernel memory",
Linus Torvaldse3ebadd2007-05-07 08:44:24 -0700797 (unsigned long)(&__init_begin),
798 (unsigned long)(&__init_end));
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800799}
Arjan van de Ven63aaf302006-01-06 00:12:02 -0800800
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801#ifdef CONFIG_BLK_DEV_INITRD
802void free_initrd_mem(unsigned long start, unsigned long end)
803{
Linus Torvaldse3ebadd2007-05-07 08:44:24 -0700804 free_init_pages("initrd memory", start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805}
806#endif