Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * |
| 3 | * Copyright (C) 1995 Linus Torvalds |
| 4 | * |
| 5 | * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 |
| 6 | */ |
| 7 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/module.h> |
| 9 | #include <linux/signal.h> |
| 10 | #include <linux/sched.h> |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/errno.h> |
| 13 | #include <linux/string.h> |
| 14 | #include <linux/types.h> |
| 15 | #include <linux/ptrace.h> |
| 16 | #include <linux/mman.h> |
| 17 | #include <linux/mm.h> |
| 18 | #include <linux/hugetlb.h> |
| 19 | #include <linux/swap.h> |
| 20 | #include <linux/smp.h> |
| 21 | #include <linux/init.h> |
| 22 | #include <linux/highmem.h> |
| 23 | #include <linux/pagemap.h> |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 24 | #include <linux/pfn.h> |
Randy Dunlap | c9cf552 | 2006-06-27 02:53:52 -0700 | [diff] [blame] | 25 | #include <linux/poison.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/bootmem.h> |
| 27 | #include <linux/slab.h> |
| 28 | #include <linux/proc_fs.h> |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 29 | #include <linux/memory_hotplug.h> |
Adrian Bunk | 27d99f7 | 2005-11-13 16:06:51 -0800 | [diff] [blame] | 30 | #include <linux/initrd.h> |
Shaohua Li | 55b2355 | 2006-06-23 02:04:49 -0700 | [diff] [blame] | 31 | #include <linux/cpumask.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | |
H. Peter Anvin | f832ff1 | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 33 | #include <asm/asm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | #include <asm/processor.h> |
| 35 | #include <asm/system.h> |
| 36 | #include <asm/uaccess.h> |
| 37 | #include <asm/pgtable.h> |
| 38 | #include <asm/dma.h> |
| 39 | #include <asm/fixmap.h> |
| 40 | #include <asm/e820.h> |
| 41 | #include <asm/apic.h> |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 42 | #include <asm/bugs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | #include <asm/tlb.h> |
| 44 | #include <asm/tlbflush.h> |
Jeremy Fitzhardinge | a5a19c6 | 2008-01-30 13:33:39 +0100 | [diff] [blame] | 45 | #include <asm/pgalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | #include <asm/sections.h> |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 47 | #include <asm/paravirt.h> |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 48 | #include <asm/setup.h> |
Harvey Harrison | 7bfeab9 | 2008-02-12 12:12:01 -0800 | [diff] [blame] | 49 | #include <asm/cacheflush.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 50 | |
| 51 | unsigned int __VMALLOC_RESERVE = 128 << 20; |
| 52 | |
Thomas Gleixner | 6779429 | 2008-03-21 21:27:10 +0100 | [diff] [blame] | 53 | unsigned long max_pfn_mapped; |
Andi Kleen | 7d1116a | 2008-03-12 03:53:27 +0100 | [diff] [blame] | 54 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); |
| 56 | unsigned long highstart_pfn, highend_pfn; |
| 57 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 58 | static noinline int do_test_wp_bit(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 60 | |
| 61 | static unsigned long __initdata table_start; |
| 62 | static unsigned long __meminitdata table_end; |
| 63 | static unsigned long __meminitdata table_top; |
| 64 | |
| 65 | static int __initdata after_init_bootmem; |
| 66 | |
| 67 | static __init void *alloc_low_page(unsigned long *phys) |
| 68 | { |
| 69 | unsigned long pfn = table_end++; |
| 70 | void *adr; |
| 71 | |
| 72 | if (pfn >= table_top) |
| 73 | panic("alloc_low_page: ran out of memory"); |
| 74 | |
| 75 | adr = __va(pfn * PAGE_SIZE); |
| 76 | memset(adr, 0, PAGE_SIZE); |
| 77 | *phys = pfn * PAGE_SIZE; |
| 78 | return adr; |
| 79 | } |
| 80 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 81 | /* |
| 82 | * Creates a middle page table and puts a pointer to it in the |
| 83 | * given global directory entry. This only returns the gd entry |
| 84 | * in non-PAE compilation mode, since the middle layer is folded. |
| 85 | */ |
| 86 | static pmd_t * __init one_md_table_init(pgd_t *pgd) |
| 87 | { |
| 88 | pud_t *pud; |
| 89 | pmd_t *pmd_table; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 90 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | #ifdef CONFIG_X86_PAE |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 92 | unsigned long phys; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 93 | if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 94 | if (after_init_bootmem) |
| 95 | pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
| 96 | else |
| 97 | pmd_table = (pmd_t *)alloc_low_page(&phys); |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 98 | paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 99 | set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); |
| 100 | pud = pud_offset(pgd, 0); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 101 | BUG_ON(pmd_table != pmd_offset(pud, 0)); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 102 | } |
| 103 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | pud = pud_offset(pgd, 0); |
| 105 | pmd_table = pmd_offset(pud, 0); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 106 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | return pmd_table; |
| 108 | } |
| 109 | |
| 110 | /* |
| 111 | * Create a page table and place a pointer to it in a middle page |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 112 | * directory entry: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 113 | */ |
| 114 | static pte_t * __init one_page_table_init(pmd_t *pmd) |
| 115 | { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 116 | if (!(pmd_val(*pmd) & _PAGE_PRESENT)) { |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 117 | pte_t *page_table = NULL; |
| 118 | |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 119 | if (after_init_bootmem) { |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 120 | #ifdef CONFIG_DEBUG_PAGEALLOC |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 121 | page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE); |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 122 | #endif |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 123 | if (!page_table) |
| 124 | page_table = |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 125 | (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 126 | } else { |
| 127 | unsigned long phys; |
| 128 | page_table = (pte_t *)alloc_low_page(&phys); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 129 | } |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 130 | |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 131 | paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 133 | BUG_ON(page_table != pte_offset_kernel(pmd, 0)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | } |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 135 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | return pte_offset_kernel(pmd, 0); |
| 137 | } |
| 138 | |
| 139 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 140 | * This function initializes a certain range of kernel virtual memory |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | * with new bootmem page tables, everywhere page tables are missing in |
| 142 | * the given range. |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 143 | * |
| 144 | * NOTE: The pagetables are allocated contiguous on the physical space |
| 145 | * so we can cache the place of the first one and move around without |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | * checking the pgd every time. |
| 147 | */ |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 148 | static void __init |
| 149 | page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | int pgd_idx, pmd_idx; |
| 152 | unsigned long vaddr; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 153 | pgd_t *pgd; |
| 154 | pmd_t *pmd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | |
| 156 | vaddr = start; |
| 157 | pgd_idx = pgd_index(vaddr); |
| 158 | pmd_idx = pmd_index(vaddr); |
| 159 | pgd = pgd_base + pgd_idx; |
| 160 | |
| 161 | for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 162 | pmd = one_md_table_init(pgd); |
| 163 | pmd = pmd + pmd_index(vaddr); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 164 | for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); |
| 165 | pmd++, pmd_idx++) { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 166 | one_page_table_init(pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 167 | |
| 168 | vaddr += PMD_SIZE; |
| 169 | } |
| 170 | pmd_idx = 0; |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | static inline int is_kernel_text(unsigned long addr) |
| 175 | { |
| 176 | if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end) |
| 177 | return 1; |
| 178 | return 0; |
| 179 | } |
| 180 | |
| 181 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 182 | * This maps the physical memory to kernel virtual address space, a total |
| 183 | * of max_low_pfn pages, by creating page tables starting from address |
| 184 | * PAGE_OFFSET: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 185 | */ |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 186 | static void __init kernel_physical_mapping_init(pgd_t *pgd_base, |
| 187 | unsigned long start, |
| 188 | unsigned long end) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 189 | { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 190 | int pgd_idx, pmd_idx, pte_ofs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | unsigned long pfn; |
| 192 | pgd_t *pgd; |
| 193 | pmd_t *pmd; |
| 194 | pte_t *pte; |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 195 | unsigned pages_2m = 0, pages_4k = 0; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 196 | unsigned limit_pfn = end >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | |
| 198 | pgd_idx = pgd_index(PAGE_OFFSET); |
| 199 | pgd = pgd_base + pgd_idx; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 200 | pfn = start >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 201 | |
| 202 | for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { |
| 203 | pmd = one_md_table_init(pgd); |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 204 | if (pfn >= limit_pfn) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | continue; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 206 | |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 207 | for (pmd_idx = 0; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 208 | pmd_idx < PTRS_PER_PMD && pfn < limit_pfn; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 209 | pmd++, pmd_idx++) { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 210 | unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 212 | /* |
| 213 | * Map with big pages if possible, otherwise |
| 214 | * create normal page tables: |
Andi Kleen | f5c24a7 | 2008-03-12 03:53:30 +0100 | [diff] [blame] | 215 | * |
| 216 | * Don't use a large page for the first 2/4MB of memory |
| 217 | * because there are often fixed size MTRRs in there |
| 218 | * and overlapping MTRRs into large pages can cause |
| 219 | * slowdowns. |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 220 | */ |
Andi Kleen | f5c24a7 | 2008-03-12 03:53:30 +0100 | [diff] [blame] | 221 | if (cpu_has_pse && !(pgd_idx == 0 && pmd_idx == 0)) { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 222 | unsigned int addr2; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 223 | pgprot_t prot = PAGE_KERNEL_LARGE; |
| 224 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 225 | addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE + |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 226 | PAGE_OFFSET + PAGE_SIZE-1; |
| 227 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 228 | if (is_kernel_text(addr) || |
| 229 | is_kernel_text(addr2)) |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 230 | prot = PAGE_KERNEL_LARGE_EXEC; |
| 231 | |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 232 | pages_2m++; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 233 | set_pmd(pmd, pfn_pmd(pfn, prot)); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 234 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | pfn += PTRS_PER_PTE; |
Thomas Gleixner | 6779429 | 2008-03-21 21:27:10 +0100 | [diff] [blame] | 236 | max_pfn_mapped = pfn; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 237 | continue; |
| 238 | } |
| 239 | pte = one_page_table_init(pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 240 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 241 | for (pte_ofs = 0; |
| 242 | pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn; |
| 243 | pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) { |
| 244 | pgprot_t prot = PAGE_KERNEL; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 245 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 246 | if (is_kernel_text(addr)) |
| 247 | prot = PAGE_KERNEL_EXEC; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 248 | |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 249 | pages_4k++; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 250 | set_pte(pte, pfn_pte(pfn, prot)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | } |
Thomas Gleixner | 6779429 | 2008-03-21 21:27:10 +0100 | [diff] [blame] | 252 | max_pfn_mapped = pfn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 253 | } |
| 254 | } |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 255 | update_page_count(PG_LEVEL_2M, pages_2m); |
| 256 | update_page_count(PG_LEVEL_4K, pages_4k); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 257 | } |
| 258 | |
Arjan van de Ven | ae531c2 | 2008-04-24 23:40:47 +0200 | [diff] [blame] | 259 | /* |
| 260 | * devmem_is_allowed() checks to see if /dev/mem access to a certain address |
| 261 | * is valid. The argument is a physical page number. |
| 262 | * |
| 263 | * |
| 264 | * On x86, access has to be given to the first megabyte of ram because that area |
| 265 | * contains bios code and data regions used by X and dosemu and similar apps. |
| 266 | * Access has to be given to non-kernel-ram areas as well, these contain the PCI |
| 267 | * mmio resources as well as potential bios/acpi data regions. |
| 268 | */ |
| 269 | int devmem_is_allowed(unsigned long pagenr) |
| 270 | { |
| 271 | if (pagenr <= 256) |
| 272 | return 1; |
| 273 | if (!page_is_ram(pagenr)) |
| 274 | return 1; |
| 275 | return 0; |
| 276 | } |
| 277 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 278 | #ifdef CONFIG_HIGHMEM |
| 279 | pte_t *kmap_pte; |
| 280 | pgprot_t kmap_prot; |
| 281 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 282 | static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr) |
| 283 | { |
| 284 | return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), |
| 285 | vaddr), vaddr), vaddr); |
| 286 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 287 | |
| 288 | static void __init kmap_init(void) |
| 289 | { |
| 290 | unsigned long kmap_vstart; |
| 291 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 292 | /* |
| 293 | * Cache the first kmap pte: |
| 294 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 295 | kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); |
| 296 | kmap_pte = kmap_get_fixmap_pte(kmap_vstart); |
| 297 | |
| 298 | kmap_prot = PAGE_KERNEL; |
| 299 | } |
| 300 | |
| 301 | static void __init permanent_kmaps_init(pgd_t *pgd_base) |
| 302 | { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 303 | unsigned long vaddr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 304 | pgd_t *pgd; |
| 305 | pud_t *pud; |
| 306 | pmd_t *pmd; |
| 307 | pte_t *pte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | |
| 309 | vaddr = PKMAP_BASE; |
| 310 | page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); |
| 311 | |
| 312 | pgd = swapper_pg_dir + pgd_index(vaddr); |
| 313 | pud = pud_offset(pgd, vaddr); |
| 314 | pmd = pmd_offset(pud, vaddr); |
| 315 | pte = pte_offset_kernel(pmd, vaddr); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 316 | pkmap_page_table = pte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 317 | } |
| 318 | |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 319 | static void __init add_one_highpage_init(struct page *page, int pfn) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | { |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 321 | ClearPageReserved(page); |
| 322 | init_page_count(page); |
| 323 | __free_page(page); |
| 324 | totalhigh_pages++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 325 | } |
| 326 | |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 327 | struct add_highpages_data { |
| 328 | unsigned long start_pfn; |
| 329 | unsigned long end_pfn; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 330 | }; |
| 331 | |
Yinghai Lu | d52d53b | 2008-06-16 20:10:55 -0700 | [diff] [blame] | 332 | static int __init add_highpages_work_fn(unsigned long start_pfn, |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 333 | unsigned long end_pfn, void *datax) |
| 334 | { |
| 335 | int node_pfn; |
| 336 | struct page *page; |
| 337 | unsigned long final_start_pfn, final_end_pfn; |
| 338 | struct add_highpages_data *data; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 339 | |
| 340 | data = (struct add_highpages_data *)datax; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 341 | |
| 342 | final_start_pfn = max(start_pfn, data->start_pfn); |
| 343 | final_end_pfn = min(end_pfn, data->end_pfn); |
| 344 | if (final_start_pfn >= final_end_pfn) |
Yinghai Lu | d52d53b | 2008-06-16 20:10:55 -0700 | [diff] [blame] | 345 | return 0; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 346 | |
| 347 | for (node_pfn = final_start_pfn; node_pfn < final_end_pfn; |
| 348 | node_pfn++) { |
| 349 | if (!pfn_valid(node_pfn)) |
| 350 | continue; |
| 351 | page = pfn_to_page(node_pfn); |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 352 | add_one_highpage_init(page, node_pfn); |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 353 | } |
| 354 | |
Yinghai Lu | d52d53b | 2008-06-16 20:10:55 -0700 | [diff] [blame] | 355 | return 0; |
| 356 | |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 357 | } |
| 358 | |
| 359 | void __init add_highpages_with_active_regions(int nid, unsigned long start_pfn, |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 360 | unsigned long end_pfn) |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 361 | { |
| 362 | struct add_highpages_data data; |
| 363 | |
| 364 | data.start_pfn = start_pfn; |
| 365 | data.end_pfn = end_pfn; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 366 | |
| 367 | work_with_active_regions(nid, add_highpages_work_fn, &data); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 368 | } |
| 369 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 370 | #ifndef CONFIG_NUMA |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 371 | static void __init set_highmem_pages_init(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 372 | { |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 373 | add_highpages_with_active_regions(0, highstart_pfn, highend_pfn); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 374 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 375 | totalram_pages += totalhigh_pages; |
| 376 | } |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 377 | #endif /* !CONFIG_NUMA */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 378 | |
| 379 | #else |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 380 | # define kmap_init() do { } while (0) |
| 381 | # define permanent_kmaps_init(pgd_base) do { } while (0) |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 382 | # define set_highmem_pages_init() do { } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | #endif /* CONFIG_HIGHMEM */ |
| 384 | |
Andi Kleen | c93c82b | 2008-01-30 13:33:50 +0100 | [diff] [blame] | 385 | pteval_t __PAGE_KERNEL = _PAGE_KERNEL; |
Alexey Dobriyan | 129f694 | 2005-06-23 00:08:33 -0700 | [diff] [blame] | 386 | EXPORT_SYMBOL(__PAGE_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 387 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 388 | pteval_t __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 389 | |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 390 | void __init native_pagetable_setup_start(pgd_t *base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | { |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 392 | unsigned long pfn, va; |
| 393 | pgd_t *pgd; |
| 394 | pud_t *pud; |
| 395 | pmd_t *pmd; |
| 396 | pte_t *pte; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 397 | |
| 398 | /* |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 399 | * Remove any mappings which extend past the end of physical |
| 400 | * memory from the boot time page table: |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 401 | */ |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 402 | for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) { |
| 403 | va = PAGE_OFFSET + (pfn<<PAGE_SHIFT); |
| 404 | pgd = base + pgd_index(va); |
| 405 | if (!pgd_present(*pgd)) |
| 406 | break; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 407 | |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 408 | pud = pud_offset(pgd, va); |
| 409 | pmd = pmd_offset(pud, va); |
| 410 | if (!pmd_present(*pmd)) |
| 411 | break; |
| 412 | |
| 413 | pte = pte_offset_kernel(pmd, va); |
| 414 | if (!pte_present(*pte)) |
| 415 | break; |
| 416 | |
| 417 | pte_clear(NULL, va, pte); |
| 418 | } |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 419 | paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 420 | } |
| 421 | |
| 422 | void __init native_pagetable_setup_done(pgd_t *base) |
| 423 | { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 424 | } |
| 425 | |
| 426 | /* |
| 427 | * Build a proper pagetable for the kernel mappings. Up until this |
| 428 | * point, we've been running on some set of pagetables constructed by |
| 429 | * the boot process. |
| 430 | * |
| 431 | * If we're booting on native hardware, this will be a pagetable |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 432 | * constructed in arch/x86/kernel/head_32.S. The root of the |
| 433 | * pagetable will be swapper_pg_dir. |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 434 | * |
| 435 | * If we're booting paravirtualized under a hypervisor, then there are |
| 436 | * more options: we may already be running PAE, and the pagetable may |
| 437 | * or may not be based in swapper_pg_dir. In any case, |
| 438 | * paravirt_pagetable_setup_start() will set up swapper_pg_dir |
| 439 | * appropriately for the rest of the initialization to work. |
| 440 | * |
| 441 | * In general, pagetable_init() assumes that the pagetable may already |
| 442 | * be partially populated, and so it avoids stomping on any existing |
| 443 | * mappings. |
| 444 | */ |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 445 | static void __init pagetable_init(void) |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 446 | { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 447 | pgd_t *pgd_base = swapper_pg_dir; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 448 | unsigned long vaddr, end; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 449 | |
| 450 | paravirt_pagetable_setup_start(pgd_base); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 451 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 452 | /* |
| 453 | * Fixed mappings, only the page table structure has to be |
| 454 | * created - mappings will be set by set_fixmap(): |
| 455 | */ |
Huang, Ying | beacfaa | 2008-01-30 13:33:44 +0100 | [diff] [blame] | 456 | early_ioremap_clear(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 457 | vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 458 | end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK; |
| 459 | page_table_range_init(vaddr, end, pgd_base); |
Huang, Ying | beacfaa | 2008-01-30 13:33:44 +0100 | [diff] [blame] | 460 | early_ioremap_reset(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 461 | |
| 462 | permanent_kmaps_init(pgd_base); |
| 463 | |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 464 | paravirt_pagetable_setup_done(pgd_base); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | } |
| 466 | |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 467 | #ifdef CONFIG_ACPI_SLEEP |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | /* |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 469 | * ACPI suspend needs this for resume, because things like the intel-agp |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 470 | * driver might have split up a kernel 4MB mapping. |
| 471 | */ |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 472 | char swsusp_pg_dir[PAGE_SIZE] |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 473 | __attribute__ ((aligned(PAGE_SIZE))); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | |
| 475 | static inline void save_pg_dir(void) |
| 476 | { |
| 477 | memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE); |
| 478 | } |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 479 | #else /* !CONFIG_ACPI_SLEEP */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 480 | static inline void save_pg_dir(void) |
| 481 | { |
| 482 | } |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 483 | #endif /* !CONFIG_ACPI_SLEEP */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 484 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 485 | void zap_low_mappings(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 486 | { |
| 487 | int i; |
| 488 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 489 | /* |
| 490 | * Zap initial low-memory mappings. |
| 491 | * |
| 492 | * Note that "pgd_clear()" doesn't do it for |
| 493 | * us, because pgd_clear() is a no-op on i386. |
| 494 | */ |
Jeremy Fitzhardinge | 68db065 | 2008-03-17 16:37:13 -0700 | [diff] [blame] | 495 | for (i = 0; i < KERNEL_PGD_BOUNDARY; i++) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 496 | #ifdef CONFIG_X86_PAE |
| 497 | set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page))); |
| 498 | #else |
| 499 | set_pgd(swapper_pg_dir+i, __pgd(0)); |
| 500 | #endif |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 501 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | flush_tlb_all(); |
| 503 | } |
| 504 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 505 | int nx_enabled; |
Jan Beulich | d5321ab | 2007-07-21 17:10:26 +0200 | [diff] [blame] | 506 | |
Jeremy Fitzhardinge | 6fdc05d | 2008-01-30 13:32:57 +0100 | [diff] [blame] | 507 | pteval_t __supported_pte_mask __read_mostly = ~_PAGE_NX; |
| 508 | EXPORT_SYMBOL_GPL(__supported_pte_mask); |
| 509 | |
Jan Beulich | d5321ab | 2007-07-21 17:10:26 +0200 | [diff] [blame] | 510 | #ifdef CONFIG_X86_PAE |
| 511 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 512 | static int disable_nx __initdata; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 513 | |
| 514 | /* |
| 515 | * noexec = on|off |
| 516 | * |
| 517 | * Control non executable mappings. |
| 518 | * |
| 519 | * on Enable |
| 520 | * off Disable |
| 521 | */ |
Rusty Russell | 1a3f239 | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 522 | static int __init noexec_setup(char *str) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 523 | { |
Rusty Russell | 1a3f239 | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 524 | if (!str || !strcmp(str, "on")) { |
| 525 | if (cpu_has_nx) { |
| 526 | __supported_pte_mask |= _PAGE_NX; |
| 527 | disable_nx = 0; |
| 528 | } |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 529 | } else { |
| 530 | if (!strcmp(str, "off")) { |
| 531 | disable_nx = 1; |
| 532 | __supported_pte_mask &= ~_PAGE_NX; |
| 533 | } else { |
| 534 | return -EINVAL; |
| 535 | } |
| 536 | } |
Rusty Russell | 1a3f239 | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 537 | |
| 538 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 539 | } |
Rusty Russell | 1a3f239 | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 540 | early_param("noexec", noexec_setup); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 541 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 542 | static void __init set_nx(void) |
| 543 | { |
| 544 | unsigned int v[4], l, h; |
| 545 | |
| 546 | if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) { |
| 547 | cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 548 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 549 | if ((v[3] & (1 << 20)) && !disable_nx) { |
| 550 | rdmsr(MSR_EFER, l, h); |
| 551 | l |= EFER_NX; |
| 552 | wrmsr(MSR_EFER, l, h); |
| 553 | nx_enabled = 1; |
| 554 | __supported_pte_mask |= _PAGE_NX; |
| 555 | } |
| 556 | } |
| 557 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 558 | #endif |
| 559 | |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 560 | /* user-defined highmem size */ |
| 561 | static unsigned int highmem_pages = -1; |
| 562 | |
| 563 | /* |
| 564 | * highmem=size forces highmem to be exactly 'size' bytes. |
| 565 | * This works even on boxes that have no highmem otherwise. |
| 566 | * This also works to reduce highmem size on bigger boxes. |
| 567 | */ |
| 568 | static int __init parse_highmem(char *arg) |
| 569 | { |
| 570 | if (!arg) |
| 571 | return -EINVAL; |
| 572 | |
| 573 | highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT; |
| 574 | return 0; |
| 575 | } |
| 576 | early_param("highmem", parse_highmem); |
| 577 | |
| 578 | /* |
| 579 | * Determine low and high memory ranges: |
| 580 | */ |
Yinghai Lu | 2ec65f8 | 2008-06-23 03:05:30 -0700 | [diff] [blame] | 581 | void __init find_low_pfn_range(void) |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 582 | { |
Yinghai Lu | 2ec65f8 | 2008-06-23 03:05:30 -0700 | [diff] [blame] | 583 | /* it could update max_pfn */ |
| 584 | |
Yinghai Lu | 346cafe | 2008-06-23 03:06:14 -0700 | [diff] [blame] | 585 | /* max_low_pfn is 0, we already have early_res support */ |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 586 | |
| 587 | max_low_pfn = max_pfn; |
| 588 | if (max_low_pfn > MAXMEM_PFN) { |
| 589 | if (highmem_pages == -1) |
| 590 | highmem_pages = max_pfn - MAXMEM_PFN; |
| 591 | if (highmem_pages + MAXMEM_PFN < max_pfn) |
| 592 | max_pfn = MAXMEM_PFN + highmem_pages; |
| 593 | if (highmem_pages + MAXMEM_PFN > max_pfn) { |
| 594 | printk(KERN_WARNING "only %luMB highmem pages " |
| 595 | "available, ignoring highmem size of %uMB.\n", |
| 596 | pages_to_mb(max_pfn - MAXMEM_PFN), |
| 597 | pages_to_mb(highmem_pages)); |
| 598 | highmem_pages = 0; |
| 599 | } |
| 600 | max_low_pfn = MAXMEM_PFN; |
| 601 | #ifndef CONFIG_HIGHMEM |
| 602 | /* Maximum memory usable is what is directly addressable */ |
| 603 | printk(KERN_WARNING "Warning only %ldMB will be used.\n", |
| 604 | MAXMEM>>20); |
| 605 | if (max_pfn > MAX_NONPAE_PFN) |
| 606 | printk(KERN_WARNING |
| 607 | "Use a HIGHMEM64G enabled kernel.\n"); |
| 608 | else |
| 609 | printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n"); |
| 610 | max_pfn = MAXMEM_PFN; |
| 611 | #else /* !CONFIG_HIGHMEM */ |
| 612 | #ifndef CONFIG_HIGHMEM64G |
| 613 | if (max_pfn > MAX_NONPAE_PFN) { |
| 614 | max_pfn = MAX_NONPAE_PFN; |
| 615 | printk(KERN_WARNING "Warning only 4GB will be used." |
| 616 | "Use a HIGHMEM64G enabled kernel.\n"); |
| 617 | } |
| 618 | #endif /* !CONFIG_HIGHMEM64G */ |
| 619 | #endif /* !CONFIG_HIGHMEM */ |
| 620 | } else { |
| 621 | if (highmem_pages == -1) |
| 622 | highmem_pages = 0; |
| 623 | #ifdef CONFIG_HIGHMEM |
| 624 | if (highmem_pages >= max_pfn) { |
| 625 | printk(KERN_ERR "highmem size specified (%uMB) is " |
| 626 | "bigger than pages available (%luMB)!.\n", |
| 627 | pages_to_mb(highmem_pages), |
| 628 | pages_to_mb(max_pfn)); |
| 629 | highmem_pages = 0; |
| 630 | } |
| 631 | if (highmem_pages) { |
| 632 | if (max_low_pfn - highmem_pages < |
| 633 | 64*1024*1024/PAGE_SIZE){ |
| 634 | printk(KERN_ERR "highmem size %uMB results in " |
| 635 | "smaller than 64MB lowmem, ignoring it.\n" |
| 636 | , pages_to_mb(highmem_pages)); |
| 637 | highmem_pages = 0; |
| 638 | } |
| 639 | max_low_pfn -= highmem_pages; |
| 640 | } |
| 641 | #else |
| 642 | if (highmem_pages) |
| 643 | printk(KERN_ERR "ignoring highmem size on non-highmem" |
| 644 | " kernel!\n"); |
| 645 | #endif |
| 646 | } |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 647 | } |
| 648 | |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 649 | #ifndef CONFIG_NEED_MULTIPLE_NODES |
Yinghai Lu | 2ec65f8 | 2008-06-23 03:05:30 -0700 | [diff] [blame] | 650 | void __init initmem_init(unsigned long start_pfn, |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 651 | unsigned long end_pfn) |
| 652 | { |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 653 | #ifdef CONFIG_HIGHMEM |
| 654 | highstart_pfn = highend_pfn = max_pfn; |
| 655 | if (max_pfn > max_low_pfn) |
| 656 | highstart_pfn = max_low_pfn; |
| 657 | memory_present(0, 0, highend_pfn); |
| 658 | printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", |
| 659 | pages_to_mb(highend_pfn - highstart_pfn)); |
| 660 | num_physpages = highend_pfn; |
| 661 | high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; |
| 662 | #else |
| 663 | memory_present(0, 0, max_low_pfn); |
| 664 | num_physpages = max_low_pfn; |
| 665 | high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; |
| 666 | #endif |
| 667 | #ifdef CONFIG_FLATMEM |
| 668 | max_mapnr = num_physpages; |
| 669 | #endif |
| 670 | printk(KERN_NOTICE "%ldMB LOWMEM available.\n", |
| 671 | pages_to_mb(max_low_pfn)); |
| 672 | |
| 673 | setup_bootmem_allocator(); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 674 | } |
| 675 | |
| 676 | void __init zone_sizes_init(void) |
| 677 | { |
| 678 | unsigned long max_zone_pfns[MAX_NR_ZONES]; |
| 679 | memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); |
| 680 | max_zone_pfns[ZONE_DMA] = |
| 681 | virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT; |
| 682 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
| 683 | remove_all_active_ranges(); |
| 684 | #ifdef CONFIG_HIGHMEM |
| 685 | max_zone_pfns[ZONE_HIGHMEM] = highend_pfn; |
| 686 | e820_register_active_regions(0, 0, highend_pfn); |
| 687 | #else |
| 688 | e820_register_active_regions(0, 0, max_low_pfn); |
| 689 | #endif |
| 690 | |
| 691 | free_area_init_nodes(max_zone_pfns); |
| 692 | } |
| 693 | #endif /* !CONFIG_NEED_MULTIPLE_NODES */ |
| 694 | |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 695 | void __init setup_bootmem_allocator(void) |
| 696 | { |
| 697 | int i; |
| 698 | unsigned long bootmap_size, bootmap; |
| 699 | /* |
| 700 | * Initialize the boot-time allocator (with low memory only): |
| 701 | */ |
| 702 | bootmap_size = bootmem_bootmap_pages(max_low_pfn)<<PAGE_SHIFT; |
| 703 | bootmap = find_e820_area(min_low_pfn<<PAGE_SHIFT, |
| 704 | max_pfn_mapped<<PAGE_SHIFT, bootmap_size, |
| 705 | PAGE_SIZE); |
| 706 | if (bootmap == -1L) |
| 707 | panic("Cannot find bootmem map of size %ld\n", bootmap_size); |
| 708 | reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP"); |
Yinghai Lu | 225c37d | 2008-06-22 02:46:58 -0700 | [diff] [blame] | 709 | |
Yinghai Lu | 346cafe | 2008-06-23 03:06:14 -0700 | [diff] [blame] | 710 | /* don't touch min_low_pfn */ |
| 711 | bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT, |
| 712 | min_low_pfn, max_low_pfn); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 713 | printk(KERN_INFO " mapped low ram: 0 - %08lx\n", |
| 714 | max_pfn_mapped<<PAGE_SHIFT); |
| 715 | printk(KERN_INFO " low ram: %08lx - %08lx\n", |
| 716 | min_low_pfn<<PAGE_SHIFT, max_low_pfn<<PAGE_SHIFT); |
| 717 | printk(KERN_INFO " bootmap %08lx - %08lx\n", |
| 718 | bootmap, bootmap + bootmap_size); |
| 719 | for_each_online_node(i) |
| 720 | free_bootmem_with_active_regions(i, max_low_pfn); |
| 721 | early_res_to_bootmem(0, max_low_pfn<<PAGE_SHIFT); |
| 722 | |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 723 | after_init_bootmem = 1; |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 724 | } |
| 725 | |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 726 | static void __init find_early_table_space(unsigned long end) |
| 727 | { |
| 728 | unsigned long puds, pmds, tables, start; |
| 729 | |
| 730 | puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; |
| 731 | tables = PAGE_ALIGN(puds * sizeof(pud_t)); |
| 732 | |
| 733 | pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; |
| 734 | tables += PAGE_ALIGN(pmds * sizeof(pmd_t)); |
| 735 | |
Jeremy Fitzhardinge | 8207c25 | 2008-06-24 17:32:48 -0400 | [diff] [blame^] | 736 | if (!cpu_has_pse) { |
| 737 | int ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| 738 | tables += PAGE_ALIGN(ptes * sizeof(pte_t)); |
| 739 | } |
| 740 | |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 741 | /* |
| 742 | * RED-PEN putting page tables only on node 0 could |
| 743 | * cause a hotspot and fill up ZONE_DMA. The page tables |
| 744 | * need roughly 0.5KB per GB. |
| 745 | */ |
| 746 | start = 0x7000; |
| 747 | table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT, |
| 748 | tables, PAGE_SIZE); |
| 749 | if (table_start == -1UL) |
| 750 | panic("Cannot find space for the kernel page tables"); |
| 751 | |
| 752 | table_start >>= PAGE_SHIFT; |
| 753 | table_end = table_start; |
| 754 | table_top = table_start + (tables>>PAGE_SHIFT); |
| 755 | |
| 756 | printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n", |
| 757 | end, table_start << PAGE_SHIFT, |
| 758 | (table_start << PAGE_SHIFT) + tables); |
| 759 | } |
| 760 | |
| 761 | unsigned long __init_refok init_memory_mapping(unsigned long start, |
| 762 | unsigned long end) |
| 763 | { |
| 764 | pgd_t *pgd_base = swapper_pg_dir; |
| 765 | |
| 766 | /* |
| 767 | * Find space for the kernel direct mapping tables. |
| 768 | */ |
| 769 | if (!after_init_bootmem) |
| 770 | find_early_table_space(end); |
| 771 | |
| 772 | #ifdef CONFIG_X86_PAE |
| 773 | set_nx(); |
| 774 | if (nx_enabled) |
| 775 | printk(KERN_INFO "NX (Execute Disable) protection: active\n"); |
| 776 | #endif |
| 777 | |
| 778 | /* Enable PSE if available */ |
| 779 | if (cpu_has_pse) |
| 780 | set_in_cr4(X86_CR4_PSE); |
| 781 | |
| 782 | /* Enable PGE if available */ |
| 783 | if (cpu_has_pge) { |
| 784 | set_in_cr4(X86_CR4_PGE); |
| 785 | __PAGE_KERNEL |= _PAGE_GLOBAL; |
| 786 | __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL; |
| 787 | } |
| 788 | |
| 789 | kernel_physical_mapping_init(pgd_base, start, end); |
| 790 | |
| 791 | load_cr3(swapper_pg_dir); |
| 792 | |
| 793 | __flush_tlb_all(); |
| 794 | |
| 795 | if (!after_init_bootmem) |
| 796 | reserve_early(table_start << PAGE_SHIFT, |
| 797 | table_end << PAGE_SHIFT, "PGTABLE"); |
| 798 | |
| 799 | return end >> PAGE_SHIFT; |
| 800 | } |
| 801 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 802 | /* |
| 803 | * paging_init() sets up the page tables - note that the first 8MB are |
| 804 | * already mapped by head.S. |
| 805 | * |
| 806 | * This routines also unmaps the page at virtual kernel address 0, so |
| 807 | * that we can trap those pesky NULL-reference errors in the kernel. |
| 808 | */ |
| 809 | void __init paging_init(void) |
| 810 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 811 | pagetable_init(); |
| 812 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 813 | __flush_tlb_all(); |
| 814 | |
| 815 | kmap_init(); |
Yinghai Lu | 11cd0bc | 2008-06-23 19:51:10 -0700 | [diff] [blame] | 816 | |
| 817 | /* |
| 818 | * NOTE: at this point the bootmem allocator is fully available. |
| 819 | */ |
Yinghai Lu | 11cd0bc | 2008-06-23 19:51:10 -0700 | [diff] [blame] | 820 | sparse_init(); |
| 821 | zone_sizes_init(); |
| 822 | |
| 823 | paravirt_post_allocator_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 824 | } |
| 825 | |
| 826 | /* |
| 827 | * Test if the WP bit works in supervisor mode. It isn't supported on 386's |
Dmitri Vorobiev | f7f17a6 | 2008-04-21 00:47:55 +0400 | [diff] [blame] | 828 | * and also on some strange 486's. All 586+'s are OK. This used to involve |
| 829 | * black magic jumps to work around some nasty CPU bugs, but fortunately the |
| 830 | * switch to using exceptions got rid of all that. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 831 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 832 | static void __init test_wp_bit(void) |
| 833 | { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 834 | printk(KERN_INFO |
| 835 | "Checking if this processor honours the WP bit even in supervisor mode..."); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 836 | |
| 837 | /* Any page-aligned address will do, the test is non-destructive */ |
| 838 | __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY); |
| 839 | boot_cpu_data.wp_works_ok = do_test_wp_bit(); |
| 840 | clear_fixmap(FIX_WP_TEST); |
| 841 | |
| 842 | if (!boot_cpu_data.wp_works_ok) { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 843 | printk(KERN_CONT "No.\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 844 | #ifdef CONFIG_X86_WP_WORKS_OK |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 845 | panic( |
| 846 | "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 847 | #endif |
| 848 | } else { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 849 | printk(KERN_CONT "Ok.\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 850 | } |
| 851 | } |
| 852 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 853 | static struct kcore_list kcore_mem, kcore_vmalloc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 854 | |
| 855 | void __init mem_init(void) |
| 856 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 857 | int codesize, reservedpages, datasize, initsize; |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 858 | int tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 859 | |
Andy Whitcroft | 05b79bd | 2005-06-23 00:07:57 -0700 | [diff] [blame] | 860 | #ifdef CONFIG_FLATMEM |
Eric Sesterhenn | 8d8f3cb | 2006-10-03 23:34:58 +0200 | [diff] [blame] | 861 | BUG_ON(!mem_map); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 862 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 863 | /* this will put all low memory onto the freelists */ |
| 864 | totalram_pages += free_all_bootmem(); |
| 865 | |
| 866 | reservedpages = 0; |
| 867 | for (tmp = 0; tmp < max_low_pfn; tmp++) |
| 868 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 869 | * Only count reserved RAM pages: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 870 | */ |
| 871 | if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp))) |
| 872 | reservedpages++; |
| 873 | |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 874 | set_highmem_pages_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 875 | |
| 876 | codesize = (unsigned long) &_etext - (unsigned long) &_text; |
| 877 | datasize = (unsigned long) &_edata - (unsigned long) &_etext; |
| 878 | initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; |
| 879 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 880 | kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT); |
| 881 | kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 882 | VMALLOC_END-VMALLOC_START); |
| 883 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 884 | printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, " |
| 885 | "%dk reserved, %dk data, %dk init, %ldk highmem)\n", |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 886 | (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), |
| 887 | num_physpages << (PAGE_SHIFT-10), |
| 888 | codesize >> 10, |
| 889 | reservedpages << (PAGE_SHIFT-10), |
| 890 | datasize >> 10, |
| 891 | initsize >> 10, |
| 892 | (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)) |
| 893 | ); |
| 894 | |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 895 | printk(KERN_INFO "virtual kernel memory layout:\n" |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 896 | " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 897 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 898 | " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 899 | #endif |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 900 | " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| 901 | " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| 902 | " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| 903 | " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| 904 | " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", |
| 905 | FIXADDR_START, FIXADDR_TOP, |
| 906 | (FIXADDR_TOP - FIXADDR_START) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 907 | |
| 908 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 909 | PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, |
| 910 | (LAST_PKMAP*PAGE_SIZE) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 911 | #endif |
| 912 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 913 | VMALLOC_START, VMALLOC_END, |
| 914 | (VMALLOC_END - VMALLOC_START) >> 20, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 915 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 916 | (unsigned long)__va(0), (unsigned long)high_memory, |
| 917 | ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 918 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 919 | (unsigned long)&__init_begin, (unsigned long)&__init_end, |
| 920 | ((unsigned long)&__init_end - |
| 921 | (unsigned long)&__init_begin) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 922 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 923 | (unsigned long)&_etext, (unsigned long)&_edata, |
| 924 | ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 925 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 926 | (unsigned long)&_text, (unsigned long)&_etext, |
| 927 | ((unsigned long)&_etext - (unsigned long)&_text) >> 10); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 928 | |
| 929 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 930 | BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START); |
| 931 | BUG_ON(VMALLOC_END > PKMAP_BASE); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 932 | #endif |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 933 | BUG_ON(VMALLOC_START > VMALLOC_END); |
| 934 | BUG_ON((unsigned long)high_memory > VMALLOC_START); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 935 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 936 | if (boot_cpu_data.wp_works_ok < 0) |
| 937 | test_wp_bit(); |
| 938 | |
Thomas Gleixner | 76ebd05 | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 939 | cpa_init(); |
Hugh Dickins | 61165d7 | 2008-05-13 14:26:57 +0100 | [diff] [blame] | 940 | save_pg_dir(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 941 | zap_low_mappings(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 942 | } |
| 943 | |
KAMEZAWA Hiroyuki | ad8f579 | 2006-05-20 15:00:03 -0700 | [diff] [blame] | 944 | #ifdef CONFIG_MEMORY_HOTPLUG |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 945 | int arch_add_memory(int nid, u64 start, u64 size) |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 946 | { |
Yasunori Goto | 7c7e942 | 2006-12-22 01:11:13 -0800 | [diff] [blame] | 947 | struct pglist_data *pgdata = NODE_DATA(nid); |
Christoph Lameter | 776ed98 | 2006-09-25 23:31:09 -0700 | [diff] [blame] | 948 | struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM; |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 949 | unsigned long start_pfn = start >> PAGE_SHIFT; |
| 950 | unsigned long nr_pages = size >> PAGE_SHIFT; |
| 951 | |
| 952 | return __add_pages(zone, start_pfn, nr_pages); |
| 953 | } |
Andi Kleen | 9d99aaa | 2006-04-07 19:49:15 +0200 | [diff] [blame] | 954 | #endif |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 955 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 956 | /* |
| 957 | * This function cannot be __init, since exceptions don't work in that |
| 958 | * section. Put this after the callers, so that it cannot be inlined. |
| 959 | */ |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 960 | static noinline int do_test_wp_bit(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 961 | { |
| 962 | char tmp_reg; |
| 963 | int flag; |
| 964 | |
| 965 | __asm__ __volatile__( |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 966 | " movb %0, %1 \n" |
| 967 | "1: movb %1, %0 \n" |
| 968 | " xorl %2, %2 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 969 | "2: \n" |
H. Peter Anvin | f832ff1 | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 970 | _ASM_EXTABLE(1b,2b) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 971 | :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)), |
| 972 | "=q" (tmp_reg), |
| 973 | "=r" (flag) |
| 974 | :"2" (1) |
| 975 | :"memory"); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 976 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 977 | return flag; |
| 978 | } |
| 979 | |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 980 | #ifdef CONFIG_DEBUG_RODATA |
Arjan van de Ven | edeed30 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 981 | const int rodata_test_data = 0xC3; |
| 982 | EXPORT_SYMBOL_GPL(rodata_test_data); |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 983 | |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 984 | void mark_rodata_ro(void) |
| 985 | { |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 986 | unsigned long start = PFN_ALIGN(_text); |
| 987 | unsigned long size = PFN_ALIGN(_etext) - start; |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 988 | |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 989 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| 990 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", |
| 991 | size >> 10); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 992 | |
| 993 | #ifdef CONFIG_CPA_DEBUG |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 994 | printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n", |
| 995 | start, start+size); |
| 996 | set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 997 | |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 998 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
| 999 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); |
Linus Torvalds | 602033e | 2007-07-26 12:07:21 -0700 | [diff] [blame] | 1000 | #endif |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 1001 | start += size; |
| 1002 | size = (unsigned long)__end_rodata - start; |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 1003 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 1004 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", |
| 1005 | size >> 10); |
Arjan van de Ven | edeed30 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 1006 | rodata_test(); |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 1007 | |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 1008 | #ifdef CONFIG_CPA_DEBUG |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 1009 | printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size); |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 1010 | set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 1011 | |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 1012 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 1013 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 1014 | #endif |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 1015 | } |
| 1016 | #endif |
| 1017 | |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 1018 | void free_init_pages(char *what, unsigned long begin, unsigned long end) |
| 1019 | { |
Ingo Molnar | ee01f11 | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 1020 | #ifdef CONFIG_DEBUG_PAGEALLOC |
| 1021 | /* |
| 1022 | * If debugging page accesses then do not free this memory but |
| 1023 | * mark them not present - any buggy init-section access will |
| 1024 | * create a kernel page fault: |
| 1025 | */ |
| 1026 | printk(KERN_INFO "debug: unmapping init memory %08lx..%08lx\n", |
| 1027 | begin, PAGE_ALIGN(end)); |
| 1028 | set_memory_np(begin, (end - begin) >> PAGE_SHIFT); |
| 1029 | #else |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 1030 | unsigned long addr; |
| 1031 | |
Arjan van de Ven | 3c1df68 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 1032 | /* |
| 1033 | * We just marked the kernel text read only above, now that |
| 1034 | * we are going to free part of that, we need to make that |
| 1035 | * writeable first. |
| 1036 | */ |
| 1037 | set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); |
| 1038 | |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 1039 | for (addr = begin; addr < end; addr += PAGE_SIZE) { |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 1040 | ClearPageReserved(virt_to_page(addr)); |
| 1041 | init_page_count(virt_to_page(addr)); |
| 1042 | memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); |
| 1043 | free_page(addr); |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 1044 | totalram_pages++; |
| 1045 | } |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 1046 | printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); |
Ingo Molnar | ee01f11 | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 1047 | #endif |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 1048 | } |
| 1049 | |
| 1050 | void free_initmem(void) |
| 1051 | { |
| 1052 | free_init_pages("unused kernel memory", |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 1053 | (unsigned long)(&__init_begin), |
| 1054 | (unsigned long)(&__init_end)); |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 1055 | } |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 1056 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1057 | #ifdef CONFIG_BLK_DEV_INITRD |
| 1058 | void free_initrd_mem(unsigned long start, unsigned long end) |
| 1059 | { |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 1060 | free_init_pages("initrd memory", start, end); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1061 | } |
| 1062 | #endif |
Yinghai Lu | d2dbf34 | 2008-06-13 02:00:56 -0700 | [diff] [blame] | 1063 | |
| 1064 | int __init reserve_bootmem_generic(unsigned long phys, unsigned long len, |
| 1065 | int flags) |
| 1066 | { |
| 1067 | return reserve_bootmem(phys, len, flags); |
| 1068 | } |