Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * |
| 3 | * Copyright (C) 1995 Linus Torvalds |
| 4 | * |
| 5 | * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 |
| 6 | */ |
| 7 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/module.h> |
| 9 | #include <linux/signal.h> |
| 10 | #include <linux/sched.h> |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/errno.h> |
| 13 | #include <linux/string.h> |
| 14 | #include <linux/types.h> |
| 15 | #include <linux/ptrace.h> |
| 16 | #include <linux/mman.h> |
| 17 | #include <linux/mm.h> |
| 18 | #include <linux/hugetlb.h> |
| 19 | #include <linux/swap.h> |
| 20 | #include <linux/smp.h> |
| 21 | #include <linux/init.h> |
| 22 | #include <linux/highmem.h> |
| 23 | #include <linux/pagemap.h> |
Jeremy Fitzhardinge | cfb80c9 | 2008-12-16 12:17:36 -0800 | [diff] [blame] | 24 | #include <linux/pci.h> |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 25 | #include <linux/pfn.h> |
Randy Dunlap | c9cf552 | 2006-06-27 02:53:52 -0700 | [diff] [blame] | 26 | #include <linux/poison.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | #include <linux/bootmem.h> |
Yinghai Lu | a9ce6bc | 2010-08-25 13:39:17 -0700 | [diff] [blame] | 28 | #include <linux/memblock.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | #include <linux/proc_fs.h> |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 30 | #include <linux/memory_hotplug.h> |
Adrian Bunk | 27d99f7 | 2005-11-13 16:06:51 -0800 | [diff] [blame] | 31 | #include <linux/initrd.h> |
Shaohua Li | 55b2355 | 2006-06-23 02:04:49 -0700 | [diff] [blame] | 32 | #include <linux/cpumask.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 33 | #include <linux/gfp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
H. Peter Anvin | f832ff1 | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 35 | #include <asm/asm.h> |
Ingo Molnar | 46eaa67 | 2008-10-12 15:06:29 +0200 | [diff] [blame] | 36 | #include <asm/bios_ebda.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | #include <asm/processor.h> |
| 38 | #include <asm/system.h> |
| 39 | #include <asm/uaccess.h> |
| 40 | #include <asm/pgtable.h> |
| 41 | #include <asm/dma.h> |
| 42 | #include <asm/fixmap.h> |
| 43 | #include <asm/e820.h> |
| 44 | #include <asm/apic.h> |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 45 | #include <asm/bugs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | #include <asm/tlb.h> |
| 47 | #include <asm/tlbflush.h> |
Jeremy Fitzhardinge | a5a19c6 | 2008-01-30 13:33:39 +0100 | [diff] [blame] | 48 | #include <asm/pgalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | #include <asm/sections.h> |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 50 | #include <asm/paravirt.h> |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 51 | #include <asm/setup.h> |
Harvey Harrison | 7bfeab9 | 2008-02-12 12:12:01 -0800 | [diff] [blame] | 52 | #include <asm/cacheflush.h> |
Pekka Enberg | 2b72394 | 2009-04-28 16:00:49 +0300 | [diff] [blame] | 53 | #include <asm/page_types.h> |
Pekka Enberg | 4fcb208 | 2009-03-05 14:55:08 +0200 | [diff] [blame] | 54 | #include <asm/init.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 56 | unsigned long highstart_pfn, highend_pfn; |
| 57 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 58 | static noinline int do_test_wp_bit(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | |
Jeremy Fitzhardinge | dc16ecf | 2009-03-04 16:10:44 -0800 | [diff] [blame] | 60 | bool __read_mostly __vmalloc_start_set = false; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 61 | |
Jan Beulich | d6be89a | 2008-12-16 11:42:45 +0000 | [diff] [blame] | 62 | static __init void *alloc_low_page(void) |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 63 | { |
Pekka Enberg | 298af9d | 2009-03-05 14:55:06 +0200 | [diff] [blame] | 64 | unsigned long pfn = e820_table_end++; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 65 | void *adr; |
| 66 | |
Pekka Enberg | 298af9d | 2009-03-05 14:55:06 +0200 | [diff] [blame] | 67 | if (pfn >= e820_table_top) |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 68 | panic("alloc_low_page: ran out of memory"); |
| 69 | |
| 70 | adr = __va(pfn * PAGE_SIZE); |
| 71 | memset(adr, 0, PAGE_SIZE); |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 72 | return adr; |
| 73 | } |
| 74 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 75 | /* |
| 76 | * Creates a middle page table and puts a pointer to it in the |
| 77 | * given global directory entry. This only returns the gd entry |
| 78 | * in non-PAE compilation mode, since the middle layer is folded. |
| 79 | */ |
| 80 | static pmd_t * __init one_md_table_init(pgd_t *pgd) |
| 81 | { |
| 82 | pud_t *pud; |
| 83 | pmd_t *pmd_table; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 84 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | #ifdef CONFIG_X86_PAE |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 86 | if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { |
Pekka Enberg | c464573 | 2009-03-05 14:54:58 +0200 | [diff] [blame] | 87 | if (after_bootmem) |
Jan Beulich | 3c1596e | 2009-09-21 17:03:06 -0700 | [diff] [blame] | 88 | pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE); |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 89 | else |
Jan Beulich | d6be89a | 2008-12-16 11:42:45 +0000 | [diff] [blame] | 90 | pmd_table = (pmd_t *)alloc_low_page(); |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 91 | paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 92 | set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); |
| 93 | pud = pud_offset(pgd, 0); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 94 | BUG_ON(pmd_table != pmd_offset(pud, 0)); |
Zhaolei | a376f30 | 2008-10-31 17:43:04 +0800 | [diff] [blame] | 95 | |
| 96 | return pmd_table; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 97 | } |
| 98 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | pud = pud_offset(pgd, 0); |
| 100 | pmd_table = pmd_offset(pud, 0); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 101 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | return pmd_table; |
| 103 | } |
| 104 | |
| 105 | /* |
| 106 | * Create a page table and place a pointer to it in a middle page |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 107 | * directory entry: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | */ |
| 109 | static pte_t * __init one_page_table_init(pmd_t *pmd) |
| 110 | { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 111 | if (!(pmd_val(*pmd) & _PAGE_PRESENT)) { |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 112 | pte_t *page_table = NULL; |
| 113 | |
Pekka Enberg | c464573 | 2009-03-05 14:54:58 +0200 | [diff] [blame] | 114 | if (after_bootmem) { |
Vegard Nossum | f856129 | 2008-04-04 00:53:23 +0200 | [diff] [blame] | 115 | #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK) |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 116 | page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE); |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 117 | #endif |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 118 | if (!page_table) |
| 119 | page_table = |
Jan Beulich | 3c1596e | 2009-09-21 17:03:06 -0700 | [diff] [blame] | 120 | (pte_t *)alloc_bootmem_pages(PAGE_SIZE); |
Jan Beulich | d6be89a | 2008-12-16 11:42:45 +0000 | [diff] [blame] | 121 | } else |
| 122 | page_table = (pte_t *)alloc_low_page(); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 123 | |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 124 | paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 125 | set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 126 | BUG_ON(page_table != pte_offset_kernel(pmd, 0)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | } |
Ingo Molnar | 509a80c | 2007-10-17 18:04:34 +0200 | [diff] [blame] | 128 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | return pte_offset_kernel(pmd, 0); |
| 130 | } |
| 131 | |
Tejun Heo | 458a3e6 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 132 | pmd_t * __init populate_extra_pmd(unsigned long vaddr) |
Tejun Heo | 1112441 | 2009-02-20 16:29:09 +0900 | [diff] [blame] | 133 | { |
| 134 | int pgd_idx = pgd_index(vaddr); |
| 135 | int pmd_idx = pmd_index(vaddr); |
Tejun Heo | 458a3e6 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 136 | |
| 137 | return one_md_table_init(swapper_pg_dir + pgd_idx) + pmd_idx; |
| 138 | } |
| 139 | |
| 140 | pte_t * __init populate_extra_pte(unsigned long vaddr) |
| 141 | { |
| 142 | int pte_idx = pte_index(vaddr); |
Tejun Heo | 1112441 | 2009-02-20 16:29:09 +0900 | [diff] [blame] | 143 | pmd_t *pmd; |
| 144 | |
Tejun Heo | 458a3e6 | 2009-02-24 11:57:21 +0900 | [diff] [blame] | 145 | pmd = populate_extra_pmd(vaddr); |
| 146 | return one_page_table_init(pmd) + pte_idx; |
Tejun Heo | 1112441 | 2009-02-20 16:29:09 +0900 | [diff] [blame] | 147 | } |
| 148 | |
Jan Beulich | a3c6018 | 2009-01-16 11:59:33 +0000 | [diff] [blame] | 149 | static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd, |
| 150 | unsigned long vaddr, pte_t *lastpte) |
| 151 | { |
| 152 | #ifdef CONFIG_HIGHMEM |
| 153 | /* |
| 154 | * Something (early fixmap) may already have put a pte |
| 155 | * page here, which causes the page table allocation |
| 156 | * to become nonlinear. Attempt to fix it, and if it |
| 157 | * is still nonlinear then we have to bug. |
| 158 | */ |
| 159 | int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT; |
| 160 | int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT; |
| 161 | |
| 162 | if (pmd_idx_kmap_begin != pmd_idx_kmap_end |
| 163 | && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin |
| 164 | && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end |
Pekka Enberg | 298af9d | 2009-03-05 14:55:06 +0200 | [diff] [blame] | 165 | && ((__pa(pte) >> PAGE_SHIFT) < e820_table_start |
| 166 | || (__pa(pte) >> PAGE_SHIFT) >= e820_table_end)) { |
Jan Beulich | a3c6018 | 2009-01-16 11:59:33 +0000 | [diff] [blame] | 167 | pte_t *newpte; |
| 168 | int i; |
| 169 | |
Pekka Enberg | c464573 | 2009-03-05 14:54:58 +0200 | [diff] [blame] | 170 | BUG_ON(after_bootmem); |
Jan Beulich | a3c6018 | 2009-01-16 11:59:33 +0000 | [diff] [blame] | 171 | newpte = alloc_low_page(); |
| 172 | for (i = 0; i < PTRS_PER_PTE; i++) |
| 173 | set_pte(newpte + i, pte[i]); |
| 174 | |
| 175 | paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT); |
| 176 | set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE)); |
| 177 | BUG_ON(newpte != pte_offset_kernel(pmd, 0)); |
| 178 | __flush_tlb_all(); |
| 179 | |
| 180 | paravirt_release_pte(__pa(pte) >> PAGE_SHIFT); |
| 181 | pte = newpte; |
| 182 | } |
| 183 | BUG_ON(vaddr < fix_to_virt(FIX_KMAP_BEGIN - 1) |
| 184 | && vaddr > fix_to_virt(FIX_KMAP_END) |
| 185 | && lastpte && lastpte + PTRS_PER_PTE != pte); |
| 186 | #endif |
| 187 | return pte; |
| 188 | } |
| 189 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 191 | * This function initializes a certain range of kernel virtual memory |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | * with new bootmem page tables, everywhere page tables are missing in |
| 193 | * the given range. |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 194 | * |
| 195 | * NOTE: The pagetables are allocated contiguous on the physical space |
| 196 | * so we can cache the place of the first one and move around without |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | * checking the pgd every time. |
| 198 | */ |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 199 | static void __init |
| 200 | page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 201 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | int pgd_idx, pmd_idx; |
| 203 | unsigned long vaddr; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 204 | pgd_t *pgd; |
| 205 | pmd_t *pmd; |
Jan Beulich | a3c6018 | 2009-01-16 11:59:33 +0000 | [diff] [blame] | 206 | pte_t *pte = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | |
| 208 | vaddr = start; |
| 209 | pgd_idx = pgd_index(vaddr); |
| 210 | pmd_idx = pmd_index(vaddr); |
| 211 | pgd = pgd_base + pgd_idx; |
| 212 | |
| 213 | for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 214 | pmd = one_md_table_init(pgd); |
| 215 | pmd = pmd + pmd_index(vaddr); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 216 | for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); |
| 217 | pmd++, pmd_idx++) { |
Jan Beulich | a3c6018 | 2009-01-16 11:59:33 +0000 | [diff] [blame] | 218 | pte = page_table_kmap_check(one_page_table_init(pmd), |
| 219 | pmd, vaddr, pte); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | |
| 221 | vaddr += PMD_SIZE; |
| 222 | } |
| 223 | pmd_idx = 0; |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | static inline int is_kernel_text(unsigned long addr) |
| 228 | { |
| 229 | if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end) |
| 230 | return 1; |
| 231 | return 0; |
| 232 | } |
| 233 | |
| 234 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 235 | * This maps the physical memory to kernel virtual address space, a total |
| 236 | * of max_low_pfn pages, by creating page tables starting from address |
| 237 | * PAGE_OFFSET: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 238 | */ |
Pekka Enberg | e53fb04 | 2009-03-05 14:55:07 +0200 | [diff] [blame] | 239 | unsigned long __init |
| 240 | kernel_physical_mapping_init(unsigned long start, |
| 241 | unsigned long end, |
| 242 | unsigned long page_size_mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | { |
Pekka Enberg | e53fb04 | 2009-03-05 14:55:07 +0200 | [diff] [blame] | 244 | int use_pse = page_size_mask == (1<<PG_LEVEL_2M); |
Pekka Enberg | c1fd1b4 | 2010-02-24 17:04:47 +0200 | [diff] [blame] | 245 | unsigned long last_map_addr = end; |
Pekka Enberg | e53fb04 | 2009-03-05 14:55:07 +0200 | [diff] [blame] | 246 | unsigned long start_pfn, end_pfn; |
Pekka Enberg | e717985 | 2009-03-05 14:54:55 +0200 | [diff] [blame] | 247 | pgd_t *pgd_base = swapper_pg_dir; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 248 | int pgd_idx, pmd_idx, pte_ofs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 249 | unsigned long pfn; |
| 250 | pgd_t *pgd; |
| 251 | pmd_t *pmd; |
| 252 | pte_t *pte; |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 253 | unsigned pages_2m, pages_4k; |
| 254 | int mapping_iter; |
| 255 | |
Pekka Enberg | e53fb04 | 2009-03-05 14:55:07 +0200 | [diff] [blame] | 256 | start_pfn = start >> PAGE_SHIFT; |
| 257 | end_pfn = end >> PAGE_SHIFT; |
| 258 | |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 259 | /* |
| 260 | * First iteration will setup identity mapping using large/small pages |
| 261 | * based on use_pse, with other attributes same as set by |
| 262 | * the early code in head_32.S |
| 263 | * |
| 264 | * Second iteration will setup the appropriate attributes (NX, GLOBAL..) |
| 265 | * as desired for the kernel identity mapping. |
| 266 | * |
| 267 | * This two pass mechanism conforms to the TLB app note which says: |
| 268 | * |
| 269 | * "Software should not write to a paging-structure entry in a way |
| 270 | * that would change, for any linear address, both the page size |
| 271 | * and either the page frame or attributes." |
| 272 | */ |
| 273 | mapping_iter = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 274 | |
Yinghai Lu | a04ad82 | 2008-06-29 00:39:06 -0700 | [diff] [blame] | 275 | if (!cpu_has_pse) |
| 276 | use_pse = 0; |
| 277 | |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 278 | repeat: |
| 279 | pages_2m = pages_4k = 0; |
Yinghai Lu | a04ad82 | 2008-06-29 00:39:06 -0700 | [diff] [blame] | 280 | pfn = start_pfn; |
| 281 | pgd_idx = pgd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | pgd = pgd_base + pgd_idx; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 283 | for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { |
| 284 | pmd = one_md_table_init(pgd); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 285 | |
Yinghai Lu | a04ad82 | 2008-06-29 00:39:06 -0700 | [diff] [blame] | 286 | if (pfn >= end_pfn) |
| 287 | continue; |
| 288 | #ifdef CONFIG_X86_PAE |
| 289 | pmd_idx = pmd_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
| 290 | pmd += pmd_idx; |
| 291 | #else |
| 292 | pmd_idx = 0; |
| 293 | #endif |
| 294 | for (; pmd_idx < PTRS_PER_PMD && pfn < end_pfn; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 295 | pmd++, pmd_idx++) { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 296 | unsigned int addr = pfn * PAGE_SIZE + PAGE_OFFSET; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 297 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 298 | /* |
| 299 | * Map with big pages if possible, otherwise |
| 300 | * create normal page tables: |
| 301 | */ |
Yinghai Lu | a04ad82 | 2008-06-29 00:39:06 -0700 | [diff] [blame] | 302 | if (use_pse) { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 303 | unsigned int addr2; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 304 | pgprot_t prot = PAGE_KERNEL_LARGE; |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 305 | /* |
| 306 | * first pass will use the same initial |
| 307 | * identity mapping attribute + _PAGE_PSE. |
| 308 | */ |
| 309 | pgprot_t init_prot = |
| 310 | __pgprot(PTE_IDENT_ATTR | |
| 311 | _PAGE_PSE); |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 312 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 313 | addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE + |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 314 | PAGE_OFFSET + PAGE_SIZE-1; |
| 315 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 316 | if (is_kernel_text(addr) || |
| 317 | is_kernel_text(addr2)) |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 318 | prot = PAGE_KERNEL_LARGE_EXEC; |
| 319 | |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 320 | pages_2m++; |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 321 | if (mapping_iter == 1) |
| 322 | set_pmd(pmd, pfn_pmd(pfn, init_prot)); |
| 323 | else |
| 324 | set_pmd(pmd, pfn_pmd(pfn, prot)); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 325 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 326 | pfn += PTRS_PER_PTE; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 327 | continue; |
| 328 | } |
| 329 | pte = one_page_table_init(pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | |
Yinghai Lu | a04ad82 | 2008-06-29 00:39:06 -0700 | [diff] [blame] | 331 | pte_ofs = pte_index((pfn<<PAGE_SHIFT) + PAGE_OFFSET); |
| 332 | pte += pte_ofs; |
| 333 | for (; pte_ofs < PTRS_PER_PTE && pfn < end_pfn; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 334 | pte++, pfn++, pte_ofs++, addr += PAGE_SIZE) { |
| 335 | pgprot_t prot = PAGE_KERNEL; |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 336 | /* |
| 337 | * first pass will use the same initial |
| 338 | * identity mapping attribute. |
| 339 | */ |
| 340 | pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR); |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 341 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 342 | if (is_kernel_text(addr)) |
| 343 | prot = PAGE_KERNEL_EXEC; |
Jeremy Fitzhardinge | f3f20de | 2008-01-30 13:31:09 +0100 | [diff] [blame] | 344 | |
Andi Kleen | ce0c0e5 | 2008-05-02 11:46:49 +0200 | [diff] [blame] | 345 | pages_4k++; |
Pekka Enberg | c1fd1b4 | 2010-02-24 17:04:47 +0200 | [diff] [blame] | 346 | if (mapping_iter == 1) { |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 347 | set_pte(pte, pfn_pte(pfn, init_prot)); |
Pekka Enberg | c1fd1b4 | 2010-02-24 17:04:47 +0200 | [diff] [blame] | 348 | last_map_addr = (pfn << PAGE_SHIFT) + PAGE_SIZE; |
| 349 | } else |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 350 | set_pte(pte, pfn_pte(pfn, prot)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 351 | } |
| 352 | } |
| 353 | } |
Suresh Siddha | a2699e4 | 2008-09-23 14:00:38 -0700 | [diff] [blame] | 354 | if (mapping_iter == 1) { |
| 355 | /* |
| 356 | * update direct mapping page count only in the first |
| 357 | * iteration. |
| 358 | */ |
| 359 | update_page_count(PG_LEVEL_2M, pages_2m); |
| 360 | update_page_count(PG_LEVEL_4K, pages_4k); |
| 361 | |
| 362 | /* |
| 363 | * local global flush tlb, which will flush the previous |
| 364 | * mappings present in both small and large page TLB's. |
| 365 | */ |
| 366 | __flush_tlb_all(); |
| 367 | |
| 368 | /* |
| 369 | * Second iteration will set the actual desired PTE attributes. |
| 370 | */ |
| 371 | mapping_iter = 2; |
| 372 | goto repeat; |
| 373 | } |
Pekka Enberg | c1fd1b4 | 2010-02-24 17:04:47 +0200 | [diff] [blame] | 374 | return last_map_addr; |
Arjan van de Ven | ae531c2 | 2008-04-24 23:40:47 +0200 | [diff] [blame] | 375 | } |
| 376 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 377 | pte_t *kmap_pte; |
| 378 | pgprot_t kmap_prot; |
| 379 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 380 | static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr) |
| 381 | { |
| 382 | return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), |
| 383 | vaddr), vaddr), vaddr); |
| 384 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 385 | |
| 386 | static void __init kmap_init(void) |
| 387 | { |
| 388 | unsigned long kmap_vstart; |
| 389 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 390 | /* |
| 391 | * Cache the first kmap pte: |
| 392 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 393 | kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); |
| 394 | kmap_pte = kmap_get_fixmap_pte(kmap_vstart); |
| 395 | |
| 396 | kmap_prot = PAGE_KERNEL; |
| 397 | } |
| 398 | |
Keith Packard | fd94093 | 2008-10-30 19:37:09 -0700 | [diff] [blame] | 399 | #ifdef CONFIG_HIGHMEM |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 400 | static void __init permanent_kmaps_init(pgd_t *pgd_base) |
| 401 | { |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 402 | unsigned long vaddr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 403 | pgd_t *pgd; |
| 404 | pud_t *pud; |
| 405 | pmd_t *pmd; |
| 406 | pte_t *pte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | |
| 408 | vaddr = PKMAP_BASE; |
| 409 | page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); |
| 410 | |
| 411 | pgd = swapper_pg_dir + pgd_index(vaddr); |
| 412 | pud = pud_offset(pgd, vaddr); |
| 413 | pmd = pmd_offset(pud, vaddr); |
| 414 | pte = pte_offset_kernel(pmd, vaddr); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 415 | pkmap_page_table = pte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 416 | } |
| 417 | |
Minchan Kim | b1258ac | 2009-10-22 11:27:22 +0900 | [diff] [blame] | 418 | static void __init add_one_highpage_init(struct page *page) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 419 | { |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 420 | ClearPageReserved(page); |
| 421 | init_page_count(page); |
| 422 | __free_page(page); |
| 423 | totalhigh_pages++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 424 | } |
| 425 | |
Yinghai Lu | 1d93126 | 2010-10-05 16:15:15 -0700 | [diff] [blame^] | 426 | void __init add_highpages_with_active_regions(int nid, |
| 427 | unsigned long start_pfn, unsigned long end_pfn) |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 428 | { |
Yinghai Lu | 1d93126 | 2010-10-05 16:15:15 -0700 | [diff] [blame^] | 429 | struct range *range; |
| 430 | int nr_range; |
| 431 | int i; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 432 | |
Yinghai Lu | 1d93126 | 2010-10-05 16:15:15 -0700 | [diff] [blame^] | 433 | nr_range = __get_free_all_memory_range(&range, nid, start_pfn, end_pfn); |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 434 | |
Yinghai Lu | 1d93126 | 2010-10-05 16:15:15 -0700 | [diff] [blame^] | 435 | for (i = 0; i < nr_range; i++) { |
| 436 | struct page *page; |
| 437 | int node_pfn; |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 438 | |
Yinghai Lu | 1d93126 | 2010-10-05 16:15:15 -0700 | [diff] [blame^] | 439 | for (node_pfn = range[i].start; node_pfn < range[i].end; |
| 440 | node_pfn++) { |
| 441 | if (!pfn_valid(node_pfn)) |
| 442 | continue; |
| 443 | page = pfn_to_page(node_pfn); |
| 444 | add_one_highpage_init(page); |
| 445 | } |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 446 | } |
Yinghai Lu | b5bc6c0 | 2008-06-14 18:32:52 -0700 | [diff] [blame] | 447 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 448 | #else |
Ingo Brueckl | e8e3232 | 2009-01-02 14:42:00 +0100 | [diff] [blame] | 449 | static inline void permanent_kmaps_init(pgd_t *pgd_base) |
| 450 | { |
| 451 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 452 | #endif /* CONFIG_HIGHMEM */ |
| 453 | |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 454 | void __init native_pagetable_setup_start(pgd_t *base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 455 | { |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 456 | unsigned long pfn, va; |
| 457 | pgd_t *pgd; |
| 458 | pud_t *pud; |
| 459 | pmd_t *pmd; |
| 460 | pte_t *pte; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 461 | |
| 462 | /* |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 463 | * Remove any mappings which extend past the end of physical |
| 464 | * memory from the boot time page table: |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 465 | */ |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 466 | for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) { |
| 467 | va = PAGE_OFFSET + (pfn<<PAGE_SHIFT); |
| 468 | pgd = base + pgd_index(va); |
| 469 | if (!pgd_present(*pgd)) |
| 470 | break; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 471 | |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 472 | pud = pud_offset(pgd, va); |
| 473 | pmd = pmd_offset(pud, va); |
| 474 | if (!pmd_present(*pmd)) |
| 475 | break; |
| 476 | |
| 477 | pte = pte_offset_kernel(pmd, va); |
| 478 | if (!pte_present(*pte)) |
| 479 | break; |
| 480 | |
| 481 | pte_clear(NULL, va, pte); |
| 482 | } |
Jeremy Fitzhardinge | 6944a9c | 2008-03-17 16:37:01 -0700 | [diff] [blame] | 483 | paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT); |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 484 | } |
| 485 | |
| 486 | void __init native_pagetable_setup_done(pgd_t *base) |
| 487 | { |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 488 | } |
| 489 | |
| 490 | /* |
| 491 | * Build a proper pagetable for the kernel mappings. Up until this |
| 492 | * point, we've been running on some set of pagetables constructed by |
| 493 | * the boot process. |
| 494 | * |
| 495 | * If we're booting on native hardware, this will be a pagetable |
Ian Campbell | 551889a | 2008-02-09 23:24:09 +0100 | [diff] [blame] | 496 | * constructed in arch/x86/kernel/head_32.S. The root of the |
| 497 | * pagetable will be swapper_pg_dir. |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 498 | * |
| 499 | * If we're booting paravirtualized under a hypervisor, then there are |
| 500 | * more options: we may already be running PAE, and the pagetable may |
| 501 | * or may not be based in swapper_pg_dir. In any case, |
| 502 | * paravirt_pagetable_setup_start() will set up swapper_pg_dir |
| 503 | * appropriately for the rest of the initialization to work. |
| 504 | * |
| 505 | * In general, pagetable_init() assumes that the pagetable may already |
| 506 | * be partially populated, and so it avoids stomping on any existing |
| 507 | * mappings. |
| 508 | */ |
Pekka Enberg | f765090 | 2009-03-05 14:55:05 +0200 | [diff] [blame] | 509 | void __init early_ioremap_page_table_range_init(void) |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 510 | { |
Pekka Enberg | e717985 | 2009-03-05 14:54:55 +0200 | [diff] [blame] | 511 | pgd_t *pgd_base = swapper_pg_dir; |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 512 | unsigned long vaddr, end; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 513 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 514 | /* |
| 515 | * Fixed mappings, only the page table structure has to be |
| 516 | * created - mappings will be set by set_fixmap(): |
| 517 | */ |
| 518 | vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; |
Jeremy Fitzhardinge | b239fb2 | 2007-05-02 19:27:13 +0200 | [diff] [blame] | 519 | end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK; |
| 520 | page_table_range_init(vaddr, end, pgd_base); |
Huang, Ying | beacfaa | 2008-01-30 13:33:44 +0100 | [diff] [blame] | 521 | early_ioremap_reset(); |
Yinghai Lu | e7b3789 | 2008-06-25 21:51:28 -0700 | [diff] [blame] | 522 | } |
| 523 | |
| 524 | static void __init pagetable_init(void) |
| 525 | { |
| 526 | pgd_t *pgd_base = swapper_pg_dir; |
| 527 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 528 | permanent_kmaps_init(pgd_base); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 529 | } |
| 530 | |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 531 | #ifdef CONFIG_ACPI_SLEEP |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 532 | /* |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 533 | * ACPI suspend needs this for resume, because things like the intel-agp |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 534 | * driver might have split up a kernel 4MB mapping. |
| 535 | */ |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 536 | char swsusp_pg_dir[PAGE_SIZE] |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 537 | __attribute__ ((aligned(PAGE_SIZE))); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 538 | |
| 539 | static inline void save_pg_dir(void) |
| 540 | { |
| 541 | memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE); |
| 542 | } |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 543 | #else /* !CONFIG_ACPI_SLEEP */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | static inline void save_pg_dir(void) |
| 545 | { |
| 546 | } |
Rafael J. Wysocki | a6eb84b | 2008-02-01 15:28:16 +0100 | [diff] [blame] | 547 | #endif /* !CONFIG_ACPI_SLEEP */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | |
Yinghai Lu | 55cd636 | 2009-06-12 11:36:52 +0300 | [diff] [blame] | 549 | void zap_low_mappings(bool early) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 550 | { |
| 551 | int i; |
| 552 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 553 | /* |
| 554 | * Zap initial low-memory mappings. |
| 555 | * |
| 556 | * Note that "pgd_clear()" doesn't do it for |
| 557 | * us, because pgd_clear() is a no-op on i386. |
| 558 | */ |
Jeremy Fitzhardinge | 68db065 | 2008-03-17 16:37:13 -0700 | [diff] [blame] | 559 | for (i = 0; i < KERNEL_PGD_BOUNDARY; i++) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 560 | #ifdef CONFIG_X86_PAE |
| 561 | set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page))); |
| 562 | #else |
| 563 | set_pgd(swapper_pg_dir+i, __pgd(0)); |
| 564 | #endif |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 565 | } |
Yinghai Lu | 55cd636 | 2009-06-12 11:36:52 +0300 | [diff] [blame] | 566 | |
| 567 | if (early) |
| 568 | __flush_tlb(); |
| 569 | else |
| 570 | flush_tlb_all(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 571 | } |
| 572 | |
Jeremy Fitzhardinge | be43d72 | 2008-09-07 15:21:13 -0700 | [diff] [blame] | 573 | pteval_t __supported_pte_mask __read_mostly = ~(_PAGE_NX | _PAGE_GLOBAL | _PAGE_IOMAP); |
Jeremy Fitzhardinge | 6fdc05d | 2008-01-30 13:32:57 +0100 | [diff] [blame] | 574 | EXPORT_SYMBOL_GPL(__supported_pte_mask); |
| 575 | |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 576 | /* user-defined highmem size */ |
| 577 | static unsigned int highmem_pages = -1; |
| 578 | |
| 579 | /* |
| 580 | * highmem=size forces highmem to be exactly 'size' bytes. |
| 581 | * This works even on boxes that have no highmem otherwise. |
| 582 | * This also works to reduce highmem size on bigger boxes. |
| 583 | */ |
| 584 | static int __init parse_highmem(char *arg) |
| 585 | { |
| 586 | if (!arg) |
| 587 | return -EINVAL; |
| 588 | |
| 589 | highmem_pages = memparse(arg, &arg) >> PAGE_SHIFT; |
| 590 | return 0; |
| 591 | } |
| 592 | early_param("highmem", parse_highmem); |
| 593 | |
Ingo Molnar | 4769843 | 2009-02-12 13:31:41 +0100 | [diff] [blame] | 594 | #define MSG_HIGHMEM_TOO_BIG \ |
| 595 | "highmem size (%luMB) is bigger than pages available (%luMB)!\n" |
| 596 | |
| 597 | #define MSG_LOWMEM_TOO_SMALL \ |
| 598 | "highmem size (%luMB) results in <64MB lowmem, ignoring it!\n" |
| 599 | /* |
| 600 | * All of RAM fits into lowmem - but if user wants highmem |
| 601 | * artificially via the highmem=x boot parameter then create |
| 602 | * it: |
| 603 | */ |
| 604 | void __init lowmem_pfn_init(void) |
| 605 | { |
Ingo Molnar | d88316c2 | 2009-02-12 15:16:03 +0100 | [diff] [blame] | 606 | /* max_low_pfn is 0, we already have early_res support */ |
| 607 | max_low_pfn = max_pfn; |
| 608 | |
Ingo Molnar | 4769843 | 2009-02-12 13:31:41 +0100 | [diff] [blame] | 609 | if (highmem_pages == -1) |
| 610 | highmem_pages = 0; |
| 611 | #ifdef CONFIG_HIGHMEM |
| 612 | if (highmem_pages >= max_pfn) { |
| 613 | printk(KERN_ERR MSG_HIGHMEM_TOO_BIG, |
| 614 | pages_to_mb(highmem_pages), pages_to_mb(max_pfn)); |
| 615 | highmem_pages = 0; |
| 616 | } |
| 617 | if (highmem_pages) { |
| 618 | if (max_low_pfn - highmem_pages < 64*1024*1024/PAGE_SIZE) { |
| 619 | printk(KERN_ERR MSG_LOWMEM_TOO_SMALL, |
| 620 | pages_to_mb(highmem_pages)); |
| 621 | highmem_pages = 0; |
| 622 | } |
| 623 | max_low_pfn -= highmem_pages; |
| 624 | } |
| 625 | #else |
| 626 | if (highmem_pages) |
| 627 | printk(KERN_ERR "ignoring highmem size on non-highmem kernel!\n"); |
| 628 | #endif |
| 629 | } |
| 630 | |
| 631 | #define MSG_HIGHMEM_TOO_SMALL \ |
| 632 | "only %luMB highmem pages available, ignoring highmem size of %luMB!\n" |
| 633 | |
| 634 | #define MSG_HIGHMEM_TRIMMED \ |
| 635 | "Warning: only 4GB will be used. Use a HIGHMEM64G enabled kernel!\n" |
| 636 | /* |
| 637 | * We have more RAM than fits into lowmem - we try to put it into |
| 638 | * highmem, also taking the highmem=x boot parameter into account: |
| 639 | */ |
| 640 | void __init highmem_pfn_init(void) |
| 641 | { |
Ingo Molnar | d88316c2 | 2009-02-12 15:16:03 +0100 | [diff] [blame] | 642 | max_low_pfn = MAXMEM_PFN; |
| 643 | |
Ingo Molnar | 4769843 | 2009-02-12 13:31:41 +0100 | [diff] [blame] | 644 | if (highmem_pages == -1) |
| 645 | highmem_pages = max_pfn - MAXMEM_PFN; |
| 646 | |
| 647 | if (highmem_pages + MAXMEM_PFN < max_pfn) |
| 648 | max_pfn = MAXMEM_PFN + highmem_pages; |
| 649 | |
| 650 | if (highmem_pages + MAXMEM_PFN > max_pfn) { |
| 651 | printk(KERN_WARNING MSG_HIGHMEM_TOO_SMALL, |
| 652 | pages_to_mb(max_pfn - MAXMEM_PFN), |
| 653 | pages_to_mb(highmem_pages)); |
| 654 | highmem_pages = 0; |
| 655 | } |
Ingo Molnar | 4769843 | 2009-02-12 13:31:41 +0100 | [diff] [blame] | 656 | #ifndef CONFIG_HIGHMEM |
| 657 | /* Maximum memory usable is what is directly addressable */ |
| 658 | printk(KERN_WARNING "Warning only %ldMB will be used.\n", MAXMEM>>20); |
| 659 | if (max_pfn > MAX_NONPAE_PFN) |
| 660 | printk(KERN_WARNING "Use a HIGHMEM64G enabled kernel.\n"); |
| 661 | else |
| 662 | printk(KERN_WARNING "Use a HIGHMEM enabled kernel.\n"); |
| 663 | max_pfn = MAXMEM_PFN; |
| 664 | #else /* !CONFIG_HIGHMEM */ |
| 665 | #ifndef CONFIG_HIGHMEM64G |
| 666 | if (max_pfn > MAX_NONPAE_PFN) { |
| 667 | max_pfn = MAX_NONPAE_PFN; |
| 668 | printk(KERN_WARNING MSG_HIGHMEM_TRIMMED); |
| 669 | } |
| 670 | #endif /* !CONFIG_HIGHMEM64G */ |
| 671 | #endif /* !CONFIG_HIGHMEM */ |
| 672 | } |
| 673 | |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 674 | /* |
| 675 | * Determine low and high memory ranges: |
| 676 | */ |
Yinghai Lu | 2ec65f8 | 2008-06-23 03:05:30 -0700 | [diff] [blame] | 677 | void __init find_low_pfn_range(void) |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 678 | { |
Yinghai Lu | 2ec65f8 | 2008-06-23 03:05:30 -0700 | [diff] [blame] | 679 | /* it could update max_pfn */ |
| 680 | |
Ingo Molnar | d88316c2 | 2009-02-12 15:16:03 +0100 | [diff] [blame] | 681 | if (max_pfn <= MAXMEM_PFN) |
Ingo Molnar | 4769843 | 2009-02-12 13:31:41 +0100 | [diff] [blame] | 682 | lowmem_pfn_init(); |
Ingo Molnar | d88316c2 | 2009-02-12 15:16:03 +0100 | [diff] [blame] | 683 | else |
| 684 | highmem_pfn_init(); |
Yinghai Lu | 90d967e | 2008-06-23 21:00:45 +0200 | [diff] [blame] | 685 | } |
| 686 | |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 687 | #ifndef CONFIG_NEED_MULTIPLE_NODES |
David Rientjes | 8ee2deb | 2009-09-25 15:20:00 -0700 | [diff] [blame] | 688 | void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn, |
| 689 | int acpi, int k8) |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 690 | { |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 691 | #ifdef CONFIG_HIGHMEM |
| 692 | highstart_pfn = highend_pfn = max_pfn; |
| 693 | if (max_pfn > max_low_pfn) |
| 694 | highstart_pfn = max_low_pfn; |
Yinghai Lu | a9ce6bc | 2010-08-25 13:39:17 -0700 | [diff] [blame] | 695 | memblock_x86_register_active_regions(0, 0, highend_pfn); |
Shaohua Li | ed077b5 | 2009-05-12 16:40:00 +0800 | [diff] [blame] | 696 | sparse_memory_present_with_active_regions(0); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 697 | printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", |
| 698 | pages_to_mb(highend_pfn - highstart_pfn)); |
| 699 | num_physpages = highend_pfn; |
| 700 | high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; |
| 701 | #else |
Yinghai Lu | a9ce6bc | 2010-08-25 13:39:17 -0700 | [diff] [blame] | 702 | memblock_x86_register_active_regions(0, 0, max_low_pfn); |
Shaohua Li | ed077b5 | 2009-05-12 16:40:00 +0800 | [diff] [blame] | 703 | sparse_memory_present_with_active_regions(0); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 704 | num_physpages = max_low_pfn; |
| 705 | high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; |
| 706 | #endif |
| 707 | #ifdef CONFIG_FLATMEM |
| 708 | max_mapnr = num_physpages; |
| 709 | #endif |
Jeremy Fitzhardinge | dc16ecf | 2009-03-04 16:10:44 -0800 | [diff] [blame] | 710 | __vmalloc_start_set = true; |
| 711 | |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 712 | printk(KERN_NOTICE "%ldMB LOWMEM available.\n", |
| 713 | pages_to_mb(max_low_pfn)); |
| 714 | |
| 715 | setup_bootmem_allocator(); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 716 | } |
Yinghai Lu | cb95a13 | 2008-07-02 00:31:02 -0700 | [diff] [blame] | 717 | #endif /* !CONFIG_NEED_MULTIPLE_NODES */ |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 718 | |
Yinghai Lu | cb95a13 | 2008-07-02 00:31:02 -0700 | [diff] [blame] | 719 | static void __init zone_sizes_init(void) |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 720 | { |
| 721 | unsigned long max_zone_pfns[MAX_NR_ZONES]; |
| 722 | memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); |
| 723 | max_zone_pfns[ZONE_DMA] = |
| 724 | virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT; |
| 725 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 726 | #ifdef CONFIG_HIGHMEM |
| 727 | max_zone_pfns[ZONE_HIGHMEM] = highend_pfn; |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 728 | #endif |
| 729 | |
| 730 | free_area_init_nodes(max_zone_pfns); |
| 731 | } |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 732 | |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 733 | void __init setup_bootmem_allocator(void) |
| 734 | { |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 735 | printk(KERN_INFO " mapped low ram: 0 - %08lx\n", |
| 736 | max_pfn_mapped<<PAGE_SHIFT); |
Yinghai Lu | fc5efe3 | 2009-03-04 12:21:24 -0800 | [diff] [blame] | 737 | printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT); |
Yinghai Lu | b2ac82a | 2008-06-22 02:45:39 -0700 | [diff] [blame] | 738 | |
Pekka Enberg | c464573 | 2009-03-05 14:54:58 +0200 | [diff] [blame] | 739 | after_bootmem = 1; |
Yinghai Lu | 4e29684 | 2008-06-24 12:18:14 -0700 | [diff] [blame] | 740 | } |
| 741 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 742 | /* |
| 743 | * paging_init() sets up the page tables - note that the first 8MB are |
| 744 | * already mapped by head.S. |
| 745 | * |
| 746 | * This routines also unmaps the page at virtual kernel address 0, so |
| 747 | * that we can trap those pesky NULL-reference errors in the kernel. |
| 748 | */ |
| 749 | void __init paging_init(void) |
| 750 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 751 | pagetable_init(); |
| 752 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 753 | __flush_tlb_all(); |
| 754 | |
| 755 | kmap_init(); |
Yinghai Lu | 11cd0bc | 2008-06-23 19:51:10 -0700 | [diff] [blame] | 756 | |
| 757 | /* |
| 758 | * NOTE: at this point the bootmem allocator is fully available. |
| 759 | */ |
Yinghai Lu | 11cd0bc | 2008-06-23 19:51:10 -0700 | [diff] [blame] | 760 | sparse_init(); |
| 761 | zone_sizes_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 762 | } |
| 763 | |
| 764 | /* |
| 765 | * Test if the WP bit works in supervisor mode. It isn't supported on 386's |
Dmitri Vorobiev | f7f17a6 | 2008-04-21 00:47:55 +0400 | [diff] [blame] | 766 | * and also on some strange 486's. All 586+'s are OK. This used to involve |
| 767 | * black magic jumps to work around some nasty CPU bugs, but fortunately the |
| 768 | * switch to using exceptions got rid of all that. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 769 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 770 | static void __init test_wp_bit(void) |
| 771 | { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 772 | printk(KERN_INFO |
| 773 | "Checking if this processor honours the WP bit even in supervisor mode..."); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 774 | |
| 775 | /* Any page-aligned address will do, the test is non-destructive */ |
| 776 | __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY); |
| 777 | boot_cpu_data.wp_works_ok = do_test_wp_bit(); |
| 778 | clear_fixmap(FIX_WP_TEST); |
| 779 | |
| 780 | if (!boot_cpu_data.wp_works_ok) { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 781 | printk(KERN_CONT "No.\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 782 | #ifdef CONFIG_X86_WP_WORKS_OK |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 783 | panic( |
| 784 | "This kernel doesn't support CPU's with broken WP. Recompile it for a 386!"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 785 | #endif |
| 786 | } else { |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 787 | printk(KERN_CONT "Ok.\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 788 | } |
| 789 | } |
| 790 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 791 | void __init mem_init(void) |
| 792 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 793 | int codesize, reservedpages, datasize, initsize; |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 794 | int tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 795 | |
Jeremy Fitzhardinge | cfb80c9 | 2008-12-16 12:17:36 -0800 | [diff] [blame] | 796 | pci_iommu_alloc(); |
| 797 | |
Andy Whitcroft | 05b79bd | 2005-06-23 00:07:57 -0700 | [diff] [blame] | 798 | #ifdef CONFIG_FLATMEM |
Eric Sesterhenn | 8d8f3cb | 2006-10-03 23:34:58 +0200 | [diff] [blame] | 799 | BUG_ON(!mem_map); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 800 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 801 | /* this will put all low memory onto the freelists */ |
| 802 | totalram_pages += free_all_bootmem(); |
| 803 | |
| 804 | reservedpages = 0; |
| 805 | for (tmp = 0; tmp < max_low_pfn; tmp++) |
| 806 | /* |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 807 | * Only count reserved RAM pages: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 808 | */ |
| 809 | if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp))) |
| 810 | reservedpages++; |
| 811 | |
Yinghai Lu | cc9f7a0 | 2008-06-16 16:11:08 -0700 | [diff] [blame] | 812 | set_highmem_pages_init(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 813 | |
| 814 | codesize = (unsigned long) &_etext - (unsigned long) &_text; |
| 815 | datasize = (unsigned long) &_edata - (unsigned long) &_etext; |
| 816 | initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; |
| 817 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 818 | printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, " |
| 819 | "%dk reserved, %dk data, %dk init, %ldk highmem)\n", |
Geert Uytterhoeven | cc013a8 | 2009-09-21 17:02:36 -0700 | [diff] [blame] | 820 | nr_free_pages() << (PAGE_SHIFT-10), |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 821 | num_physpages << (PAGE_SHIFT-10), |
| 822 | codesize >> 10, |
| 823 | reservedpages << (PAGE_SHIFT-10), |
| 824 | datasize >> 10, |
| 825 | initsize >> 10, |
Andreas Fenkart | 4b52940 | 2010-01-08 14:42:31 -0800 | [diff] [blame] | 826 | totalhigh_pages << (PAGE_SHIFT-10)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 827 | |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 828 | printk(KERN_INFO "virtual kernel memory layout:\n" |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 829 | " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 830 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 831 | " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 832 | #endif |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 833 | " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| 834 | " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" |
| 835 | " .init : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| 836 | " .data : 0x%08lx - 0x%08lx (%4ld kB)\n" |
| 837 | " .text : 0x%08lx - 0x%08lx (%4ld kB)\n", |
| 838 | FIXADDR_START, FIXADDR_TOP, |
| 839 | (FIXADDR_TOP - FIXADDR_START) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 840 | |
| 841 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 842 | PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, |
| 843 | (LAST_PKMAP*PAGE_SIZE) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 844 | #endif |
| 845 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 846 | VMALLOC_START, VMALLOC_END, |
| 847 | (VMALLOC_END - VMALLOC_START) >> 20, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 848 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 849 | (unsigned long)__va(0), (unsigned long)high_memory, |
| 850 | ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 851 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 852 | (unsigned long)&__init_begin, (unsigned long)&__init_end, |
| 853 | ((unsigned long)&__init_end - |
| 854 | (unsigned long)&__init_begin) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 855 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 856 | (unsigned long)&_etext, (unsigned long)&_edata, |
| 857 | ((unsigned long)&_edata - (unsigned long)&_etext) >> 10, |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 858 | |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 859 | (unsigned long)&_text, (unsigned long)&_etext, |
| 860 | ((unsigned long)&_etext - (unsigned long)&_text) >> 10); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 861 | |
Jan Beulich | beeb419 | 2008-12-16 11:45:56 +0000 | [diff] [blame] | 862 | /* |
| 863 | * Check boundaries twice: Some fundamental inconsistencies can |
| 864 | * be detected at build time already. |
| 865 | */ |
| 866 | #define __FIXADDR_TOP (-PAGE_SIZE) |
| 867 | #ifdef CONFIG_HIGHMEM |
| 868 | BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START); |
| 869 | BUILD_BUG_ON(VMALLOC_END > PKMAP_BASE); |
| 870 | #endif |
| 871 | #define high_memory (-128UL << 20) |
| 872 | BUILD_BUG_ON(VMALLOC_START >= VMALLOC_END); |
| 873 | #undef high_memory |
| 874 | #undef __FIXADDR_TOP |
| 875 | |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 876 | #ifdef CONFIG_HIGHMEM |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 877 | BUG_ON(PKMAP_BASE + LAST_PKMAP*PAGE_SIZE > FIXADDR_START); |
| 878 | BUG_ON(VMALLOC_END > PKMAP_BASE); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 879 | #endif |
Jan Beulich | beeb419 | 2008-12-16 11:45:56 +0000 | [diff] [blame] | 880 | BUG_ON(VMALLOC_START >= VMALLOC_END); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 881 | BUG_ON((unsigned long)high_memory > VMALLOC_START); |
Jeremy Fitzhardinge | 052e799 | 2006-09-25 23:32:25 -0700 | [diff] [blame] | 882 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 883 | if (boot_cpu_data.wp_works_ok < 0) |
| 884 | test_wp_bit(); |
| 885 | |
Hugh Dickins | 61165d7 | 2008-05-13 14:26:57 +0100 | [diff] [blame] | 886 | save_pg_dir(); |
Yinghai Lu | 55cd636 | 2009-06-12 11:36:52 +0300 | [diff] [blame] | 887 | zap_low_mappings(true); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 888 | } |
| 889 | |
KAMEZAWA Hiroyuki | ad8f579 | 2006-05-20 15:00:03 -0700 | [diff] [blame] | 890 | #ifdef CONFIG_MEMORY_HOTPLUG |
Yasunori Goto | bc02af9 | 2006-06-27 02:53:30 -0700 | [diff] [blame] | 891 | int arch_add_memory(int nid, u64 start, u64 size) |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 892 | { |
Yasunori Goto | 7c7e942 | 2006-12-22 01:11:13 -0800 | [diff] [blame] | 893 | struct pglist_data *pgdata = NODE_DATA(nid); |
Christoph Lameter | 776ed98 | 2006-09-25 23:31:09 -0700 | [diff] [blame] | 894 | struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM; |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 895 | unsigned long start_pfn = start >> PAGE_SHIFT; |
| 896 | unsigned long nr_pages = size >> PAGE_SHIFT; |
| 897 | |
Gary Hade | c04fc58 | 2009-01-06 14:39:14 -0800 | [diff] [blame] | 898 | return __add_pages(nid, zone, start_pfn, nr_pages); |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 899 | } |
Andi Kleen | 9d99aaa | 2006-04-07 19:49:15 +0200 | [diff] [blame] | 900 | #endif |
Dave Hansen | 05039b9 | 2005-10-29 18:16:57 -0700 | [diff] [blame] | 901 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 902 | /* |
| 903 | * This function cannot be __init, since exceptions don't work in that |
| 904 | * section. Put this after the callers, so that it cannot be inlined. |
| 905 | */ |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 906 | static noinline int do_test_wp_bit(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 907 | { |
| 908 | char tmp_reg; |
| 909 | int flag; |
| 910 | |
| 911 | __asm__ __volatile__( |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 912 | " movb %0, %1 \n" |
| 913 | "1: movb %1, %0 \n" |
| 914 | " xorl %2, %2 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 915 | "2: \n" |
H. Peter Anvin | f832ff1 | 2008-02-04 16:47:58 +0100 | [diff] [blame] | 916 | _ASM_EXTABLE(1b,2b) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 917 | :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)), |
| 918 | "=q" (tmp_reg), |
| 919 | "=r" (flag) |
| 920 | :"2" (1) |
| 921 | :"memory"); |
Ingo Molnar | 8550eb9 | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 922 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 923 | return flag; |
| 924 | } |
| 925 | |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 926 | #ifdef CONFIG_DEBUG_RODATA |
Arjan van de Ven | edeed30 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 927 | const int rodata_test_data = 0xC3; |
| 928 | EXPORT_SYMBOL_GPL(rodata_test_data); |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 929 | |
Suresh Siddha | 502f660 | 2009-10-28 18:46:56 -0800 | [diff] [blame] | 930 | int kernel_set_to_readonly __read_mostly; |
Steven Rostedt | 1623963 | 2009-02-17 17:57:30 -0500 | [diff] [blame] | 931 | |
| 932 | void set_kernel_text_rw(void) |
| 933 | { |
| 934 | unsigned long start = PFN_ALIGN(_text); |
| 935 | unsigned long size = PFN_ALIGN(_etext) - start; |
| 936 | |
| 937 | if (!kernel_set_to_readonly) |
| 938 | return; |
| 939 | |
| 940 | pr_debug("Set kernel text: %lx - %lx for read write\n", |
| 941 | start, start+size); |
| 942 | |
| 943 | set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); |
| 944 | } |
| 945 | |
| 946 | void set_kernel_text_ro(void) |
| 947 | { |
| 948 | unsigned long start = PFN_ALIGN(_text); |
| 949 | unsigned long size = PFN_ALIGN(_etext) - start; |
| 950 | |
| 951 | if (!kernel_set_to_readonly) |
| 952 | return; |
| 953 | |
| 954 | pr_debug("Set kernel text: %lx - %lx for read only\n", |
| 955 | start, start+size); |
| 956 | |
| 957 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| 958 | } |
| 959 | |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 960 | void mark_rodata_ro(void) |
| 961 | { |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 962 | unsigned long start = PFN_ALIGN(_text); |
| 963 | unsigned long size = PFN_ALIGN(_etext) - start; |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 964 | |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 965 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
| 966 | printk(KERN_INFO "Write protecting the kernel text: %luk\n", |
| 967 | size >> 10); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 968 | |
Steven Rostedt | 1623963 | 2009-02-17 17:57:30 -0500 | [diff] [blame] | 969 | kernel_set_to_readonly = 1; |
| 970 | |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 971 | #ifdef CONFIG_CPA_DEBUG |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 972 | printk(KERN_INFO "Testing CPA: Reverting %lx-%lx\n", |
| 973 | start, start+size); |
| 974 | set_pages_rw(virt_to_page(start), size>>PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 975 | |
Mathieu Desnoyers | 4e4eee0 | 2008-02-02 15:42:20 -0500 | [diff] [blame] | 976 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
| 977 | set_pages_ro(virt_to_page(start), size>>PAGE_SHIFT); |
Linus Torvalds | 602033e | 2007-07-26 12:07:21 -0700 | [diff] [blame] | 978 | #endif |
Steven Rostedt | 8f0f996 | 2008-05-12 21:20:56 +0200 | [diff] [blame] | 979 | |
Jan Beulich | 6fb1475 | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 980 | start += size; |
| 981 | size = (unsigned long)__end_rodata - start; |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 982 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 983 | printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", |
| 984 | size >> 10); |
Arjan van de Ven | edeed30 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 985 | rodata_test(); |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 986 | |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 987 | #ifdef CONFIG_CPA_DEBUG |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 988 | printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, start + size); |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 989 | set_pages_rw(virt_to_page(start), size >> PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 990 | |
Ingo Molnar | d7d119d | 2008-01-30 13:34:10 +0100 | [diff] [blame] | 991 | printk(KERN_INFO "Testing CPA: write protecting again\n"); |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 992 | set_pages_ro(virt_to_page(start), size >> PAGE_SHIFT); |
Andi Kleen | 0c42f39 | 2008-01-30 13:33:42 +0100 | [diff] [blame] | 993 | #endif |
Arjan van de Ven | 63aaf30 | 2006-01-06 00:12:02 -0800 | [diff] [blame] | 994 | } |
| 995 | #endif |
Yinghai Lu | a9ce6bc | 2010-08-25 13:39:17 -0700 | [diff] [blame] | 996 | |