Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * Thanks to Ben LaHaise for precious feedback. |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 4 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | #include <linux/highmem.h> |
Ingo Molnar | 8192206 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 6 | #include <linux/bootmem.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | #include <linux/module.h> |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 8 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | #include <linux/slab.h> |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 10 | #include <linux/mm.h> |
| 11 | |
Thomas Gleixner | 950f9d9 | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 12 | #include <asm/e820.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <asm/processor.h> |
| 14 | #include <asm/tlbflush.h> |
Dave Jones | f8af095 | 2006-01-06 00:12:10 -0800 | [diff] [blame] | 15 | #include <asm/sections.h> |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 16 | #include <asm/uaccess.h> |
| 17 | #include <asm/pgalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | |
Arjan van de Ven | ed724be | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 19 | static inline int |
| 20 | within(unsigned long addr, unsigned long start, unsigned long end) |
Ingo Molnar | 687c482 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 21 | { |
Arjan van de Ven | ed724be | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 22 | return addr >= start && addr < end; |
| 23 | } |
| 24 | |
| 25 | /* |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 26 | * Flushing functions |
| 27 | */ |
Thomas Gleixner | cd8ddf1 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 28 | |
Thomas Gleixner | cd8ddf1 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 29 | /** |
| 30 | * clflush_cache_range - flush a cache range with clflush |
| 31 | * @addr: virtual start address |
| 32 | * @size: number of bytes to flush |
| 33 | * |
| 34 | * clflush is an unordered instruction which needs fencing with mfence |
| 35 | * to avoid ordering issues. |
| 36 | */ |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 37 | void clflush_cache_range(void *vaddr, unsigned int size) |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 38 | { |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 39 | void *vend = vaddr + size - 1; |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 40 | |
Thomas Gleixner | cd8ddf1 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 41 | mb(); |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 42 | |
| 43 | for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size) |
| 44 | clflush(vaddr); |
| 45 | /* |
| 46 | * Flush any possible final partial cacheline: |
| 47 | */ |
| 48 | clflush(vend); |
| 49 | |
Thomas Gleixner | cd8ddf1 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 50 | mb(); |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 51 | } |
| 52 | |
Thomas Gleixner | af1e684 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 53 | static void __cpa_flush_all(void *arg) |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 54 | { |
| 55 | /* |
| 56 | * Flush all to work around Errata in early athlons regarding |
| 57 | * large page flushing. |
| 58 | */ |
| 59 | __flush_tlb_all(); |
| 60 | |
| 61 | if (boot_cpu_data.x86_model >= 4) |
| 62 | wbinvd(); |
| 63 | } |
| 64 | |
Thomas Gleixner | af1e684 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 65 | static void cpa_flush_all(void) |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 66 | { |
| 67 | BUG_ON(irqs_disabled()); |
| 68 | |
Thomas Gleixner | af1e684 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 69 | on_each_cpu(__cpa_flush_all, NULL, 1, 1); |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 70 | } |
| 71 | |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 72 | static void __cpa_flush_range(void *arg) |
| 73 | { |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 74 | /* |
| 75 | * We could optimize that further and do individual per page |
| 76 | * tlb invalidates for a low number of pages. Caveat: we must |
| 77 | * flush the high aliases on 64bit as well. |
| 78 | */ |
| 79 | __flush_tlb_all(); |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 80 | } |
| 81 | |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 82 | static void cpa_flush_range(unsigned long start, int numpages) |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 83 | { |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 84 | unsigned int i, level; |
| 85 | unsigned long addr; |
| 86 | |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 87 | BUG_ON(irqs_disabled()); |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 88 | WARN_ON(PAGE_ALIGN(start) != start); |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 89 | |
Thomas Gleixner | 3b233e5 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 90 | on_each_cpu(__cpa_flush_range, NULL, 1, 1); |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 91 | |
Thomas Gleixner | 3b233e5 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 92 | /* |
| 93 | * We only need to flush on one CPU, |
| 94 | * clflush is a MESI-coherent instruction that |
| 95 | * will cause all other CPUs to flush the same |
| 96 | * cachelines: |
| 97 | */ |
Ingo Molnar | 4c61afc | 2008-01-30 13:34:09 +0100 | [diff] [blame] | 98 | for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) { |
| 99 | pte_t *pte = lookup_address(addr, &level); |
| 100 | |
| 101 | /* |
| 102 | * Only flush present addresses: |
| 103 | */ |
| 104 | if (pte && pte_present(*pte)) |
| 105 | clflush_cache_range((void *) addr, PAGE_SIZE); |
| 106 | } |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 107 | } |
| 108 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 109 | /* |
Arjan van de Ven | ed724be | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 110 | * Certain areas of memory on x86 require very specific protection flags, |
| 111 | * for example the BIOS area or kernel text. Callers don't always get this |
| 112 | * right (again, ioremap() on BIOS memory is not uncommon) so this function |
| 113 | * checks and fixes these known static required protection bits. |
| 114 | */ |
| 115 | static inline pgprot_t static_protections(pgprot_t prot, unsigned long address) |
| 116 | { |
| 117 | pgprot_t forbidden = __pgprot(0); |
| 118 | |
Ingo Molnar | 687c482 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 119 | /* |
Arjan van de Ven | ed724be | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 120 | * The BIOS area between 640k and 1Mb needs to be executable for |
| 121 | * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support. |
Ingo Molnar | 687c482 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 122 | */ |
Arjan van de Ven | ed724be | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 123 | if (within(__pa(address), BIOS_BEGIN, BIOS_END)) |
| 124 | pgprot_val(forbidden) |= _PAGE_NX; |
| 125 | |
| 126 | /* |
| 127 | * The kernel text needs to be executable for obvious reasons |
| 128 | * Does not cover __inittext since that is gone later on |
| 129 | */ |
| 130 | if (within(address, (unsigned long)_text, (unsigned long)_etext)) |
| 131 | pgprot_val(forbidden) |= _PAGE_NX; |
| 132 | |
| 133 | #ifdef CONFIG_DEBUG_RODATA |
| 134 | /* The .rodata section needs to be read-only */ |
| 135 | if (within(address, (unsigned long)__start_rodata, |
| 136 | (unsigned long)__end_rodata)) |
| 137 | pgprot_val(forbidden) |= _PAGE_RW; |
| 138 | #endif |
| 139 | |
| 140 | prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden)); |
Ingo Molnar | 687c482 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 141 | |
| 142 | return prot; |
| 143 | } |
| 144 | |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 145 | pte_t *lookup_address(unsigned long address, int *level) |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 146 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | pgd_t *pgd = pgd_offset_k(address); |
| 148 | pud_t *pud; |
| 149 | pmd_t *pmd; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 150 | |
Thomas Gleixner | 30551bb | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 151 | *level = PG_LEVEL_NONE; |
| 152 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | if (pgd_none(*pgd)) |
| 154 | return NULL; |
| 155 | pud = pud_offset(pgd, address); |
| 156 | if (pud_none(*pud)) |
| 157 | return NULL; |
| 158 | pmd = pmd_offset(pud, address); |
| 159 | if (pmd_none(*pmd)) |
| 160 | return NULL; |
Thomas Gleixner | 30551bb | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 161 | |
| 162 | *level = PG_LEVEL_2M; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 163 | if (pmd_large(*pmd)) |
| 164 | return (pte_t *)pmd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | |
Thomas Gleixner | 30551bb | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 166 | *level = PG_LEVEL_4K; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 167 | return pte_offset_kernel(pmd, address); |
| 168 | } |
| 169 | |
Ingo Molnar | 9a3dc78 | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 170 | static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 171 | { |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 172 | /* change init_mm */ |
| 173 | set_pte_atomic(kpte, pte); |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 174 | #ifdef CONFIG_X86_32 |
Ingo Molnar | e4b71dc | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 175 | if (!SHARED_KERNEL_PMD) { |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 176 | struct page *page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 178 | for (page = pgd_list; page; page = (struct page *)page->index) { |
| 179 | pgd_t *pgd; |
| 180 | pud_t *pud; |
| 181 | pmd_t *pmd; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 182 | |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 183 | pgd = (pgd_t *)page_address(page) + pgd_index(address); |
| 184 | pud = pud_offset(pgd, address); |
| 185 | pmd = pmd_offset(pud, address); |
| 186 | set_pte_atomic((pte_t *)pmd, pte); |
| 187 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | } |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 189 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | } |
| 191 | |
Ingo Molnar | 7afe15b | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 192 | static int split_large_page(pte_t *kpte, unsigned long address) |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 193 | { |
Ingo Molnar | 7afe15b | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 194 | pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte)); |
Ingo Molnar | 12d6f21 | 2008-01-30 13:33:58 +0100 | [diff] [blame] | 195 | gfp_t gfp_flags = GFP_KERNEL; |
Ingo Molnar | 9a3dc78 | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 196 | unsigned long flags; |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 197 | unsigned long addr; |
| 198 | pte_t *pbase, *tmp; |
| 199 | struct page *base; |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 200 | unsigned int i, level; |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 201 | |
Ingo Molnar | 12d6f21 | 2008-01-30 13:33:58 +0100 | [diff] [blame] | 202 | #ifdef CONFIG_DEBUG_PAGEALLOC |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 203 | gfp_flags = __GFP_HIGH | __GFP_NOFAIL | __GFP_NOWARN; |
| 204 | gfp_flags = GFP_ATOMIC | __GFP_NOWARN; |
Ingo Molnar | 12d6f21 | 2008-01-30 13:33:58 +0100 | [diff] [blame] | 205 | #endif |
| 206 | base = alloc_pages(gfp_flags, 0); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 207 | if (!base) |
| 208 | return -ENOMEM; |
| 209 | |
Ingo Molnar | 9a3dc78 | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 210 | spin_lock_irqsave(&pgd_lock, flags); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 211 | /* |
| 212 | * Check for races, another CPU might have split this page |
| 213 | * up for us already: |
| 214 | */ |
| 215 | tmp = lookup_address(address, &level); |
Ingo Molnar | 5508a748 | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 216 | if (tmp != kpte) { |
| 217 | WARN_ON_ONCE(1); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 218 | goto out_unlock; |
Ingo Molnar | 5508a748 | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 219 | } |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 220 | |
| 221 | address = __pa(address); |
| 222 | addr = address & LARGE_PAGE_MASK; |
| 223 | pbase = (pte_t *)page_address(base); |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 224 | #ifdef CONFIG_X86_32 |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 225 | paravirt_alloc_pt(&init_mm, page_to_pfn(base)); |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 226 | #endif |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 227 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 228 | pgprot_val(ref_prot) &= ~_PAGE_NX; |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 229 | for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) |
| 230 | set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot)); |
| 231 | |
| 232 | /* |
Huang, Ying | 4c881ca | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 233 | * Install the new, split up pagetable. Important detail here: |
| 234 | * |
| 235 | * On Intel the NX bit of all levels must be cleared to make a |
| 236 | * page executable. See section 4.13.2 of Intel 64 and IA-32 |
| 237 | * Architectures Software Developer's Manual). |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 238 | */ |
Huang, Ying | 4c881ca | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 239 | ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte))); |
Ingo Molnar | 9a3dc78 | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 240 | __set_pmd_pte(kpte, address, mk_pte(base, ref_prot)); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 241 | base = NULL; |
| 242 | |
| 243 | out_unlock: |
Ingo Molnar | 9a3dc78 | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 244 | spin_unlock_irqrestore(&pgd_lock, flags); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 245 | |
| 246 | if (base) |
| 247 | __free_pages(base, 0); |
| 248 | |
| 249 | return 0; |
| 250 | } |
| 251 | |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 252 | static int |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 253 | __change_page_attr(unsigned long address, unsigned long pfn, |
| 254 | pgprot_t mask_set, pgprot_t mask_clr) |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 255 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | struct page *kpte_page; |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 257 | int level, err = 0; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 258 | pte_t *kpte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | |
Ingo Molnar | 8192206 | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 260 | #ifdef CONFIG_X86_32 |
| 261 | BUG_ON(pfn > max_low_pfn); |
| 262 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | |
Ingo Molnar | 97f99fe | 2008-01-30 13:33:55 +0100 | [diff] [blame] | 264 | repeat: |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 265 | kpte = lookup_address(address, &level); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | if (!kpte) |
| 267 | return -EINVAL; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 268 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | kpte_page = virt_to_page(kpte); |
Andi Kleen | 65d2f0b | 2007-07-21 17:09:51 +0200 | [diff] [blame] | 270 | BUG_ON(PageLRU(kpte_page)); |
| 271 | BUG_ON(PageCompound(kpte_page)); |
| 272 | |
Thomas Gleixner | 30551bb | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 273 | if (level == PG_LEVEL_4K) { |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 274 | pgprot_t new_prot = pte_pgprot(*kpte); |
| 275 | pte_t new_pte, old_pte = *kpte; |
Thomas Gleixner | a72a08a | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 276 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 277 | pgprot_val(new_prot) &= ~pgprot_val(mask_clr); |
| 278 | pgprot_val(new_prot) |= pgprot_val(mask_set); |
| 279 | |
| 280 | new_prot = static_protections(new_prot, address); |
| 281 | |
| 282 | new_pte = pfn_pte(pfn, canon_pgprot(new_prot)); |
| 283 | BUG_ON(pte_pfn(new_pte) != pte_pfn(old_pte)); |
| 284 | |
| 285 | set_pte_atomic(kpte, new_pte); |
| 286 | } else { |
Ingo Molnar | 7afe15b | 2008-01-30 13:33:57 +0100 | [diff] [blame] | 287 | err = split_large_page(kpte, address); |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 288 | if (!err) |
| 289 | goto repeat; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 290 | } |
Ingo Molnar | bb5c2db | 2008-01-30 13:33:56 +0100 | [diff] [blame] | 291 | return err; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 292 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 293 | |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 294 | /** |
| 295 | * change_page_attr_addr - Change page table attributes in linear mapping |
| 296 | * @address: Virtual address in linear mapping. |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 297 | * @prot: New page table attribute (PAGE_*) |
| 298 | * |
| 299 | * Change page attributes of a page in the direct mapping. This is a variant |
| 300 | * of change_page_attr() that also works on memory holes that do not have |
| 301 | * mem_map entry (pfn_valid() is false). |
| 302 | * |
| 303 | * See change_page_attr() documentation for more details. |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 304 | * |
| 305 | * Modules and drivers should use the set_memory_* APIs instead. |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 306 | */ |
| 307 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 308 | static int |
| 309 | change_page_attr_addr(unsigned long address, pgprot_t mask_set, |
| 310 | pgprot_t mask_clr) |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 311 | { |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 312 | int err = 0, kernel_map = 0; |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 313 | unsigned long pfn; |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 314 | |
| 315 | #ifdef CONFIG_X86_64 |
| 316 | if (address >= __START_KERNEL_map && |
| 317 | address < __START_KERNEL_map + KERNEL_TEXT_SIZE) { |
| 318 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 319 | address = (unsigned long)__va(__pa((void *)address)); |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 320 | kernel_map = 1; |
| 321 | } |
| 322 | #endif |
| 323 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 324 | pfn = __pa(address) >> PAGE_SHIFT; |
| 325 | |
| 326 | if (!kernel_map || 1) { |
| 327 | err = __change_page_attr(address, pfn, mask_set, mask_clr); |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 328 | if (err) |
| 329 | return err; |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 330 | } |
| 331 | |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 332 | #ifdef CONFIG_X86_64 |
| 333 | /* |
| 334 | * Handle kernel mapping too which aliases part of |
| 335 | * lowmem: |
| 336 | */ |
| 337 | if (__pa(address) < KERNEL_TEXT_SIZE) { |
| 338 | unsigned long addr2; |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 339 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 340 | addr2 = __pa(address) + __START_KERNEL_map - phys_base; |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 341 | /* Make sure the kernel mappings stay executable */ |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 342 | pgprot_val(mask_clr) |= _PAGE_NX; |
| 343 | /* |
| 344 | * Our high aliases are imprecise, so do not propagate |
| 345 | * failures back to users: |
| 346 | */ |
| 347 | __change_page_attr(addr2, pfn, mask_set, mask_clr); |
Arjan van de Ven | 488fd99 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 348 | } |
| 349 | #endif |
| 350 | |
Ingo Molnar | 44af6c4 | 2008-01-30 13:34:03 +0100 | [diff] [blame] | 351 | return err; |
| 352 | } |
| 353 | |
Thomas Gleixner | ff31452 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 354 | static int __change_page_attr_set_clr(unsigned long addr, int numpages, |
| 355 | pgprot_t mask_set, pgprot_t mask_clr) |
| 356 | { |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 357 | unsigned int i; |
| 358 | int ret; |
Thomas Gleixner | ff31452 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 359 | |
Ingo Molnar | 86f0398 | 2008-01-30 13:34:09 +0100 | [diff] [blame^] | 360 | for (i = 0; i < numpages ; i++, addr += PAGE_SIZE) { |
| 361 | ret = change_page_attr_addr(addr, mask_set, mask_clr); |
Thomas Gleixner | ff31452 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 362 | if (ret) |
| 363 | return ret; |
Thomas Gleixner | ff31452 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 364 | } |
| 365 | |
| 366 | return 0; |
| 367 | } |
| 368 | |
| 369 | static int change_page_attr_set_clr(unsigned long addr, int numpages, |
| 370 | pgprot_t mask_set, pgprot_t mask_clr) |
| 371 | { |
| 372 | int ret = __change_page_attr_set_clr(addr, numpages, mask_set, |
| 373 | mask_clr); |
| 374 | |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 375 | /* |
| 376 | * On success we use clflush, when the CPU supports it to |
| 377 | * avoid the wbindv. If the CPU does not support it and in the |
Thomas Gleixner | af1e684 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 378 | * error case we fall back to cpa_flush_all (which uses |
Thomas Gleixner | 57a6a46 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 379 | * wbindv): |
| 380 | */ |
| 381 | if (!ret && cpu_has_clflush) |
| 382 | cpa_flush_range(addr, numpages); |
| 383 | else |
Thomas Gleixner | af1e684 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 384 | cpa_flush_all(); |
Thomas Gleixner | ff31452 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 385 | |
| 386 | return ret; |
| 387 | } |
| 388 | |
Thomas Gleixner | 5674454 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 389 | static inline int change_page_attr_set(unsigned long addr, int numpages, |
| 390 | pgprot_t mask) |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 391 | { |
Thomas Gleixner | 5674454 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 392 | return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 393 | } |
| 394 | |
Thomas Gleixner | 5674454 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 395 | static inline int change_page_attr_clear(unsigned long addr, int numpages, |
| 396 | pgprot_t mask) |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 397 | { |
Thomas Gleixner | 5674454 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 398 | return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask); |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 399 | |
| 400 | } |
| 401 | |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 402 | int set_memory_uc(unsigned long addr, int numpages) |
| 403 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 404 | return change_page_attr_set(addr, numpages, |
| 405 | __pgprot(_PAGE_PCD | _PAGE_PWT)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 406 | } |
| 407 | EXPORT_SYMBOL(set_memory_uc); |
| 408 | |
| 409 | int set_memory_wb(unsigned long addr, int numpages) |
| 410 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 411 | return change_page_attr_clear(addr, numpages, |
| 412 | __pgprot(_PAGE_PCD | _PAGE_PWT)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 413 | } |
| 414 | EXPORT_SYMBOL(set_memory_wb); |
| 415 | |
| 416 | int set_memory_x(unsigned long addr, int numpages) |
| 417 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 418 | return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_NX)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 419 | } |
| 420 | EXPORT_SYMBOL(set_memory_x); |
| 421 | |
| 422 | int set_memory_nx(unsigned long addr, int numpages) |
| 423 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 424 | return change_page_attr_set(addr, numpages, __pgprot(_PAGE_NX)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 425 | } |
| 426 | EXPORT_SYMBOL(set_memory_nx); |
| 427 | |
| 428 | int set_memory_ro(unsigned long addr, int numpages) |
| 429 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 430 | return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 431 | } |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 432 | |
| 433 | int set_memory_rw(unsigned long addr, int numpages) |
| 434 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 435 | return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW)); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 436 | } |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 437 | |
| 438 | int set_memory_np(unsigned long addr, int numpages) |
| 439 | { |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 440 | return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_PRESENT)); |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 441 | } |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 442 | |
| 443 | int set_pages_uc(struct page *page, int numpages) |
| 444 | { |
| 445 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 446 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 447 | return set_memory_uc(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 448 | } |
| 449 | EXPORT_SYMBOL(set_pages_uc); |
| 450 | |
| 451 | int set_pages_wb(struct page *page, int numpages) |
| 452 | { |
| 453 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 454 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 455 | return set_memory_wb(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 456 | } |
| 457 | EXPORT_SYMBOL(set_pages_wb); |
| 458 | |
| 459 | int set_pages_x(struct page *page, int numpages) |
| 460 | { |
| 461 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 462 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 463 | return set_memory_x(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 464 | } |
| 465 | EXPORT_SYMBOL(set_pages_x); |
| 466 | |
| 467 | int set_pages_nx(struct page *page, int numpages) |
| 468 | { |
| 469 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 470 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 471 | return set_memory_nx(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 472 | } |
| 473 | EXPORT_SYMBOL(set_pages_nx); |
| 474 | |
| 475 | int set_pages_ro(struct page *page, int numpages) |
| 476 | { |
| 477 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 478 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 479 | return set_memory_ro(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 480 | } |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 481 | |
| 482 | int set_pages_rw(struct page *page, int numpages) |
| 483 | { |
| 484 | unsigned long addr = (unsigned long)page_address(page); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 485 | |
Thomas Gleixner | d7c8f21 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 486 | return set_memory_rw(addr, numpages); |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 487 | } |
Arjan van de Ven | 75cbade | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 488 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 489 | |
Thomas Gleixner | 5674454 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 490 | #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_CPA_DEBUG) |
| 491 | static inline int __change_page_attr_set(unsigned long addr, int numpages, |
| 492 | pgprot_t mask) |
| 493 | { |
| 494 | return __change_page_attr_set_clr(addr, numpages, mask, __pgprot(0)); |
| 495 | } |
| 496 | |
| 497 | static inline int __change_page_attr_clear(unsigned long addr, int numpages, |
| 498 | pgprot_t mask) |
| 499 | { |
| 500 | return __change_page_attr_set_clr(addr, numpages, __pgprot(0), mask); |
| 501 | } |
| 502 | #endif |
| 503 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 504 | #ifdef CONFIG_DEBUG_PAGEALLOC |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 505 | |
| 506 | static int __set_pages_p(struct page *page, int numpages) |
| 507 | { |
| 508 | unsigned long addr = (unsigned long)page_address(page); |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 509 | |
| 510 | return __change_page_attr_set(addr, numpages, |
| 511 | __pgprot(_PAGE_PRESENT | _PAGE_RW)); |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 512 | } |
| 513 | |
| 514 | static int __set_pages_np(struct page *page, int numpages) |
| 515 | { |
| 516 | unsigned long addr = (unsigned long)page_address(page); |
Thomas Gleixner | 72932c7 | 2008-01-30 13:34:08 +0100 | [diff] [blame] | 517 | |
| 518 | return __change_page_attr_clear(addr, numpages, |
| 519 | __pgprot(_PAGE_PRESENT)); |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 520 | } |
| 521 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 522 | void kernel_map_pages(struct page *page, int numpages, int enable) |
| 523 | { |
| 524 | if (PageHighMem(page)) |
| 525 | return; |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 526 | if (!enable) { |
Ingo Molnar | f9b8404 | 2006-06-27 02:54:49 -0700 | [diff] [blame] | 527 | debug_check_no_locks_freed(page_address(page), |
| 528 | numpages * PAGE_SIZE); |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 529 | } |
Ingo Molnar | de5097c | 2006-01-09 15:59:21 -0800 | [diff] [blame] | 530 | |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 531 | /* |
Ingo Molnar | 12d6f21 | 2008-01-30 13:33:58 +0100 | [diff] [blame] | 532 | * If page allocator is not up yet then do not call c_p_a(): |
| 533 | */ |
| 534 | if (!debug_pagealloc_enabled) |
| 535 | return; |
| 536 | |
| 537 | /* |
Ingo Molnar | e4b71dc | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 538 | * The return value is ignored - the calls cannot fail, |
| 539 | * large pages are disabled at boot time: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 540 | */ |
Ingo Molnar | f62d0f0 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 541 | if (enable) |
| 542 | __set_pages_p(page, numpages); |
| 543 | else |
| 544 | __set_pages_np(page, numpages); |
Ingo Molnar | 9f4c815 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 545 | |
| 546 | /* |
Ingo Molnar | e4b71dc | 2008-01-30 13:34:04 +0100 | [diff] [blame] | 547 | * We should perform an IPI and flush all tlbs, |
| 548 | * but that can deadlock->flush only current cpu: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 549 | */ |
| 550 | __flush_tlb_all(); |
| 551 | } |
| 552 | #endif |
Arjan van de Ven | d1028a1 | 2008-01-30 13:34:07 +0100 | [diff] [blame] | 553 | |
| 554 | /* |
| 555 | * The testcases use internal knowledge of the implementation that shouldn't |
| 556 | * be exposed to the rest of the kernel. Include these directly here. |
| 557 | */ |
| 558 | #ifdef CONFIG_CPA_DEBUG |
| 559 | #include "pageattr-test.c" |
| 560 | #endif |