Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * Thanks to Ben LaHaise for precious feedback. |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 4 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | #include <linux/highmem.h> |
| 7 | #include <linux/module.h> |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 8 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | #include <linux/slab.h> |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 10 | #include <linux/mm.h> |
| 11 | |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 12 | void clflush_cache_range(void *addr, int size) |
| 13 | { |
| 14 | int i; |
| 15 | |
| 16 | for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size) |
| 17 | clflush(addr+i); |
| 18 | } |
| 19 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <asm/processor.h> |
| 21 | #include <asm/tlbflush.h> |
Ingo Molnar | 5e5224a | 2008-01-30 13:34:00 +0100 | [diff] [blame^] | 22 | #include <asm/sections.h> |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 23 | #include <asm/uaccess.h> |
Ingo Molnar | 5e5224a | 2008-01-30 13:34:00 +0100 | [diff] [blame^] | 24 | #include <asm/pgalloc.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 26 | pte_t *lookup_address(unsigned long address, int *level) |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 27 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | pgd_t *pgd = pgd_offset_k(address); |
| 29 | pud_t *pud; |
| 30 | pmd_t *pmd; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 31 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | if (pgd_none(*pgd)) |
| 33 | return NULL; |
| 34 | pud = pud_offset(pgd, address); |
Ingo Molnar | a5f5503 | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 35 | if (pud_none(*pud)) |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 36 | return NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | pmd = pmd_offset(pud, address); |
Ingo Molnar | a5f5503 | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 38 | if (pmd_none(*pmd)) |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 39 | return NULL; |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 40 | *level = 3; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | if (pmd_large(*pmd)) |
| 42 | return (pte_t *)pmd; |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 43 | *level = 4; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 44 | |
Ingo Molnar | a5f5503 | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 45 | return pte_offset_kernel(pmd, address); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 46 | } |
| 47 | |
| 48 | static struct page * |
| 49 | split_large_page(unsigned long address, pgprot_t prot, pgprot_t ref_prot) |
| 50 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | unsigned long addr; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 52 | struct page *base; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | pte_t *pbase; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 54 | int i; |
| 55 | |
| 56 | base = alloc_pages(GFP_KERNEL, 0); |
| 57 | if (!base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | return NULL; |
Nick Piggin | 4fa4f53 | 2006-03-22 00:08:33 -0800 | [diff] [blame] | 59 | |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 60 | address = __pa(address); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 61 | addr = address & LARGE_PAGE_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | pbase = (pte_t *)page_address(base); |
| 63 | for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) { |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 64 | pbase[i] = pfn_pte(addr >> PAGE_SHIFT, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | addr == address ? prot : ref_prot); |
| 66 | } |
| 67 | return base; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 68 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | static int |
| 71 | __change_page_attr(unsigned long address, unsigned long pfn, pgprot_t prot, |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 72 | pgprot_t ref_prot) |
| 73 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | struct page *kpte_page; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 75 | pte_t *kpte; |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 76 | pgprot_t ref_prot2, oldprot; |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 77 | int level; |
Andi Kleen | 65d2f0b | 2007-07-21 17:09:51 +0200 | [diff] [blame] | 78 | |
Ingo Molnar | f0646e4 | 2008-01-30 13:33:43 +0100 | [diff] [blame] | 79 | kpte = lookup_address(address, &level); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 80 | if (!kpte) |
| 81 | return 0; |
| 82 | |
Andi Kleen | a3ae91b | 2008-01-30 13:33:50 +0100 | [diff] [blame] | 83 | kpte_page = virt_to_page(kpte); |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 84 | oldprot = pte_pgprot(*kpte); |
Andi Kleen | 65d2f0b | 2007-07-21 17:09:51 +0200 | [diff] [blame] | 85 | BUG_ON(PageLRU(kpte_page)); |
| 86 | BUG_ON(PageCompound(kpte_page)); |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 87 | ref_prot = canon_pgprot(ref_prot); |
| 88 | prot = canon_pgprot(prot); |
| 89 | |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 90 | if (pgprot_val(prot) != pgprot_val(ref_prot)) { |
Andi Kleen | 895bdc2 | 2008-01-30 13:33:52 +0100 | [diff] [blame] | 91 | if (level == 4) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | set_pte(kpte, pfn_pte(pfn, prot)); |
| 93 | } else { |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 94 | /* |
Nick Piggin | 4fa4f53 | 2006-03-22 00:08:33 -0800 | [diff] [blame] | 95 | * split_large_page will take the reference for this |
| 96 | * change_page_attr on the split page. |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 97 | */ |
Arjan van de Ven | c728252 | 2006-01-06 00:12:03 -0800 | [diff] [blame] | 98 | struct page *split; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 99 | |
Andi Kleen | 5e6b0bf | 2006-09-26 10:52:37 +0200 | [diff] [blame] | 100 | ref_prot2 = pte_pgprot(pte_clrhuge(*kpte)); |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 101 | split = split_large_page(address, prot, ref_prot2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | if (!split) |
| 103 | return -ENOMEM; |
Huang, Ying | 84e0fdb | 2007-10-17 18:04:35 +0200 | [diff] [blame] | 104 | pgprot_val(ref_prot2) &= ~_PAGE_NX; |
Andi Kleen | 5e6b0bf | 2006-09-26 10:52:37 +0200 | [diff] [blame] | 105 | set_pte(kpte, mk_pte(split, ref_prot2)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | kpte_page = split; |
Andi Kleen | 5e6b0bf | 2006-09-26 10:52:37 +0200 | [diff] [blame] | 107 | } |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 108 | } else { |
Andi Kleen | 895bdc2 | 2008-01-30 13:33:52 +0100 | [diff] [blame] | 109 | if (level == 4) { |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 110 | set_pte(kpte, pfn_pte(pfn, ref_prot)); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 111 | } else |
| 112 | BUG(); |
| 113 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 115 | return 0; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 116 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 118 | /** |
| 119 | * change_page_attr_addr - Change page table attributes in linear mapping |
| 120 | * @address: Virtual address in linear mapping. |
| 121 | * @numpages: Number of pages to change |
| 122 | * @prot: New page table attribute (PAGE_*) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | * |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 124 | * Change page attributes of a page in the direct mapping. This is a variant |
| 125 | * of change_page_attr() that also works on memory holes that do not have |
| 126 | * mem_map entry (pfn_valid() is false). |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 127 | * |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 128 | * See change_page_attr() documentation for more details. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | */ |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 130 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | int change_page_attr_addr(unsigned long address, int numpages, pgprot_t prot) |
| 132 | { |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 133 | int err = 0, kernel_map = 0, i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 135 | if (address >= __START_KERNEL_map && |
| 136 | address < __START_KERNEL_map + KERNEL_TEXT_SIZE) { |
| 137 | |
Jan Beulich | d01ad8d | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 138 | address = (unsigned long)__va(__pa(address)); |
| 139 | kernel_map = 1; |
| 140 | } |
| 141 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | down_write(&init_mm.mmap_sem); |
| 143 | for (i = 0; i < numpages; i++, address += PAGE_SIZE) { |
| 144 | unsigned long pfn = __pa(address) >> PAGE_SHIFT; |
| 145 | |
Jan Beulich | d01ad8d | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 146 | if (!kernel_map || pte_present(pfn_pte(0, prot))) { |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 147 | err = __change_page_attr(address, pfn, prot, |
| 148 | PAGE_KERNEL); |
Jan Beulich | d01ad8d | 2007-05-02 19:27:10 +0200 | [diff] [blame] | 149 | if (err) |
| 150 | break; |
| 151 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | /* Handle kernel mapping too which aliases part of the |
| 153 | * lowmem */ |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 154 | if (__pa(address) < KERNEL_TEXT_SIZE) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | unsigned long addr2; |
Andi Kleen | df992848 | 2006-09-26 10:52:37 +0200 | [diff] [blame] | 156 | pgprot_t prot2; |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 157 | |
Linus Torvalds | e3ebadd | 2007-05-07 08:44:24 -0700 | [diff] [blame] | 158 | addr2 = __START_KERNEL_map + __pa(address); |
Andi Kleen | df992848 | 2006-09-26 10:52:37 +0200 | [diff] [blame] | 159 | /* Make sure the kernel mappings stay executable */ |
| 160 | prot2 = pte_pgprot(pte_mkexec(pfn_pte(0, prot))); |
| 161 | err = __change_page_attr(addr2, pfn, prot2, |
| 162 | PAGE_KERNEL_EXEC); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 163 | } |
| 164 | } |
| 165 | up_write(&init_mm.mmap_sem); |
| 166 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 167 | return err; |
| 168 | } |
| 169 | |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 170 | /** |
| 171 | * change_page_attr - Change page table attributes in the linear mapping. |
| 172 | * @page: First page to change |
| 173 | * @numpages: Number of pages to change |
| 174 | * @prot: New protection/caching type (PAGE_*) |
| 175 | * |
| 176 | * Returns 0 on success, otherwise a negated errno. |
| 177 | * |
| 178 | * This should be used when a page is mapped with a different caching policy |
| 179 | * than write-back somewhere - some CPUs do not like it when mappings with |
| 180 | * different caching policies exist. This changes the page attributes of the |
| 181 | * in kernel linear mapping too. |
| 182 | * |
| 183 | * Caller must call global_flush_tlb() later to make the changes active. |
| 184 | * |
| 185 | * The caller needs to ensure that there are no conflicting mappings elsewhere |
| 186 | * (e.g. in user space) * This function only deals with the kernel linear map. |
| 187 | * |
| 188 | * For MMIO areas without mem_map use change_page_attr_addr() instead. |
| 189 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | int change_page_attr(struct page *page, int numpages, pgprot_t prot) |
| 191 | { |
| 192 | unsigned long addr = (unsigned long)page_address(page); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 193 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | return change_page_attr_addr(addr, numpages, prot); |
| 195 | } |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 196 | EXPORT_SYMBOL(change_page_attr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 198 | static void flush_kernel_map(void *arg) |
| 199 | { |
| 200 | /* |
| 201 | * Flush all to work around Errata in early athlons regarding |
| 202 | * large page flushing. |
| 203 | */ |
| 204 | __flush_tlb_all(); |
| 205 | |
| 206 | if (boot_cpu_data.x86_model >= 4) |
| 207 | wbinvd(); |
| 208 | } |
| 209 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | void global_flush_tlb(void) |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 211 | { |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 212 | BUG_ON(irqs_disabled()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 213 | |
Ingo Molnar | d6ee09a | 2008-01-30 13:33:59 +0100 | [diff] [blame] | 214 | on_each_cpu(flush_kernel_map, NULL, 1, 1); |
Ingo Molnar | b4416a1 | 2008-01-30 13:33:41 +0100 | [diff] [blame] | 215 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | EXPORT_SYMBOL(global_flush_tlb); |