blob: 985eef80c552007116348dc49b2562c41e390b05 [file] [log] [blame]
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001/*
2 * Copyright 2002 Andi Kleen, SuSE Labs.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 * Thanks to Ben LaHaise for precious feedback.
Ingo Molnar9f4c8152008-01-30 13:33:41 +01004 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/highmem.h>
Ingo Molnar81922062008-01-30 13:34:04 +01006#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/module.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +01008#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/slab.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +010010#include <linux/mm.h>
Thomas Gleixner76ebd052008-02-09 23:24:09 +010011#include <linux/interrupt.h>
Thomas Gleixneree7ae7a2008-04-17 17:40:45 +020012#include <linux/seq_file.h>
13#include <linux/debugfs.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +010014
Thomas Gleixner950f9d92008-01-30 13:34:06 +010015#include <asm/e820.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <asm/processor.h>
17#include <asm/tlbflush.h>
Dave Jonesf8af0952006-01-06 00:12:10 -080018#include <asm/sections.h>
Jeremy Fitzhardinge93dbda72009-02-26 17:35:44 -080019#include <asm/setup.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +010020#include <asm/uaccess.h>
21#include <asm/pgalloc.h>
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010022#include <asm/proto.h>
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -070023#include <asm/pat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Ingo Molnar9df84992008-02-04 16:48:09 +010025/*
26 * The current flushing context - we pass it instead of 5 arguments:
27 */
Thomas Gleixner72e458d2008-02-04 16:48:07 +010028struct cpa_data {
Shaohua Lid75586a2008-08-21 10:46:06 +080029 unsigned long *vaddr;
Thomas Gleixner72e458d2008-02-04 16:48:07 +010030 pgprot_t mask_set;
31 pgprot_t mask_clr;
Thomas Gleixner65e074d2008-02-04 16:48:07 +010032 int numpages;
Shaohua Lid75586a2008-08-21 10:46:06 +080033 int flags;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010034 unsigned long pfn;
Andi Kleenc9caa022008-03-12 03:53:29 +010035 unsigned force_split : 1;
Shaohua Lid75586a2008-08-21 10:46:06 +080036 int curpage;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -070037 struct page **pages;
Thomas Gleixner72e458d2008-02-04 16:48:07 +010038};
39
Suresh Siddhaad5ca552008-09-23 14:00:42 -070040/*
41 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
42 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
43 * entries change the page attribute in parallel to some other cpu
44 * splitting a large page entry along with changing the attribute.
45 */
46static DEFINE_SPINLOCK(cpa_lock);
47
Shaohua Lid75586a2008-08-21 10:46:06 +080048#define CPA_FLUSHTLB 1
49#define CPA_ARRAY 2
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -070050#define CPA_PAGES_ARRAY 4
Shaohua Lid75586a2008-08-21 10:46:06 +080051
Thomas Gleixner65280e62008-05-05 16:35:21 +020052#ifdef CONFIG_PROC_FS
Andi Kleence0c0e52008-05-02 11:46:49 +020053static unsigned long direct_pages_count[PG_LEVEL_NUM];
54
Thomas Gleixner65280e62008-05-05 16:35:21 +020055void update_page_count(int level, unsigned long pages)
Andi Kleence0c0e52008-05-02 11:46:49 +020056{
Andi Kleence0c0e52008-05-02 11:46:49 +020057 unsigned long flags;
Thomas Gleixner65280e62008-05-05 16:35:21 +020058
Andi Kleence0c0e52008-05-02 11:46:49 +020059 /* Protect against CPA */
60 spin_lock_irqsave(&pgd_lock, flags);
61 direct_pages_count[level] += pages;
62 spin_unlock_irqrestore(&pgd_lock, flags);
Andi Kleence0c0e52008-05-02 11:46:49 +020063}
64
Thomas Gleixner65280e62008-05-05 16:35:21 +020065static void split_page_count(int level)
66{
67 direct_pages_count[level]--;
68 direct_pages_count[level - 1] += PTRS_PER_PTE;
69}
70
Alexey Dobriyane1759c22008-10-15 23:50:22 +040071void arch_report_meminfo(struct seq_file *m)
Thomas Gleixner65280e62008-05-05 16:35:21 +020072{
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000073 seq_printf(m, "DirectMap4k: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010074 direct_pages_count[PG_LEVEL_4K] << 2);
75#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000076 seq_printf(m, "DirectMap2M: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010077 direct_pages_count[PG_LEVEL_2M] << 11);
78#else
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000079 seq_printf(m, "DirectMap4M: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010080 direct_pages_count[PG_LEVEL_2M] << 12);
81#endif
Thomas Gleixner65280e62008-05-05 16:35:21 +020082#ifdef CONFIG_X86_64
Hugh Dickinsa06de632008-08-15 13:58:32 +010083 if (direct_gbpages)
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000084 seq_printf(m, "DirectMap1G: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010085 direct_pages_count[PG_LEVEL_1G] << 20);
Thomas Gleixner65280e62008-05-05 16:35:21 +020086#endif
Thomas Gleixner65280e62008-05-05 16:35:21 +020087}
88#else
89static inline void split_page_count(int level) { }
90#endif
91
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010092#ifdef CONFIG_X86_64
93
94static inline unsigned long highmap_start_pfn(void)
95{
96 return __pa(_text) >> PAGE_SHIFT;
97}
98
99static inline unsigned long highmap_end_pfn(void)
100{
Jeremy Fitzhardinge93dbda72009-02-26 17:35:44 -0800101 return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100102}
103
104#endif
105
Ingo Molnar92cb54a2008-02-13 14:37:52 +0100106#ifdef CONFIG_DEBUG_PAGEALLOC
107# define debug_pagealloc 1
108#else
109# define debug_pagealloc 0
110#endif
111
Arjan van de Vened724be2008-01-30 13:34:04 +0100112static inline int
113within(unsigned long addr, unsigned long start, unsigned long end)
Ingo Molnar687c4822008-01-30 13:34:04 +0100114{
Arjan van de Vened724be2008-01-30 13:34:04 +0100115 return addr >= start && addr < end;
116}
117
118/*
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100119 * Flushing functions
120 */
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100121
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100122/**
123 * clflush_cache_range - flush a cache range with clflush
124 * @addr: virtual start address
125 * @size: number of bytes to flush
126 *
127 * clflush is an unordered instruction which needs fencing with mfence
128 * to avoid ordering issues.
129 */
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100130void clflush_cache_range(void *vaddr, unsigned int size)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100131{
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100132 void *vend = vaddr + size - 1;
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100133
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100134 mb();
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100135
136 for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
137 clflush(vaddr);
138 /*
139 * Flush any possible final partial cacheline:
140 */
141 clflush(vend);
142
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100143 mb();
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100144}
145
Thomas Gleixneraf1e6842008-01-30 13:34:08 +0100146static void __cpa_flush_all(void *arg)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100147{
Andi Kleen6bb83832008-02-04 16:48:06 +0100148 unsigned long cache = (unsigned long)arg;
149
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100150 /*
151 * Flush all to work around Errata in early athlons regarding
152 * large page flushing.
153 */
154 __flush_tlb_all();
155
Andi Kleen6bb83832008-02-04 16:48:06 +0100156 if (cache && boot_cpu_data.x86_model >= 4)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100157 wbinvd();
158}
159
Andi Kleen6bb83832008-02-04 16:48:06 +0100160static void cpa_flush_all(unsigned long cache)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100161{
162 BUG_ON(irqs_disabled());
163
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200164 on_each_cpu(__cpa_flush_all, (void *) cache, 1);
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100165}
166
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100167static void __cpa_flush_range(void *arg)
168{
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100169 /*
170 * We could optimize that further and do individual per page
171 * tlb invalidates for a low number of pages. Caveat: we must
172 * flush the high aliases on 64bit as well.
173 */
174 __flush_tlb_all();
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100175}
176
Andi Kleen6bb83832008-02-04 16:48:06 +0100177static void cpa_flush_range(unsigned long start, int numpages, int cache)
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100178{
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100179 unsigned int i, level;
180 unsigned long addr;
181
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100182 BUG_ON(irqs_disabled());
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100183 WARN_ON(PAGE_ALIGN(start) != start);
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100184
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200185 on_each_cpu(__cpa_flush_range, NULL, 1);
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100186
Andi Kleen6bb83832008-02-04 16:48:06 +0100187 if (!cache)
188 return;
189
Thomas Gleixner3b233e52008-01-30 13:34:08 +0100190 /*
191 * We only need to flush on one CPU,
192 * clflush is a MESI-coherent instruction that
193 * will cause all other CPUs to flush the same
194 * cachelines:
195 */
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100196 for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
197 pte_t *pte = lookup_address(addr, &level);
198
199 /*
200 * Only flush present addresses:
201 */
Thomas Gleixner7bfb72e2008-02-04 16:48:08 +0100202 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100203 clflush_cache_range((void *) addr, PAGE_SIZE);
204 }
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100205}
206
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700207static void cpa_flush_array(unsigned long *start, int numpages, int cache,
208 int in_flags, struct page **pages)
Shaohua Lid75586a2008-08-21 10:46:06 +0800209{
210 unsigned int i, level;
Shaohua Lid75586a2008-08-21 10:46:06 +0800211
212 BUG_ON(irqs_disabled());
213
214 on_each_cpu(__cpa_flush_range, NULL, 1);
215
216 if (!cache)
217 return;
218
219 /* 4M threshold */
220 if (numpages >= 1024) {
221 if (boot_cpu_data.x86_model >= 4)
222 wbinvd();
223 return;
224 }
225 /*
226 * We only need to flush on one CPU,
227 * clflush is a MESI-coherent instruction that
228 * will cause all other CPUs to flush the same
229 * cachelines:
230 */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700231 for (i = 0; i < numpages; i++) {
232 unsigned long addr;
233 pte_t *pte;
234
235 if (in_flags & CPA_PAGES_ARRAY)
236 addr = (unsigned long)page_address(pages[i]);
237 else
238 addr = start[i];
239
240 pte = lookup_address(addr, &level);
Shaohua Lid75586a2008-08-21 10:46:06 +0800241
242 /*
243 * Only flush present addresses:
244 */
245 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700246 clflush_cache_range((void *)addr, PAGE_SIZE);
Shaohua Lid75586a2008-08-21 10:46:06 +0800247 }
248}
249
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100250/*
Arjan van de Vened724be2008-01-30 13:34:04 +0100251 * Certain areas of memory on x86 require very specific protection flags,
252 * for example the BIOS area or kernel text. Callers don't always get this
253 * right (again, ioremap() on BIOS memory is not uncommon) so this function
254 * checks and fixes these known static required protection bits.
255 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100256static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
257 unsigned long pfn)
Arjan van de Vened724be2008-01-30 13:34:04 +0100258{
259 pgprot_t forbidden = __pgprot(0);
260
Ingo Molnar687c4822008-01-30 13:34:04 +0100261 /*
Arjan van de Vened724be2008-01-30 13:34:04 +0100262 * The BIOS area between 640k and 1Mb needs to be executable for
263 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
Ingo Molnar687c4822008-01-30 13:34:04 +0100264 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100265 if (within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
Arjan van de Vened724be2008-01-30 13:34:04 +0100266 pgprot_val(forbidden) |= _PAGE_NX;
267
268 /*
269 * The kernel text needs to be executable for obvious reasons
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100270 * Does not cover __inittext since that is gone later on. On
271 * 64bit we do not enforce !NX on the low mapping
Arjan van de Vened724be2008-01-30 13:34:04 +0100272 */
273 if (within(address, (unsigned long)_text, (unsigned long)_etext))
274 pgprot_val(forbidden) |= _PAGE_NX;
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100275
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100276 /*
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100277 * The .rodata section needs to be read-only. Using the pfn
278 * catches all aliases.
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100279 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100280 if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT,
281 __pa((unsigned long)__end_rodata) >> PAGE_SHIFT))
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100282 pgprot_val(forbidden) |= _PAGE_RW;
Arjan van de Vened724be2008-01-30 13:34:04 +0100283
284 prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
Ingo Molnar687c4822008-01-30 13:34:04 +0100285
286 return prot;
287}
288
Thomas Gleixner9a14aef2008-02-04 16:48:07 +0100289/*
290 * Lookup the page table entry for a virtual address. Return a pointer
291 * to the entry and the level of the mapping.
292 *
293 * Note: We return pud and pmd either when the entry is marked large
294 * or when the present bit is not set. Otherwise we would return a
295 * pointer to a nonexisting mapping.
296 */
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100297pte_t *lookup_address(unsigned long address, unsigned int *level)
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100298{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 pgd_t *pgd = pgd_offset_k(address);
300 pud_t *pud;
301 pmd_t *pmd;
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100302
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100303 *level = PG_LEVEL_NONE;
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 if (pgd_none(*pgd))
306 return NULL;
Ingo Molnar9df84992008-02-04 16:48:09 +0100307
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 pud = pud_offset(pgd, address);
309 if (pud_none(*pud))
310 return NULL;
Andi Kleenc2f71ee2008-02-04 16:48:09 +0100311
312 *level = PG_LEVEL_1G;
313 if (pud_large(*pud) || !pud_present(*pud))
314 return (pte_t *)pud;
315
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316 pmd = pmd_offset(pud, address);
317 if (pmd_none(*pmd))
318 return NULL;
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100319
320 *level = PG_LEVEL_2M;
Thomas Gleixner9a14aef2008-02-04 16:48:07 +0100321 if (pmd_large(*pmd) || !pmd_present(*pmd))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 return (pte_t *)pmd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100324 *level = PG_LEVEL_4K;
Ingo Molnar9df84992008-02-04 16:48:09 +0100325
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100326 return pte_offset_kernel(pmd, address);
327}
Pekka Paalanen75bb8832008-05-12 21:20:56 +0200328EXPORT_SYMBOL_GPL(lookup_address);
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100329
Ingo Molnar9df84992008-02-04 16:48:09 +0100330/*
331 * Set the new pmd in all the pgds we know about:
332 */
Ingo Molnar9a3dc782008-01-30 13:33:57 +0100333static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100334{
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100335 /* change init_mm */
336 set_pte_atomic(kpte, pte);
Ingo Molnar44af6c42008-01-30 13:34:03 +0100337#ifdef CONFIG_X86_32
Ingo Molnare4b71dc2008-01-30 13:34:04 +0100338 if (!SHARED_KERNEL_PMD) {
Ingo Molnar44af6c42008-01-30 13:34:03 +0100339 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
Jeremy Fitzhardingee3ed9102008-01-30 13:34:11 +0100341 list_for_each_entry(page, &pgd_list, lru) {
Ingo Molnar44af6c42008-01-30 13:34:03 +0100342 pgd_t *pgd;
343 pud_t *pud;
344 pmd_t *pmd;
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100345
Ingo Molnar44af6c42008-01-30 13:34:03 +0100346 pgd = (pgd_t *)page_address(page) + pgd_index(address);
347 pud = pud_offset(pgd, address);
348 pmd = pmd_offset(pud, address);
349 set_pte_atomic((pte_t *)pmd, pte);
350 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 }
Ingo Molnar44af6c42008-01-30 13:34:03 +0100352#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353}
354
Ingo Molnar9df84992008-02-04 16:48:09 +0100355static int
356try_preserve_large_page(pte_t *kpte, unsigned long address,
357 struct cpa_data *cpa)
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100358{
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100359 unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100360 pte_t new_pte, old_pte, *tmp;
361 pgprot_t old_prot, new_prot;
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100362 int i, do_split = 1;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100363 unsigned int level;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100364
Andi Kleenc9caa022008-03-12 03:53:29 +0100365 if (cpa->force_split)
366 return 1;
367
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100368 spin_lock_irqsave(&pgd_lock, flags);
369 /*
370 * Check for races, another CPU might have split this page
371 * up already:
372 */
373 tmp = lookup_address(address, &level);
374 if (tmp != kpte)
375 goto out_unlock;
376
377 switch (level) {
378 case PG_LEVEL_2M:
Andi Kleen31422c52008-02-04 16:48:08 +0100379 psize = PMD_PAGE_SIZE;
380 pmask = PMD_PAGE_MASK;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100381 break;
Andi Kleenf07333f2008-02-04 16:48:09 +0100382#ifdef CONFIG_X86_64
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100383 case PG_LEVEL_1G:
Andi Kleen5d3c8b22008-02-13 16:20:35 +0100384 psize = PUD_PAGE_SIZE;
385 pmask = PUD_PAGE_MASK;
Andi Kleenf07333f2008-02-04 16:48:09 +0100386 break;
387#endif
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100388 default:
Ingo Molnarbeaff632008-02-04 16:48:09 +0100389 do_split = -EINVAL;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100390 goto out_unlock;
391 }
392
393 /*
394 * Calculate the number of pages, which fit into this large
395 * page starting at address:
396 */
397 nextpage_addr = (address + psize) & pmask;
398 numpages = (nextpage_addr - address) >> PAGE_SHIFT;
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100399 if (numpages < cpa->numpages)
400 cpa->numpages = numpages;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100401
402 /*
403 * We are safe now. Check whether the new pgprot is the same:
404 */
405 old_pte = *kpte;
406 old_prot = new_prot = pte_pgprot(old_pte);
407
408 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
409 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100410
411 /*
412 * old_pte points to the large page base address. So we need
413 * to add the offset of the virtual address:
414 */
415 pfn = pte_pfn(old_pte) + ((address & (psize - 1)) >> PAGE_SHIFT);
416 cpa->pfn = pfn;
417
418 new_prot = static_protections(new_prot, address, pfn);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100419
420 /*
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100421 * We need to check the full range, whether
422 * static_protection() requires a different pgprot for one of
423 * the pages in the range we try to preserve:
424 */
425 addr = address + PAGE_SIZE;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100426 pfn++;
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100427 for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE, pfn++) {
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100428 pgprot_t chk_prot = static_protections(new_prot, addr, pfn);
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100429
430 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
431 goto out_unlock;
432 }
433
434 /*
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100435 * If there are no changes, return. maxpages has been updated
436 * above:
437 */
438 if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
Ingo Molnarbeaff632008-02-04 16:48:09 +0100439 do_split = 0;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100440 goto out_unlock;
441 }
442
443 /*
444 * We need to change the attributes. Check, whether we can
445 * change the large page in one go. We request a split, when
446 * the address is not aligned and the number of pages is
447 * smaller than the number of pages in the large page. Note
448 * that we limited the number of possible pages already to
449 * the number of pages in the large page.
450 */
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100451 if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100452 /*
453 * The address is aligned and the number of pages
454 * covers the full page.
455 */
456 new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
457 __set_pmd_pte(kpte, address, new_pte);
Shaohua Lid75586a2008-08-21 10:46:06 +0800458 cpa->flags |= CPA_FLUSHTLB;
Ingo Molnarbeaff632008-02-04 16:48:09 +0100459 do_split = 0;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100460 }
461
462out_unlock:
463 spin_unlock_irqrestore(&pgd_lock, flags);
Ingo Molnar9df84992008-02-04 16:48:09 +0100464
Ingo Molnarbeaff632008-02-04 16:48:09 +0100465 return do_split;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100466}
467
Ingo Molnar7afe15b2008-01-30 13:33:57 +0100468static int split_large_page(pte_t *kpte, unsigned long address)
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100469{
Thomas Gleixner7b610ee2008-02-04 16:48:10 +0100470 unsigned long flags, pfn, pfninc = 1;
Ingo Molnar86f03982008-01-30 13:34:09 +0100471 unsigned int i, level;
Ingo Molnar9df84992008-02-04 16:48:09 +0100472 pte_t *pbase, *tmp;
473 pgprot_t ref_prot;
Suresh Siddhaad5ca552008-09-23 14:00:42 -0700474 struct page *base;
475
476 if (!debug_pagealloc)
477 spin_unlock(&cpa_lock);
478 base = alloc_pages(GFP_KERNEL, 0);
479 if (!debug_pagealloc)
480 spin_lock(&cpa_lock);
Suresh Siddha8311eb82008-09-23 14:00:41 -0700481 if (!base)
482 return -ENOMEM;
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100483
Ingo Molnar9a3dc782008-01-30 13:33:57 +0100484 spin_lock_irqsave(&pgd_lock, flags);
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100485 /*
486 * Check for races, another CPU might have split this page
487 * up for us already:
488 */
489 tmp = lookup_address(address, &level);
Ingo Molnar6ce9fc12008-02-04 16:48:08 +0100490 if (tmp != kpte)
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100491 goto out_unlock;
492
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100493 pbase = (pte_t *)page_address(base);
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700494 paravirt_alloc_pte(&init_mm, page_to_pfn(base));
Thomas Gleixner07cf89c2008-02-04 16:48:08 +0100495 ref_prot = pte_pgprot(pte_clrhuge(*kpte));
Ingo Molnar7a5714e2009-02-20 17:44:21 +0100496 /*
497 * If we ever want to utilize the PAT bit, we need to
498 * update this function to make sure it's converted from
499 * bit 12 to bit 7 when we cross from the 2MB level to
500 * the 4K level:
501 */
502 WARN_ON_ONCE(pgprot_val(ref_prot) & _PAGE_PAT_LARGE);
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100503
Andi Kleenf07333f2008-02-04 16:48:09 +0100504#ifdef CONFIG_X86_64
505 if (level == PG_LEVEL_1G) {
506 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
507 pgprot_val(ref_prot) |= _PAGE_PSE;
Andi Kleenf07333f2008-02-04 16:48:09 +0100508 }
509#endif
510
Thomas Gleixner63c1dcf2008-02-04 16:48:05 +0100511 /*
512 * Get the target pfn from the original entry:
513 */
514 pfn = pte_pfn(*kpte);
Andi Kleenf07333f2008-02-04 16:48:09 +0100515 for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
Thomas Gleixner63c1dcf2008-02-04 16:48:05 +0100516 set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100517
Andi Kleence0c0e52008-05-02 11:46:49 +0200518 if (address >= (unsigned long)__va(0) &&
Yinghai Luf361a452008-07-10 20:38:26 -0700519 address < (unsigned long)__va(max_low_pfn_mapped << PAGE_SHIFT))
520 split_page_count(level);
521
522#ifdef CONFIG_X86_64
523 if (address >= (unsigned long)__va(1UL<<32) &&
Thomas Gleixner65280e62008-05-05 16:35:21 +0200524 address < (unsigned long)__va(max_pfn_mapped << PAGE_SHIFT))
525 split_page_count(level);
Yinghai Luf361a452008-07-10 20:38:26 -0700526#endif
Andi Kleence0c0e52008-05-02 11:46:49 +0200527
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100528 /*
Ingo Molnar07a66d72009-02-20 08:04:13 +0100529 * Install the new, split up pagetable.
Huang, Ying4c881ca2008-01-30 13:34:04 +0100530 *
Ingo Molnar07a66d72009-02-20 08:04:13 +0100531 * We use the standard kernel pagetable protections for the new
532 * pagetable protections, the actual ptes set above control the
533 * primary protection behavior:
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100534 */
Ingo Molnar07a66d72009-02-20 08:04:13 +0100535 __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
Ingo Molnar211b3d02009-03-10 22:31:03 +0100536
537 /*
538 * Intel Atom errata AAH41 workaround.
539 *
540 * The real fix should be in hw or in a microcode update, but
541 * we also probabilistically try to reduce the window of having
542 * a large TLB mixed with 4K TLBs while instruction fetches are
543 * going on.
544 */
545 __flush_tlb_all();
546
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100547 base = NULL;
548
549out_unlock:
Thomas Gleixnereb5b5f02008-02-09 23:24:09 +0100550 /*
551 * If we dropped out via the lookup_address check under
552 * pgd_lock then stick the page back into the pool:
553 */
Suresh Siddha8311eb82008-09-23 14:00:41 -0700554 if (base)
555 __free_page(base);
Ingo Molnar9a3dc782008-01-30 13:33:57 +0100556 spin_unlock_irqrestore(&pgd_lock, flags);
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100557
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100558 return 0;
559}
560
Suresh Siddhaa1e46212009-01-20 14:20:21 -0800561static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
562 int primary)
563{
564 /*
565 * Ignore all non primary paths.
566 */
567 if (!primary)
568 return 0;
569
570 /*
571 * Ignore the NULL PTE for kernel identity mapping, as it is expected
572 * to have holes.
573 * Also set numpages to '1' indicating that we processed cpa req for
574 * one virtual address page and its pfn. TBD: numpages can be set based
575 * on the initial value and the level returned by lookup_address().
576 */
577 if (within(vaddr, PAGE_OFFSET,
578 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
579 cpa->numpages = 1;
580 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
581 return 0;
582 } else {
583 WARN(1, KERN_WARNING "CPA: called for zero pte. "
584 "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
585 *cpa->vaddr);
586
587 return -EFAULT;
588 }
589}
590
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100591static int __change_page_attr(struct cpa_data *cpa, int primary)
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100592{
Shaohua Lid75586a2008-08-21 10:46:06 +0800593 unsigned long address;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +0100594 int do_split, err;
595 unsigned int level;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100596 pte_t *kpte, old_pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700598 if (cpa->flags & CPA_PAGES_ARRAY)
599 address = (unsigned long)page_address(cpa->pages[cpa->curpage]);
600 else if (cpa->flags & CPA_ARRAY)
Shaohua Lid75586a2008-08-21 10:46:06 +0800601 address = cpa->vaddr[cpa->curpage];
602 else
603 address = *cpa->vaddr;
Ingo Molnar97f99fe2008-01-30 13:33:55 +0100604repeat:
Ingo Molnarf0646e42008-01-30 13:33:43 +0100605 kpte = lookup_address(address, &level);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 if (!kpte)
Suresh Siddhaa1e46212009-01-20 14:20:21 -0800607 return __cpa_process_fault(cpa, address, primary);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100608
609 old_pte = *kpte;
Suresh Siddhaa1e46212009-01-20 14:20:21 -0800610 if (!pte_val(old_pte))
611 return __cpa_process_fault(cpa, address, primary);
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100612
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100613 if (level == PG_LEVEL_4K) {
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100614 pte_t new_pte;
Arjan van de Ven626c2c92008-02-04 16:48:05 +0100615 pgprot_t new_prot = pte_pgprot(old_pte);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100616 unsigned long pfn = pte_pfn(old_pte);
Thomas Gleixnera72a08a2008-01-30 13:34:07 +0100617
Thomas Gleixner72e458d2008-02-04 16:48:07 +0100618 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
619 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
Ingo Molnar86f03982008-01-30 13:34:09 +0100620
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100621 new_prot = static_protections(new_prot, address, pfn);
Ingo Molnar86f03982008-01-30 13:34:09 +0100622
Arjan van de Ven626c2c92008-02-04 16:48:05 +0100623 /*
624 * We need to keep the pfn from the existing PTE,
625 * after all we're only going to change it's attributes
626 * not the memory it points to
627 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100628 new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
629 cpa->pfn = pfn;
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +0100630 /*
631 * Do we really change anything ?
632 */
633 if (pte_val(old_pte) != pte_val(new_pte)) {
634 set_pte_atomic(kpte, new_pte);
Shaohua Lid75586a2008-08-21 10:46:06 +0800635 cpa->flags |= CPA_FLUSHTLB;
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +0100636 }
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100637 cpa->numpages = 1;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100638 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 }
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100640
641 /*
642 * Check, whether we can keep the large page intact
643 * and just change the pte:
644 */
Ingo Molnarbeaff632008-02-04 16:48:09 +0100645 do_split = try_preserve_large_page(kpte, address, cpa);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100646 /*
647 * When the range fits into the existing large page,
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100648 * return. cp->numpages and cpa->tlbflush have been updated in
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100649 * try_large_page:
650 */
Ingo Molnar87f7f8f2008-02-04 16:48:10 +0100651 if (do_split <= 0)
652 return do_split;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100653
654 /*
655 * We have to split the large page:
656 */
Ingo Molnar87f7f8f2008-02-04 16:48:10 +0100657 err = split_large_page(kpte, address);
658 if (!err) {
Suresh Siddhaad5ca552008-09-23 14:00:42 -0700659 /*
660 * Do a global flush tlb after splitting the large page
661 * and before we do the actual change page attribute in the PTE.
662 *
663 * With out this, we violate the TLB application note, that says
664 * "The TLBs may contain both ordinary and large-page
665 * translations for a 4-KByte range of linear addresses. This
666 * may occur if software modifies the paging structures so that
667 * the page size used for the address range changes. If the two
668 * translations differ with respect to page frame or attributes
669 * (e.g., permissions), processor behavior is undefined and may
670 * be implementation-specific."
671 *
672 * We do this global tlb flush inside the cpa_lock, so that we
673 * don't allow any other cpu, with stale tlb entries change the
674 * page attribute in parallel, that also falls into the
675 * just split large page entry.
676 */
677 flush_tlb_all();
Ingo Molnar87f7f8f2008-02-04 16:48:10 +0100678 goto repeat;
679 }
Ingo Molnarbeaff632008-02-04 16:48:09 +0100680
Ingo Molnar87f7f8f2008-02-04 16:48:10 +0100681 return err;
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100682}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100684static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
685
686static int cpa_process_alias(struct cpa_data *cpa)
Ingo Molnar44af6c42008-01-30 13:34:03 +0100687{
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100688 struct cpa_data alias_cpa;
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100689 int ret = 0;
Shaohua Lid75586a2008-08-21 10:46:06 +0800690 unsigned long temp_cpa_vaddr, vaddr;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100691
Yinghai Lu965194c2008-07-12 14:31:28 -0700692 if (cpa->pfn >= max_pfn_mapped)
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100693 return 0;
694
Yinghai Luf361a452008-07-10 20:38:26 -0700695#ifdef CONFIG_X86_64
Yinghai Lu965194c2008-07-12 14:31:28 -0700696 if (cpa->pfn >= max_low_pfn_mapped && cpa->pfn < (1UL<<(32-PAGE_SHIFT)))
Yinghai Luf361a452008-07-10 20:38:26 -0700697 return 0;
698#endif
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100699 /*
700 * No need to redo, when the primary call touched the direct
701 * mapping already:
702 */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700703 if (cpa->flags & CPA_PAGES_ARRAY)
704 vaddr = (unsigned long)page_address(cpa->pages[cpa->curpage]);
705 else if (cpa->flags & CPA_ARRAY)
Shaohua Lid75586a2008-08-21 10:46:06 +0800706 vaddr = cpa->vaddr[cpa->curpage];
707 else
708 vaddr = *cpa->vaddr;
709
710 if (!(within(vaddr, PAGE_OFFSET,
Suresh Siddhaa1e46212009-01-20 14:20:21 -0800711 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100712
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100713 alias_cpa = *cpa;
Shaohua Lid75586a2008-08-21 10:46:06 +0800714 temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT);
715 alias_cpa.vaddr = &temp_cpa_vaddr;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700716 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
Shaohua Lid75586a2008-08-21 10:46:06 +0800717
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100718
719 ret = __change_page_attr_set_clr(&alias_cpa, 0);
720 }
Ingo Molnar44af6c42008-01-30 13:34:03 +0100721
Arjan van de Ven488fd992008-01-30 13:34:07 +0100722#ifdef CONFIG_X86_64
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100723 if (ret)
724 return ret;
Thomas Gleixner08797502008-01-30 13:34:09 +0100725 /*
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100726 * No need to redo, when the primary call touched the high
727 * mapping already:
728 */
Jeremy Fitzhardinge93dbda72009-02-26 17:35:44 -0800729 if (within(vaddr, (unsigned long) _text, _brk_end))
Thomas Gleixnerf34b4392008-02-15 22:17:57 +0100730 return 0;
731
732 /*
Thomas Gleixner08797502008-01-30 13:34:09 +0100733 * If the physical address is inside the kernel map, we need
734 * to touch the high mapped kernel as well:
735 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100736 if (!within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn()))
737 return 0;
Thomas Gleixner08797502008-01-30 13:34:09 +0100738
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100739 alias_cpa = *cpa;
Shaohua Lid75586a2008-08-21 10:46:06 +0800740 temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base;
741 alias_cpa.vaddr = &temp_cpa_vaddr;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700742 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100743
744 /*
745 * The high mapping range is imprecise, so ignore the return value.
746 */
747 __change_page_attr_set_clr(&alias_cpa, 0);
Thomas Gleixner08797502008-01-30 13:34:09 +0100748#endif
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100749 return ret;
Ingo Molnar44af6c42008-01-30 13:34:03 +0100750}
751
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100752static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
Thomas Gleixnerff314522008-01-30 13:34:08 +0100753{
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100754 int ret, numpages = cpa->numpages;
Thomas Gleixnerff314522008-01-30 13:34:08 +0100755
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100756 while (numpages) {
757 /*
758 * Store the remaining nr of pages for the large page
759 * preservation check.
760 */
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100761 cpa->numpages = numpages;
Shaohua Lid75586a2008-08-21 10:46:06 +0800762 /* for array changes, we can't use large page */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700763 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
Shaohua Lid75586a2008-08-21 10:46:06 +0800764 cpa->numpages = 1;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100765
Suresh Siddhaad5ca552008-09-23 14:00:42 -0700766 if (!debug_pagealloc)
767 spin_lock(&cpa_lock);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100768 ret = __change_page_attr(cpa, checkalias);
Suresh Siddhaad5ca552008-09-23 14:00:42 -0700769 if (!debug_pagealloc)
770 spin_unlock(&cpa_lock);
Thomas Gleixnerff314522008-01-30 13:34:08 +0100771 if (ret)
772 return ret;
Thomas Gleixnerff314522008-01-30 13:34:08 +0100773
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100774 if (checkalias) {
775 ret = cpa_process_alias(cpa);
776 if (ret)
777 return ret;
778 }
779
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100780 /*
781 * Adjust the number of pages with the result of the
782 * CPA operation. Either a large page has been
783 * preserved or a single page update happened.
784 */
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100785 BUG_ON(cpa->numpages > numpages);
786 numpages -= cpa->numpages;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700787 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
Shaohua Lid75586a2008-08-21 10:46:06 +0800788 cpa->curpage++;
789 else
790 *cpa->vaddr += cpa->numpages * PAGE_SIZE;
791
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100792 }
Thomas Gleixnerff314522008-01-30 13:34:08 +0100793 return 0;
794}
795
Andi Kleen6bb83832008-02-04 16:48:06 +0100796static inline int cache_attr(pgprot_t attr)
797{
798 return pgprot_val(attr) &
799 (_PAGE_PAT | _PAGE_PAT_LARGE | _PAGE_PWT | _PAGE_PCD);
800}
801
Shaohua Lid75586a2008-08-21 10:46:06 +0800802static int change_page_attr_set_clr(unsigned long *addr, int numpages,
Andi Kleenc9caa022008-03-12 03:53:29 +0100803 pgprot_t mask_set, pgprot_t mask_clr,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700804 int force_split, int in_flag,
805 struct page **pages)
Thomas Gleixnerff314522008-01-30 13:34:08 +0100806{
Thomas Gleixner72e458d2008-02-04 16:48:07 +0100807 struct cpa_data cpa;
Ingo Molnarcacf8902008-08-21 13:46:33 +0200808 int ret, cache, checkalias;
Thomas Gleixner331e4062008-02-04 16:48:06 +0100809
810 /*
811 * Check, if we are requested to change a not supported
812 * feature:
813 */
814 mask_set = canon_pgprot(mask_set);
815 mask_clr = canon_pgprot(mask_clr);
Andi Kleenc9caa022008-03-12 03:53:29 +0100816 if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
Thomas Gleixner331e4062008-02-04 16:48:06 +0100817 return 0;
818
Thomas Gleixner69b14152008-02-13 11:04:50 +0100819 /* Ensure we are PAGE_SIZE aligned */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700820 if (in_flag & CPA_ARRAY) {
Shaohua Lid75586a2008-08-21 10:46:06 +0800821 int i;
822 for (i = 0; i < numpages; i++) {
823 if (addr[i] & ~PAGE_MASK) {
824 addr[i] &= PAGE_MASK;
825 WARN_ON_ONCE(1);
826 }
827 }
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700828 } else if (!(in_flag & CPA_PAGES_ARRAY)) {
829 /*
830 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
831 * No need to cehck in that case
832 */
833 if (*addr & ~PAGE_MASK) {
834 *addr &= PAGE_MASK;
835 /*
836 * People should not be passing in unaligned addresses:
837 */
838 WARN_ON_ONCE(1);
839 }
Thomas Gleixner69b14152008-02-13 11:04:50 +0100840 }
841
Nick Piggin5843d9a2008-08-01 03:15:21 +0200842 /* Must avoid aliasing mappings in the highmem code */
843 kmap_flush_unused();
844
Nick Piggindb64fe02008-10-18 20:27:03 -0700845 vm_unmap_aliases();
846
Thomas Gleixner7ad9de62009-02-12 21:16:09 +0100847 /*
848 * If we're called with lazy mmu updates enabled, the
849 * in-memory pte state may be stale. Flush pending updates to
850 * bring them up to date.
851 */
852 arch_flush_lazy_mmu_mode();
853
Thomas Gleixner72e458d2008-02-04 16:48:07 +0100854 cpa.vaddr = addr;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700855 cpa.pages = pages;
Thomas Gleixner72e458d2008-02-04 16:48:07 +0100856 cpa.numpages = numpages;
857 cpa.mask_set = mask_set;
858 cpa.mask_clr = mask_clr;
Shaohua Lid75586a2008-08-21 10:46:06 +0800859 cpa.flags = 0;
860 cpa.curpage = 0;
Andi Kleenc9caa022008-03-12 03:53:29 +0100861 cpa.force_split = force_split;
Thomas Gleixner72e458d2008-02-04 16:48:07 +0100862
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700863 if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
864 cpa.flags |= in_flag;
Shaohua Lid75586a2008-08-21 10:46:06 +0800865
Thomas Gleixneraf96e442008-02-15 21:49:46 +0100866 /* No alias checking for _NX bit modifications */
867 checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
868
869 ret = __change_page_attr_set_clr(&cpa, checkalias);
Thomas Gleixnerff314522008-01-30 13:34:08 +0100870
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100871 /*
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +0100872 * Check whether we really changed something:
873 */
Shaohua Lid75586a2008-08-21 10:46:06 +0800874 if (!(cpa.flags & CPA_FLUSHTLB))
Shaohua Li1ac2f7d2008-08-04 14:51:24 +0800875 goto out;
Ingo Molnarcacf8902008-08-21 13:46:33 +0200876
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +0100877 /*
Andi Kleen6bb83832008-02-04 16:48:06 +0100878 * No need to flush, when we did not set any of the caching
879 * attributes:
880 */
881 cache = cache_attr(mask_set);
882
883 /*
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100884 * On success we use clflush, when the CPU supports it to
885 * avoid the wbindv. If the CPU does not support it and in the
Thomas Gleixneraf1e6842008-01-30 13:34:08 +0100886 * error case we fall back to cpa_flush_all (which uses
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100887 * wbindv):
888 */
Shaohua Lid75586a2008-08-21 10:46:06 +0800889 if (!ret && cpu_has_clflush) {
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700890 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
891 cpa_flush_array(addr, numpages, cache,
892 cpa.flags, pages);
893 } else
Shaohua Lid75586a2008-08-21 10:46:06 +0800894 cpa_flush_range(*addr, numpages, cache);
895 } else
Andi Kleen6bb83832008-02-04 16:48:06 +0100896 cpa_flush_all(cache);
Ingo Molnarcacf8902008-08-21 13:46:33 +0200897
Jeremy Fitzhardinge4f06b042009-02-11 09:32:19 -0800898 /*
899 * If we've been called with lazy mmu updates enabled, then
900 * make sure that everything gets flushed out before we
901 * return.
902 */
903 arch_flush_lazy_mmu_mode();
904
Thomas Gleixner76ebd052008-02-09 23:24:09 +0100905out:
Thomas Gleixnerff314522008-01-30 13:34:08 +0100906 return ret;
907}
908
Shaohua Lid75586a2008-08-21 10:46:06 +0800909static inline int change_page_attr_set(unsigned long *addr, int numpages,
910 pgprot_t mask, int array)
Arjan van de Ven75cbade2008-01-30 13:34:06 +0100911{
Shaohua Lid75586a2008-08-21 10:46:06 +0800912 return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700913 (array ? CPA_ARRAY : 0), NULL);
Arjan van de Ven75cbade2008-01-30 13:34:06 +0100914}
915
Shaohua Lid75586a2008-08-21 10:46:06 +0800916static inline int change_page_attr_clear(unsigned long *addr, int numpages,
917 pgprot_t mask, int array)
Thomas Gleixner72932c72008-01-30 13:34:08 +0100918{
Shaohua Lid75586a2008-08-21 10:46:06 +0800919 return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700920 (array ? CPA_ARRAY : 0), NULL);
Thomas Gleixner72932c72008-01-30 13:34:08 +0100921}
922
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -0700923static inline int cpa_set_pages_array(struct page **pages, int numpages,
924 pgprot_t mask)
925{
926 return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
927 CPA_PAGES_ARRAY, pages);
928}
929
930static inline int cpa_clear_pages_array(struct page **pages, int numpages,
931 pgprot_t mask)
932{
933 return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
934 CPA_PAGES_ARRAY, pages);
935}
936
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -0700937int _set_memory_uc(unsigned long addr, int numpages)
Arjan van de Ven75cbade2008-01-30 13:34:06 +0100938{
Suresh Siddhade33c442008-04-25 17:07:22 -0700939 /*
940 * for now UC MINUS. see comments in ioremap_nocache()
941 */
Shaohua Lid75586a2008-08-21 10:46:06 +0800942 return change_page_attr_set(&addr, numpages,
943 __pgprot(_PAGE_CACHE_UC_MINUS), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +0100944}
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -0700945
946int set_memory_uc(unsigned long addr, int numpages)
947{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700948 int ret;
949
Suresh Siddhade33c442008-04-25 17:07:22 -0700950 /*
951 * for now UC MINUS. see comments in ioremap_nocache()
952 */
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700953 ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
954 _PAGE_CACHE_UC_MINUS, NULL);
955 if (ret)
956 goto out_err;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -0700957
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700958 ret = _set_memory_uc(addr, numpages);
959 if (ret)
960 goto out_free;
961
962 return 0;
963
964out_free:
965 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
966out_err:
967 return ret;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -0700968}
Arjan van de Ven75cbade2008-01-30 13:34:06 +0100969EXPORT_SYMBOL(set_memory_uc);
970
Shaohua Lid75586a2008-08-21 10:46:06 +0800971int set_memory_array_uc(unsigned long *addr, int addrinarray)
972{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700973 int i, j;
974 int ret;
975
Shaohua Lid75586a2008-08-21 10:46:06 +0800976 /*
977 * for now UC MINUS. see comments in ioremap_nocache()
978 */
979 for (i = 0; i < addrinarray; i++) {
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700980 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
981 _PAGE_CACHE_UC_MINUS, NULL);
982 if (ret)
983 goto out_free;
Shaohua Lid75586a2008-08-21 10:46:06 +0800984 }
985
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700986 ret = change_page_attr_set(addr, addrinarray,
Shaohua Lid75586a2008-08-21 10:46:06 +0800987 __pgprot(_PAGE_CACHE_UC_MINUS), 1);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700988 if (ret)
989 goto out_free;
Rene Hermanc5e147c2008-08-22 01:02:20 +0200990
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -0700991 return 0;
992
993out_free:
994 for (j = 0; j < i; j++)
995 free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
996
997 return ret;
Shaohua Lid75586a2008-08-21 10:46:06 +0800998}
999EXPORT_SYMBOL(set_memory_array_uc);
1000
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001001int _set_memory_wc(unsigned long addr, int numpages)
1002{
Shaohua Lid75586a2008-08-21 10:46:06 +08001003 return change_page_attr_set(&addr, numpages,
1004 __pgprot(_PAGE_CACHE_WC), 0);
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001005}
1006
1007int set_memory_wc(unsigned long addr, int numpages)
1008{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001009 int ret;
1010
Andreas Herrmann499f8f82008-06-10 16:06:21 +02001011 if (!pat_enabled)
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001012 return set_memory_uc(addr, numpages);
1013
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001014 ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1015 _PAGE_CACHE_WC, NULL);
1016 if (ret)
1017 goto out_err;
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001018
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001019 ret = _set_memory_wc(addr, numpages);
1020 if (ret)
1021 goto out_free;
1022
1023 return 0;
1024
1025out_free:
1026 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1027out_err:
1028 return ret;
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001029}
1030EXPORT_SYMBOL(set_memory_wc);
1031
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001032int _set_memory_wb(unsigned long addr, int numpages)
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001033{
Shaohua Lid75586a2008-08-21 10:46:06 +08001034 return change_page_attr_clear(&addr, numpages,
1035 __pgprot(_PAGE_CACHE_MASK), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001036}
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001037
1038int set_memory_wb(unsigned long addr, int numpages)
1039{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001040 int ret;
1041
1042 ret = _set_memory_wb(addr, numpages);
1043 if (ret)
1044 return ret;
1045
venkatesh.pallipadi@intel.comc15238d2008-08-20 16:45:51 -07001046 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001047 return 0;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001048}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001049EXPORT_SYMBOL(set_memory_wb);
1050
Shaohua Lid75586a2008-08-21 10:46:06 +08001051int set_memory_array_wb(unsigned long *addr, int addrinarray)
1052{
1053 int i;
venkatesh.pallipadi@intel.coma5593e02009-04-09 14:26:48 -07001054 int ret;
1055
1056 ret = change_page_attr_clear(addr, addrinarray,
1057 __pgprot(_PAGE_CACHE_MASK), 1);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001058 if (ret)
1059 return ret;
Shaohua Lid75586a2008-08-21 10:46:06 +08001060
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001061 for (i = 0; i < addrinarray; i++)
1062 free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
Rene Hermanc5e147c2008-08-22 01:02:20 +02001063
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001064 return 0;
Shaohua Lid75586a2008-08-21 10:46:06 +08001065}
1066EXPORT_SYMBOL(set_memory_array_wb);
1067
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001068int set_memory_x(unsigned long addr, int numpages)
1069{
Shaohua Lid75586a2008-08-21 10:46:06 +08001070 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001071}
1072EXPORT_SYMBOL(set_memory_x);
1073
1074int set_memory_nx(unsigned long addr, int numpages)
1075{
Shaohua Lid75586a2008-08-21 10:46:06 +08001076 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001077}
1078EXPORT_SYMBOL(set_memory_nx);
1079
1080int set_memory_ro(unsigned long addr, int numpages)
1081{
Shaohua Lid75586a2008-08-21 10:46:06 +08001082 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001083}
Bruce Allana03352d2008-09-29 20:19:22 -07001084EXPORT_SYMBOL_GPL(set_memory_ro);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001085
1086int set_memory_rw(unsigned long addr, int numpages)
1087{
Shaohua Lid75586a2008-08-21 10:46:06 +08001088 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001089}
Bruce Allana03352d2008-09-29 20:19:22 -07001090EXPORT_SYMBOL_GPL(set_memory_rw);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001091
1092int set_memory_np(unsigned long addr, int numpages)
1093{
Shaohua Lid75586a2008-08-21 10:46:06 +08001094 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001095}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001096
Andi Kleenc9caa022008-03-12 03:53:29 +01001097int set_memory_4k(unsigned long addr, int numpages)
1098{
Shaohua Lid75586a2008-08-21 10:46:06 +08001099 return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001100 __pgprot(0), 1, 0, NULL);
Andi Kleenc9caa022008-03-12 03:53:29 +01001101}
1102
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001103int set_pages_uc(struct page *page, int numpages)
1104{
1105 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001106
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001107 return set_memory_uc(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001108}
1109EXPORT_SYMBOL(set_pages_uc);
1110
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001111int set_pages_array_uc(struct page **pages, int addrinarray)
1112{
1113 unsigned long start;
1114 unsigned long end;
1115 int i;
1116 int free_idx;
1117
1118 for (i = 0; i < addrinarray; i++) {
1119 start = (unsigned long)page_address(pages[i]);
1120 end = start + PAGE_SIZE;
1121 if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL))
1122 goto err_out;
1123 }
1124
1125 if (cpa_set_pages_array(pages, addrinarray,
1126 __pgprot(_PAGE_CACHE_UC_MINUS)) == 0) {
1127 return 0; /* Success */
1128 }
1129err_out:
1130 free_idx = i;
1131 for (i = 0; i < free_idx; i++) {
1132 start = (unsigned long)page_address(pages[i]);
1133 end = start + PAGE_SIZE;
1134 free_memtype(start, end);
1135 }
1136 return -EINVAL;
1137}
1138EXPORT_SYMBOL(set_pages_array_uc);
1139
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001140int set_pages_wb(struct page *page, int numpages)
1141{
1142 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001143
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001144 return set_memory_wb(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001145}
1146EXPORT_SYMBOL(set_pages_wb);
1147
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001148int set_pages_array_wb(struct page **pages, int addrinarray)
1149{
1150 int retval;
1151 unsigned long start;
1152 unsigned long end;
1153 int i;
1154
1155 retval = cpa_clear_pages_array(pages, addrinarray,
1156 __pgprot(_PAGE_CACHE_MASK));
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001157 if (retval)
1158 return retval;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001159
1160 for (i = 0; i < addrinarray; i++) {
1161 start = (unsigned long)page_address(pages[i]);
1162 end = start + PAGE_SIZE;
1163 free_memtype(start, end);
1164 }
1165
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001166 return 0;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001167}
1168EXPORT_SYMBOL(set_pages_array_wb);
1169
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001170int set_pages_x(struct page *page, int numpages)
1171{
1172 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001173
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001174 return set_memory_x(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001175}
1176EXPORT_SYMBOL(set_pages_x);
1177
1178int set_pages_nx(struct page *page, int numpages)
1179{
1180 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001181
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001182 return set_memory_nx(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001183}
1184EXPORT_SYMBOL(set_pages_nx);
1185
1186int set_pages_ro(struct page *page, int numpages)
1187{
1188 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001189
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001190 return set_memory_ro(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001191}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001192
1193int set_pages_rw(struct page *page, int numpages)
1194{
1195 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001196
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001197 return set_memory_rw(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001198}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001199
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200#ifdef CONFIG_DEBUG_PAGEALLOC
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001201
1202static int __set_pages_p(struct page *page, int numpages)
1203{
Shaohua Lid75586a2008-08-21 10:46:06 +08001204 unsigned long tempaddr = (unsigned long) page_address(page);
1205 struct cpa_data cpa = { .vaddr = &tempaddr,
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001206 .numpages = numpages,
1207 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
Shaohua Lid75586a2008-08-21 10:46:06 +08001208 .mask_clr = __pgprot(0),
1209 .flags = 0};
Thomas Gleixner72932c72008-01-30 13:34:08 +01001210
Suresh Siddha55121b42008-09-23 14:00:40 -07001211 /*
1212 * No alias checking needed for setting present flag. otherwise,
1213 * we may need to break large pages for 64-bit kernel text
1214 * mappings (this adds to complexity if we want to do this from
1215 * atomic context especially). Let's keep it simple!
1216 */
1217 return __change_page_attr_set_clr(&cpa, 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001218}
1219
1220static int __set_pages_np(struct page *page, int numpages)
1221{
Shaohua Lid75586a2008-08-21 10:46:06 +08001222 unsigned long tempaddr = (unsigned long) page_address(page);
1223 struct cpa_data cpa = { .vaddr = &tempaddr,
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001224 .numpages = numpages,
1225 .mask_set = __pgprot(0),
Shaohua Lid75586a2008-08-21 10:46:06 +08001226 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1227 .flags = 0};
Thomas Gleixner72932c72008-01-30 13:34:08 +01001228
Suresh Siddha55121b42008-09-23 14:00:40 -07001229 /*
1230 * No alias checking needed for setting not present flag. otherwise,
1231 * we may need to break large pages for 64-bit kernel text
1232 * mappings (this adds to complexity if we want to do this from
1233 * atomic context especially). Let's keep it simple!
1234 */
1235 return __change_page_attr_set_clr(&cpa, 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001236}
1237
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238void kernel_map_pages(struct page *page, int numpages, int enable)
1239{
1240 if (PageHighMem(page))
1241 return;
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001242 if (!enable) {
Ingo Molnarf9b84042006-06-27 02:54:49 -07001243 debug_check_no_locks_freed(page_address(page),
1244 numpages * PAGE_SIZE);
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001245 }
Ingo Molnarde5097c2006-01-09 15:59:21 -08001246
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001247 /*
Ingo Molnar12d6f212008-01-30 13:33:58 +01001248 * If page allocator is not up yet then do not call c_p_a():
1249 */
1250 if (!debug_pagealloc_enabled)
1251 return;
1252
1253 /*
Ingo Molnarf8d84062008-02-13 14:09:53 +01001254 * The return value is ignored as the calls cannot fail.
Suresh Siddha55121b42008-09-23 14:00:40 -07001255 * Large pages for identity mappings are not used at boot time
1256 * and hence no memory allocations during large page split.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 */
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001258 if (enable)
1259 __set_pages_p(page, numpages);
1260 else
1261 __set_pages_np(page, numpages);
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001262
1263 /*
Ingo Molnare4b71dc2008-01-30 13:34:04 +01001264 * We should perform an IPI and flush all tlbs,
1265 * but that can deadlock->flush only current cpu:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 */
1267 __flush_tlb_all();
1268}
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01001269
1270#ifdef CONFIG_HIBERNATION
1271
1272bool kernel_page_present(struct page *page)
1273{
1274 unsigned int level;
1275 pte_t *pte;
1276
1277 if (PageHighMem(page))
1278 return false;
1279
1280 pte = lookup_address((unsigned long)page_address(page), &level);
1281 return (pte_val(*pte) & _PAGE_PRESENT);
1282}
1283
1284#endif /* CONFIG_HIBERNATION */
1285
1286#endif /* CONFIG_DEBUG_PAGEALLOC */
Arjan van de Vend1028a12008-01-30 13:34:07 +01001287
1288/*
1289 * The testcases use internal knowledge of the implementation that shouldn't
1290 * be exposed to the rest of the kernel. Include these directly here.
1291 */
1292#ifdef CONFIG_CPA_DEBUG
1293#include "pageattr-test.c"
1294#endif