blob: 007ebe2d81572b003e4bb2bbacb58814ee9e22f8 [file] [log] [blame]
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001/*
2 * Copyright 2002 Andi Kleen, SuSE Labs.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 * Thanks to Ben LaHaise for precious feedback.
Ingo Molnar9f4c8152008-01-30 13:33:41 +01004 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005#include <linux/highmem.h>
Ingo Molnar81922062008-01-30 13:34:04 +01006#include <linux/bootmem.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +01007#include <linux/sched.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +01008#include <linux/mm.h>
Thomas Gleixner76ebd052008-02-09 23:24:09 +01009#include <linux/interrupt.h>
Thomas Gleixneree7ae7a2008-04-17 17:40:45 +020010#include <linux/seq_file.h>
11#include <linux/debugfs.h>
Tejun Heoe59a1bb2009-06-22 11:56:24 +090012#include <linux/pfn.h>
Tejun Heo8c4bfc62009-07-04 08:10:59 +090013#include <linux/percpu.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090014#include <linux/gfp.h>
Matthieu Castet5bd5a452010-11-16 22:31:26 +010015#include <linux/pci.h>
Stephen Rothwelld6472302015-06-02 19:01:38 +100016#include <linux/vmalloc.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +010017
Thomas Gleixner950f9d92008-01-30 13:34:06 +010018#include <asm/e820.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/processor.h>
20#include <asm/tlbflush.h>
Dave Jonesf8af0952006-01-06 00:12:10 -080021#include <asm/sections.h>
Jeremy Fitzhardinge93dbda72009-02-26 17:35:44 -080022#include <asm/setup.h>
Ingo Molnar9f4c8152008-01-30 13:33:41 +010023#include <asm/uaccess.h>
24#include <asm/pgalloc.h>
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010025#include <asm/proto.h>
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -070026#include <asm/pat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Ingo Molnar9df84992008-02-04 16:48:09 +010028/*
29 * The current flushing context - we pass it instead of 5 arguments:
30 */
Thomas Gleixner72e458d2008-02-04 16:48:07 +010031struct cpa_data {
Shaohua Lid75586a2008-08-21 10:46:06 +080032 unsigned long *vaddr;
Borislav Petkov0fd64c22013-10-31 17:25:00 +010033 pgd_t *pgd;
Thomas Gleixner72e458d2008-02-04 16:48:07 +010034 pgprot_t mask_set;
35 pgprot_t mask_clr;
Matt Fleming74256372016-01-29 11:36:10 +000036 unsigned long numpages;
Shaohua Lid75586a2008-08-21 10:46:06 +080037 int flags;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010038 unsigned long pfn;
Andi Kleenc9caa022008-03-12 03:53:29 +010039 unsigned force_split : 1;
Shaohua Lid75586a2008-08-21 10:46:06 +080040 int curpage;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -070041 struct page **pages;
Thomas Gleixner72e458d2008-02-04 16:48:07 +010042};
43
Suresh Siddhaad5ca552008-09-23 14:00:42 -070044/*
45 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
46 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
47 * entries change the page attribute in parallel to some other cpu
48 * splitting a large page entry along with changing the attribute.
49 */
50static DEFINE_SPINLOCK(cpa_lock);
51
Shaohua Lid75586a2008-08-21 10:46:06 +080052#define CPA_FLUSHTLB 1
53#define CPA_ARRAY 2
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -070054#define CPA_PAGES_ARRAY 4
Shaohua Lid75586a2008-08-21 10:46:06 +080055
Thomas Gleixner65280e62008-05-05 16:35:21 +020056#ifdef CONFIG_PROC_FS
Andi Kleence0c0e52008-05-02 11:46:49 +020057static unsigned long direct_pages_count[PG_LEVEL_NUM];
58
Thomas Gleixner65280e62008-05-05 16:35:21 +020059void update_page_count(int level, unsigned long pages)
Andi Kleence0c0e52008-05-02 11:46:49 +020060{
Andi Kleence0c0e52008-05-02 11:46:49 +020061 /* Protect against CPA */
Andrea Arcangelia79e53d2011-02-16 15:45:22 -080062 spin_lock(&pgd_lock);
Andi Kleence0c0e52008-05-02 11:46:49 +020063 direct_pages_count[level] += pages;
Andrea Arcangelia79e53d2011-02-16 15:45:22 -080064 spin_unlock(&pgd_lock);
Andi Kleence0c0e52008-05-02 11:46:49 +020065}
66
Thomas Gleixner65280e62008-05-05 16:35:21 +020067static void split_page_count(int level)
68{
Dave Jonesc9e0d392016-01-11 12:04:28 -050069 if (direct_pages_count[level] == 0)
70 return;
71
Thomas Gleixner65280e62008-05-05 16:35:21 +020072 direct_pages_count[level]--;
73 direct_pages_count[level - 1] += PTRS_PER_PTE;
74}
75
Alexey Dobriyane1759c22008-10-15 23:50:22 +040076void arch_report_meminfo(struct seq_file *m)
Thomas Gleixner65280e62008-05-05 16:35:21 +020077{
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000078 seq_printf(m, "DirectMap4k: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010079 direct_pages_count[PG_LEVEL_4K] << 2);
80#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000081 seq_printf(m, "DirectMap2M: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010082 direct_pages_count[PG_LEVEL_2M] << 11);
83#else
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000084 seq_printf(m, "DirectMap4M: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010085 direct_pages_count[PG_LEVEL_2M] << 12);
86#endif
Hugh Dickinsa06de632008-08-15 13:58:32 +010087 if (direct_gbpages)
Hugh Dickinsb9c3bfc2008-11-06 12:05:40 +000088 seq_printf(m, "DirectMap1G: %8lu kB\n",
Hugh Dickinsa06de632008-08-15 13:58:32 +010089 direct_pages_count[PG_LEVEL_1G] << 20);
Thomas Gleixner65280e62008-05-05 16:35:21 +020090}
91#else
92static inline void split_page_count(int level) { }
93#endif
94
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +010095#ifdef CONFIG_X86_64
96
97static inline unsigned long highmap_start_pfn(void)
98{
Alexander Duyckfc8d7822012-11-16 13:57:13 -080099 return __pa_symbol(_text) >> PAGE_SHIFT;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100100}
101
102static inline unsigned long highmap_end_pfn(void)
103{
Alexander Duyckfc8d7822012-11-16 13:57:13 -0800104 return __pa_symbol(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100105}
106
107#endif
108
Ingo Molnar92cb54a2008-02-13 14:37:52 +0100109#ifdef CONFIG_DEBUG_PAGEALLOC
110# define debug_pagealloc 1
111#else
112# define debug_pagealloc 0
113#endif
114
Arjan van de Vened724be2008-01-30 13:34:04 +0100115static inline int
116within(unsigned long addr, unsigned long start, unsigned long end)
Ingo Molnar687c4822008-01-30 13:34:04 +0100117{
Arjan van de Vened724be2008-01-30 13:34:04 +0100118 return addr >= start && addr < end;
119}
120
121/*
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100122 * Flushing functions
123 */
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100124
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100125/**
126 * clflush_cache_range - flush a cache range with clflush
Wanpeng Li9efc31b2012-06-10 10:50:52 +0800127 * @vaddr: virtual start address
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100128 * @size: number of bytes to flush
129 *
Ross Zwisler8b80fd82014-02-26 12:06:50 -0700130 * clflushopt is an unordered instruction which needs fencing with mfence or
131 * sfence to avoid ordering issues.
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100132 */
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100133void clflush_cache_range(void *vaddr, unsigned int size)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100134{
Chris Wilson1f1a89a2016-01-08 09:55:33 +0000135 const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
136 void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
Ross Zwisler6c434d62015-05-11 10:15:49 +0200137 void *vend = vaddr + size;
Chris Wilson1f1a89a2016-01-08 09:55:33 +0000138
139 if (p >= vend)
140 return;
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100141
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100142 mb();
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100143
Chris Wilson1f1a89a2016-01-08 09:55:33 +0000144 for (; p < vend; p += clflush_size)
Ross Zwisler6c434d62015-05-11 10:15:49 +0200145 clflushopt(p);
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100146
Thomas Gleixnercd8ddf12008-01-30 13:34:08 +0100147 mb();
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100148}
Eric Anholte517a5e2009-09-10 17:48:48 -0700149EXPORT_SYMBOL_GPL(clflush_cache_range);
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100150
Thomas Gleixneraf1e6842008-01-30 13:34:08 +0100151static void __cpa_flush_all(void *arg)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100152{
Andi Kleen6bb83832008-02-04 16:48:06 +0100153 unsigned long cache = (unsigned long)arg;
154
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100155 /*
156 * Flush all to work around Errata in early athlons regarding
157 * large page flushing.
158 */
159 __flush_tlb_all();
160
venkatesh.pallipadi@intel.com0b827532009-05-22 13:23:37 -0700161 if (cache && boot_cpu_data.x86 >= 4)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100162 wbinvd();
163}
164
Andi Kleen6bb83832008-02-04 16:48:06 +0100165static void cpa_flush_all(unsigned long cache)
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100166{
167 BUG_ON(irqs_disabled());
168
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200169 on_each_cpu(__cpa_flush_all, (void *) cache, 1);
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100170}
171
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100172static void __cpa_flush_range(void *arg)
173{
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100174 /*
175 * We could optimize that further and do individual per page
176 * tlb invalidates for a low number of pages. Caveat: we must
177 * flush the high aliases on 64bit as well.
178 */
179 __flush_tlb_all();
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100180}
181
Andi Kleen6bb83832008-02-04 16:48:06 +0100182static void cpa_flush_range(unsigned long start, int numpages, int cache)
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100183{
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100184 unsigned int i, level;
185 unsigned long addr;
186
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100187 BUG_ON(irqs_disabled());
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100188 WARN_ON(PAGE_ALIGN(start) != start);
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100189
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200190 on_each_cpu(__cpa_flush_range, NULL, 1);
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100191
Andi Kleen6bb83832008-02-04 16:48:06 +0100192 if (!cache)
193 return;
194
Thomas Gleixner3b233e52008-01-30 13:34:08 +0100195 /*
196 * We only need to flush on one CPU,
197 * clflush is a MESI-coherent instruction that
198 * will cause all other CPUs to flush the same
199 * cachelines:
200 */
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100201 for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
202 pte_t *pte = lookup_address(addr, &level);
203
204 /*
205 * Only flush present addresses:
206 */
Thomas Gleixner7bfb72e2008-02-04 16:48:08 +0100207 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
Ingo Molnar4c61afc2008-01-30 13:34:09 +0100208 clflush_cache_range((void *) addr, PAGE_SIZE);
209 }
Thomas Gleixner57a6a462008-01-30 13:34:08 +0100210}
211
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700212static void cpa_flush_array(unsigned long *start, int numpages, int cache,
213 int in_flags, struct page **pages)
Shaohua Lid75586a2008-08-21 10:46:06 +0800214{
215 unsigned int i, level;
Pallipadi, Venkatesh21717872009-05-26 10:33:35 -0700216 unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
Shaohua Lid75586a2008-08-21 10:46:06 +0800217
218 BUG_ON(irqs_disabled());
219
Pallipadi, Venkatesh21717872009-05-26 10:33:35 -0700220 on_each_cpu(__cpa_flush_all, (void *) do_wbinvd, 1);
Shaohua Lid75586a2008-08-21 10:46:06 +0800221
Pallipadi, Venkatesh21717872009-05-26 10:33:35 -0700222 if (!cache || do_wbinvd)
Shaohua Lid75586a2008-08-21 10:46:06 +0800223 return;
224
Shaohua Lid75586a2008-08-21 10:46:06 +0800225 /*
226 * We only need to flush on one CPU,
227 * clflush is a MESI-coherent instruction that
228 * will cause all other CPUs to flush the same
229 * cachelines:
230 */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700231 for (i = 0; i < numpages; i++) {
232 unsigned long addr;
233 pte_t *pte;
234
235 if (in_flags & CPA_PAGES_ARRAY)
236 addr = (unsigned long)page_address(pages[i]);
237 else
238 addr = start[i];
239
240 pte = lookup_address(addr, &level);
Shaohua Lid75586a2008-08-21 10:46:06 +0800241
242 /*
243 * Only flush present addresses:
244 */
245 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -0700246 clflush_cache_range((void *)addr, PAGE_SIZE);
Shaohua Lid75586a2008-08-21 10:46:06 +0800247 }
248}
249
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +0100250/*
Arjan van de Vened724be2008-01-30 13:34:04 +0100251 * Certain areas of memory on x86 require very specific protection flags,
252 * for example the BIOS area or kernel text. Callers don't always get this
253 * right (again, ioremap() on BIOS memory is not uncommon) so this function
254 * checks and fixes these known static required protection bits.
255 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100256static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
257 unsigned long pfn)
Arjan van de Vened724be2008-01-30 13:34:04 +0100258{
259 pgprot_t forbidden = __pgprot(0);
260
Ingo Molnar687c4822008-01-30 13:34:04 +0100261 /*
Arjan van de Vened724be2008-01-30 13:34:04 +0100262 * The BIOS area between 640k and 1Mb needs to be executable for
263 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
Ingo Molnar687c4822008-01-30 13:34:04 +0100264 */
Matthieu Castet5bd5a452010-11-16 22:31:26 +0100265#ifdef CONFIG_PCI_BIOS
266 if (pcibios_enabled && within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
Arjan van de Vened724be2008-01-30 13:34:04 +0100267 pgprot_val(forbidden) |= _PAGE_NX;
Matthieu Castet5bd5a452010-11-16 22:31:26 +0100268#endif
Arjan van de Vened724be2008-01-30 13:34:04 +0100269
270 /*
271 * The kernel text needs to be executable for obvious reasons
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100272 * Does not cover __inittext since that is gone later on. On
273 * 64bit we do not enforce !NX on the low mapping
Arjan van de Vened724be2008-01-30 13:34:04 +0100274 */
275 if (within(address, (unsigned long)_text, (unsigned long)_etext))
276 pgprot_val(forbidden) |= _PAGE_NX;
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100277
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100278 /*
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100279 * The .rodata section needs to be read-only. Using the pfn
280 * catches all aliases.
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100281 */
Alexander Duyckfc8d7822012-11-16 13:57:13 -0800282 if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
283 __pa_symbol(__end_rodata) >> PAGE_SHIFT))
Arjan van de Vencc0f21b2008-02-04 16:48:05 +0100284 pgprot_val(forbidden) |= _PAGE_RW;
Arjan van de Vened724be2008-01-30 13:34:04 +0100285
Kees Cook9ccaf772016-02-17 14:41:14 -0800286#if defined(CONFIG_X86_64)
Suresh Siddha74e08172009-10-14 14:46:56 -0700287 /*
Suresh Siddha502f6602009-10-28 18:46:56 -0800288 * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
289 * kernel text mappings for the large page aligned text, rodata sections
290 * will be always read-only. For the kernel identity mappings covering
291 * the holes caused by this alignment can be anything that user asks.
Suresh Siddha74e08172009-10-14 14:46:56 -0700292 *
293 * This will preserve the large page mappings for kernel text/data
294 * at no extra cost.
295 */
Suresh Siddha502f6602009-10-28 18:46:56 -0800296 if (kernel_set_to_readonly &&
297 within(address, (unsigned long)_text,
Suresh Siddha281ff332010-02-18 11:51:40 -0800298 (unsigned long)__end_rodata_hpage_align)) {
299 unsigned int level;
300
301 /*
302 * Don't enforce the !RW mapping for the kernel text mapping,
303 * if the current mapping is already using small page mapping.
304 * No need to work hard to preserve large page mappings in this
305 * case.
306 *
307 * This also fixes the Linux Xen paravirt guest boot failure
308 * (because of unexpected read-only mappings for kernel identity
309 * mappings). In this paravirt guest case, the kernel text
310 * mapping and the kernel identity mapping share the same
311 * page-table pages. Thus we can't really use different
312 * protections for the kernel text and identity mappings. Also,
313 * these shared mappings are made of small page mappings.
314 * Thus this don't enforce !RW mapping for small page kernel
315 * text mapping logic will help Linux Xen parvirt guest boot
Lucas De Marchi0d2eb442011-03-17 16:24:16 -0300316 * as well.
Suresh Siddha281ff332010-02-18 11:51:40 -0800317 */
318 if (lookup_address(address, &level) && (level != PG_LEVEL_4K))
319 pgprot_val(forbidden) |= _PAGE_RW;
320 }
Suresh Siddha74e08172009-10-14 14:46:56 -0700321#endif
322
Arjan van de Vened724be2008-01-30 13:34:04 +0100323 prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
Ingo Molnar687c4822008-01-30 13:34:04 +0100324
325 return prot;
326}
327
Matt Fleming426e34c2013-12-06 21:13:04 +0000328/*
329 * Lookup the page table entry for a virtual address in a specific pgd.
330 * Return a pointer to the entry and the level of the mapping.
331 */
332pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
333 unsigned int *level)
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100334{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335 pud_t *pud;
336 pmd_t *pmd;
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100337
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100338 *level = PG_LEVEL_NONE;
339
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 if (pgd_none(*pgd))
341 return NULL;
Ingo Molnar9df84992008-02-04 16:48:09 +0100342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 pud = pud_offset(pgd, address);
344 if (pud_none(*pud))
345 return NULL;
Andi Kleenc2f71ee2008-02-04 16:48:09 +0100346
347 *level = PG_LEVEL_1G;
348 if (pud_large(*pud) || !pud_present(*pud))
349 return (pte_t *)pud;
350
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 pmd = pmd_offset(pud, address);
352 if (pmd_none(*pmd))
353 return NULL;
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100354
355 *level = PG_LEVEL_2M;
Thomas Gleixner9a14aef2008-02-04 16:48:07 +0100356 if (pmd_large(*pmd) || !pmd_present(*pmd))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 return (pte_t *)pmd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358
Thomas Gleixner30551bb2008-01-30 13:34:04 +0100359 *level = PG_LEVEL_4K;
Ingo Molnar9df84992008-02-04 16:48:09 +0100360
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100361 return pte_offset_kernel(pmd, address);
362}
Borislav Petkov0fd64c22013-10-31 17:25:00 +0100363
364/*
365 * Lookup the page table entry for a virtual address. Return a pointer
366 * to the entry and the level of the mapping.
367 *
368 * Note: We return pud and pmd either when the entry is marked large
369 * or when the present bit is not set. Otherwise we would return a
370 * pointer to a nonexisting mapping.
371 */
372pte_t *lookup_address(unsigned long address, unsigned int *level)
373{
Matt Fleming426e34c2013-12-06 21:13:04 +0000374 return lookup_address_in_pgd(pgd_offset_k(address), address, level);
Borislav Petkov0fd64c22013-10-31 17:25:00 +0100375}
Pekka Paalanen75bb8832008-05-12 21:20:56 +0200376EXPORT_SYMBOL_GPL(lookup_address);
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100377
Borislav Petkov0fd64c22013-10-31 17:25:00 +0100378static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
379 unsigned int *level)
380{
381 if (cpa->pgd)
Matt Fleming426e34c2013-12-06 21:13:04 +0000382 return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
Borislav Petkov0fd64c22013-10-31 17:25:00 +0100383 address, level);
384
385 return lookup_address(address, level);
386}
387
Ingo Molnar9df84992008-02-04 16:48:09 +0100388/*
Juergen Gross792230c2014-11-28 11:53:56 +0100389 * Lookup the PMD entry for a virtual address. Return a pointer to the entry
390 * or NULL if not present.
391 */
392pmd_t *lookup_pmd_address(unsigned long address)
393{
394 pgd_t *pgd;
395 pud_t *pud;
396
397 pgd = pgd_offset_k(address);
398 if (pgd_none(*pgd))
399 return NULL;
400
401 pud = pud_offset(pgd, address);
402 if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
403 return NULL;
404
405 return pmd_offset(pud, address);
406}
407
408/*
Dave Hansend7656532013-01-22 13:24:33 -0800409 * This is necessary because __pa() does not work on some
410 * kinds of memory, like vmalloc() or the alloc_remap()
411 * areas on 32-bit NUMA systems. The percpu areas can
412 * end up in this kind of memory, for instance.
413 *
414 * This could be optimized, but it is only intended to be
415 * used at inititalization time, and keeping it
416 * unoptimized should increase the testing coverage for
417 * the more obscure platforms.
418 */
419phys_addr_t slow_virt_to_phys(void *__virt_addr)
420{
421 unsigned long virt_addr = (unsigned long)__virt_addr;
Dexuan Cuibf70e552016-02-25 01:58:12 -0800422 phys_addr_t phys_addr;
423 unsigned long offset;
Dave Hansend7656532013-01-22 13:24:33 -0800424 enum pg_level level;
Dave Hansend7656532013-01-22 13:24:33 -0800425 pte_t *pte;
426
427 pte = lookup_address(virt_addr, &level);
428 BUG_ON(!pte);
Toshi Kani34437e62015-09-17 12:24:20 -0600429
Dexuan Cuibf70e552016-02-25 01:58:12 -0800430 /*
431 * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
432 * before being left-shifted PAGE_SHIFT bits -- this trick is to
433 * make 32-PAE kernel work correctly.
434 */
Toshi Kani34437e62015-09-17 12:24:20 -0600435 switch (level) {
436 case PG_LEVEL_1G:
Dexuan Cuibf70e552016-02-25 01:58:12 -0800437 phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
Toshi Kani34437e62015-09-17 12:24:20 -0600438 offset = virt_addr & ~PUD_PAGE_MASK;
439 break;
440 case PG_LEVEL_2M:
Dexuan Cuibf70e552016-02-25 01:58:12 -0800441 phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
Toshi Kani34437e62015-09-17 12:24:20 -0600442 offset = virt_addr & ~PMD_PAGE_MASK;
443 break;
444 default:
Dexuan Cuibf70e552016-02-25 01:58:12 -0800445 phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
Toshi Kani34437e62015-09-17 12:24:20 -0600446 offset = virt_addr & ~PAGE_MASK;
447 }
448
449 return (phys_addr_t)(phys_addr | offset);
Dave Hansend7656532013-01-22 13:24:33 -0800450}
451EXPORT_SYMBOL_GPL(slow_virt_to_phys);
452
453/*
Ingo Molnar9df84992008-02-04 16:48:09 +0100454 * Set the new pmd in all the pgds we know about:
455 */
Ingo Molnar9a3dc782008-01-30 13:33:57 +0100456static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100457{
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100458 /* change init_mm */
459 set_pte_atomic(kpte, pte);
Ingo Molnar44af6c42008-01-30 13:34:03 +0100460#ifdef CONFIG_X86_32
Ingo Molnare4b71dc2008-01-30 13:34:04 +0100461 if (!SHARED_KERNEL_PMD) {
Ingo Molnar44af6c42008-01-30 13:34:03 +0100462 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
Jeremy Fitzhardingee3ed9102008-01-30 13:34:11 +0100464 list_for_each_entry(page, &pgd_list, lru) {
Ingo Molnar44af6c42008-01-30 13:34:03 +0100465 pgd_t *pgd;
466 pud_t *pud;
467 pmd_t *pmd;
Ingo Molnar9f4c8152008-01-30 13:33:41 +0100468
Ingo Molnar44af6c42008-01-30 13:34:03 +0100469 pgd = (pgd_t *)page_address(page) + pgd_index(address);
470 pud = pud_offset(pgd, address);
471 pmd = pmd_offset(pud, address);
472 set_pte_atomic((pte_t *)pmd, pte);
473 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 }
Ingo Molnar44af6c42008-01-30 13:34:03 +0100475#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476}
477
Ingo Molnar9df84992008-02-04 16:48:09 +0100478static int
479try_preserve_large_page(pte_t *kpte, unsigned long address,
480 struct cpa_data *cpa)
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100481{
Toshi Kani3a191092015-09-17 12:24:22 -0600482 unsigned long nextpage_addr, numpages, pmask, psize, addr, pfn, old_pfn;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100483 pte_t new_pte, old_pte, *tmp;
matthieu castet64edc8e2010-11-16 22:30:27 +0100484 pgprot_t old_prot, new_prot, req_prot;
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100485 int i, do_split = 1;
Dave Hansenf3c4fbb2013-01-22 13:24:32 -0800486 enum pg_level level;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100487
Andi Kleenc9caa022008-03-12 03:53:29 +0100488 if (cpa->force_split)
489 return 1;
490
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800491 spin_lock(&pgd_lock);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100492 /*
493 * Check for races, another CPU might have split this page
494 * up already:
495 */
Borislav Petkov82f07122013-10-31 17:25:07 +0100496 tmp = _lookup_address_cpa(cpa, address, &level);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100497 if (tmp != kpte)
498 goto out_unlock;
499
500 switch (level) {
501 case PG_LEVEL_2M:
Toshi Kani3a191092015-09-17 12:24:22 -0600502 old_prot = pmd_pgprot(*(pmd_t *)kpte);
503 old_pfn = pmd_pfn(*(pmd_t *)kpte);
504 break;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100505 case PG_LEVEL_1G:
Toshi Kani3a191092015-09-17 12:24:22 -0600506 old_prot = pud_pgprot(*(pud_t *)kpte);
507 old_pfn = pud_pfn(*(pud_t *)kpte);
Dave Hansenf3c4fbb2013-01-22 13:24:32 -0800508 break;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100509 default:
Ingo Molnarbeaff632008-02-04 16:48:09 +0100510 do_split = -EINVAL;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100511 goto out_unlock;
512 }
513
Toshi Kani3a191092015-09-17 12:24:22 -0600514 psize = page_level_size(level);
515 pmask = page_level_mask(level);
516
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100517 /*
518 * Calculate the number of pages, which fit into this large
519 * page starting at address:
520 */
521 nextpage_addr = (address + psize) & pmask;
522 numpages = (nextpage_addr - address) >> PAGE_SHIFT;
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +0100523 if (numpages < cpa->numpages)
524 cpa->numpages = numpages;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100525
526 /*
527 * We are safe now. Check whether the new pgprot is the same:
Juergen Grossf5b28312014-11-03 14:02:02 +0100528 * Convert protection attributes to 4k-format, as cpa->mask* are set
529 * up accordingly.
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100530 */
531 old_pte = *kpte;
Toshi Kani55696b12015-09-17 12:24:24 -0600532 req_prot = pgprot_large_2_4k(old_prot);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100533
matthieu castet64edc8e2010-11-16 22:30:27 +0100534 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
535 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100536
537 /*
Juergen Grossf5b28312014-11-03 14:02:02 +0100538 * req_prot is in format of 4k pages. It must be converted to large
539 * page format: the caching mode includes the PAT bit located at
540 * different bit positions in the two formats.
541 */
542 req_prot = pgprot_4k_2_large(req_prot);
543
544 /*
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800545 * Set the PSE and GLOBAL flags only if the PRESENT flag is
546 * set otherwise pmd_present/pmd_huge will return true even on
547 * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
548 * for the ancient hardware that doesn't support it.
549 */
Andrea Arcangelif76cfa32013-04-10 15:28:25 +0200550 if (pgprot_val(req_prot) & _PAGE_PRESENT)
551 pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800552 else
Andrea Arcangelif76cfa32013-04-10 15:28:25 +0200553 pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800554
Andrea Arcangelif76cfa32013-04-10 15:28:25 +0200555 req_prot = canon_pgprot(req_prot);
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800556
557 /*
Toshi Kani3a191092015-09-17 12:24:22 -0600558 * old_pfn points to the large page base pfn. So we need
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100559 * to add the offset of the virtual address:
560 */
Toshi Kani3a191092015-09-17 12:24:22 -0600561 pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +0100562 cpa->pfn = pfn;
563
matthieu castet64edc8e2010-11-16 22:30:27 +0100564 new_prot = static_protections(req_prot, address, pfn);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100565
566 /*
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100567 * We need to check the full range, whether
568 * static_protection() requires a different pgprot for one of
569 * the pages in the range we try to preserve:
570 */
matthieu castet64edc8e2010-11-16 22:30:27 +0100571 addr = address & pmask;
Toshi Kani3a191092015-09-17 12:24:22 -0600572 pfn = old_pfn;
matthieu castet64edc8e2010-11-16 22:30:27 +0100573 for (i = 0; i < (psize >> PAGE_SHIFT); i++, addr += PAGE_SIZE, pfn++) {
574 pgprot_t chk_prot = static_protections(req_prot, addr, pfn);
Thomas Gleixnerfac84932008-02-09 23:24:09 +0100575
576 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
577 goto out_unlock;
578 }
579
580 /*
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100581 * If there are no changes, return. maxpages has been updated
582 * above:
583 */
584 if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
Ingo Molnarbeaff632008-02-04 16:48:09 +0100585 do_split = 0;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100586 goto out_unlock;
587 }
588
589 /*
590 * We need to change the attributes. Check, whether we can
591 * change the large page in one go. We request a split, when
592 * the address is not aligned and the number of pages is
593 * smaller than the number of pages in the large page. Note
594 * that we limited the number of possible pages already to
595 * the number of pages in the large page.
596 */
matthieu castet64edc8e2010-11-16 22:30:27 +0100597 if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) {
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100598 /*
599 * The address is aligned and the number of pages
600 * covers the full page.
601 */
Toshi Kani3a191092015-09-17 12:24:22 -0600602 new_pte = pfn_pte(old_pfn, new_prot);
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100603 __set_pmd_pte(kpte, address, new_pte);
Shaohua Lid75586a2008-08-21 10:46:06 +0800604 cpa->flags |= CPA_FLUSHTLB;
Ingo Molnarbeaff632008-02-04 16:48:09 +0100605 do_split = 0;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100606 }
607
608out_unlock:
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800609 spin_unlock(&pgd_lock);
Ingo Molnar9df84992008-02-04 16:48:09 +0100610
Ingo Molnarbeaff632008-02-04 16:48:09 +0100611 return do_split;
Thomas Gleixner65e074d2008-02-04 16:48:07 +0100612}
613
Borislav Petkov59528862013-03-21 18:16:57 +0100614static int
Borislav Petkov82f07122013-10-31 17:25:07 +0100615__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
616 struct page *base)
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100617{
Borislav Petkov59528862013-03-21 18:16:57 +0100618 pte_t *pbase = (pte_t *)page_address(base);
Toshi Kanid551aaa2015-09-17 12:24:23 -0600619 unsigned long ref_pfn, pfn, pfninc = 1;
Ingo Molnar86f03982008-01-30 13:34:09 +0100620 unsigned int i, level;
Wen Congyangae9aae92013-02-22 16:33:04 -0800621 pte_t *tmp;
Ingo Molnar9df84992008-02-04 16:48:09 +0100622 pgprot_t ref_prot;
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100623
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800624 spin_lock(&pgd_lock);
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100625 /*
626 * Check for races, another CPU might have split this page
627 * up for us already:
628 */
Borislav Petkov82f07122013-10-31 17:25:07 +0100629 tmp = _lookup_address_cpa(cpa, address, &level);
Wen Congyangae9aae92013-02-22 16:33:04 -0800630 if (tmp != kpte) {
631 spin_unlock(&pgd_lock);
632 return 1;
633 }
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100634
Jeremy Fitzhardinge6944a9c2008-03-17 16:37:01 -0700635 paravirt_alloc_pte(&init_mm, page_to_pfn(base));
Juergen Grossf5b28312014-11-03 14:02:02 +0100636
Toshi Kanid551aaa2015-09-17 12:24:23 -0600637 switch (level) {
638 case PG_LEVEL_2M:
639 ref_prot = pmd_pgprot(*(pmd_t *)kpte);
640 /* clear PSE and promote PAT bit to correct position */
Juergen Grossf5b28312014-11-03 14:02:02 +0100641 ref_prot = pgprot_large_2_4k(ref_prot);
Toshi Kanid551aaa2015-09-17 12:24:23 -0600642 ref_pfn = pmd_pfn(*(pmd_t *)kpte);
643 break;
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100644
Toshi Kanid551aaa2015-09-17 12:24:23 -0600645 case PG_LEVEL_1G:
646 ref_prot = pud_pgprot(*(pud_t *)kpte);
647 ref_pfn = pud_pfn(*(pud_t *)kpte);
Andi Kleenf07333f2008-02-04 16:48:09 +0100648 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
Toshi Kanid551aaa2015-09-17 12:24:23 -0600649
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800650 /*
Toshi Kanid551aaa2015-09-17 12:24:23 -0600651 * Clear the PSE flags if the PRESENT flag is not set
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800652 * otherwise pmd_present/pmd_huge will return true
653 * even on a non present pmd.
654 */
Toshi Kanid551aaa2015-09-17 12:24:23 -0600655 if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800656 pgprot_val(ref_prot) &= ~_PAGE_PSE;
Toshi Kanid551aaa2015-09-17 12:24:23 -0600657 break;
658
659 default:
660 spin_unlock(&pgd_lock);
661 return 1;
Andi Kleenf07333f2008-02-04 16:48:09 +0100662 }
Andi Kleenf07333f2008-02-04 16:48:09 +0100663
Thomas Gleixner63c1dcf2008-02-04 16:48:05 +0100664 /*
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800665 * Set the GLOBAL flags only if the PRESENT flag is set
666 * otherwise pmd/pte_present will return true even on a non
667 * present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL
668 * for the ancient hardware that doesn't support it.
669 */
670 if (pgprot_val(ref_prot) & _PAGE_PRESENT)
671 pgprot_val(ref_prot) |= _PAGE_GLOBAL;
672 else
673 pgprot_val(ref_prot) &= ~_PAGE_GLOBAL;
674
675 /*
Thomas Gleixner63c1dcf2008-02-04 16:48:05 +0100676 * Get the target pfn from the original entry:
677 */
Toshi Kanid551aaa2015-09-17 12:24:23 -0600678 pfn = ref_pfn;
Andi Kleenf07333f2008-02-04 16:48:09 +0100679 for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -0800680 set_pte(&pbase[i], pfn_pte(pfn, canon_pgprot(ref_prot)));
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100681
Sai Praneeth2c66e24d2015-10-16 16:20:27 -0700682 if (virt_addr_valid(address)) {
683 unsigned long pfn = PFN_DOWN(__pa(address));
684
685 if (pfn_range_is_mapped(pfn, pfn + 1))
686 split_page_count(level);
687 }
Yinghai Luf361a452008-07-10 20:38:26 -0700688
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100689 /*
Ingo Molnar07a66d72009-02-20 08:04:13 +0100690 * Install the new, split up pagetable.
Huang, Ying4c881ca2008-01-30 13:34:04 +0100691 *
Ingo Molnar07a66d72009-02-20 08:04:13 +0100692 * We use the standard kernel pagetable protections for the new
693 * pagetable protections, the actual ptes set above control the
694 * primary protection behavior:
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100695 */
Ingo Molnar07a66d72009-02-20 08:04:13 +0100696 __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
Ingo Molnar211b3d02009-03-10 22:31:03 +0100697
698 /*
699 * Intel Atom errata AAH41 workaround.
700 *
701 * The real fix should be in hw or in a microcode update, but
702 * we also probabilistically try to reduce the window of having
703 * a large TLB mixed with 4K TLBs while instruction fetches are
704 * going on.
705 */
706 __flush_tlb_all();
Andrea Arcangelia79e53d2011-02-16 15:45:22 -0800707 spin_unlock(&pgd_lock);
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100708
Ingo Molnarbb5c2db2008-01-30 13:33:56 +0100709 return 0;
710}
711
Borislav Petkov82f07122013-10-31 17:25:07 +0100712static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
713 unsigned long address)
Wen Congyangae9aae92013-02-22 16:33:04 -0800714{
Wen Congyangae9aae92013-02-22 16:33:04 -0800715 struct page *base;
716
717 if (!debug_pagealloc)
718 spin_unlock(&cpa_lock);
719 base = alloc_pages(GFP_KERNEL | __GFP_NOTRACK, 0);
720 if (!debug_pagealloc)
721 spin_lock(&cpa_lock);
722 if (!base)
723 return -ENOMEM;
724
Borislav Petkov82f07122013-10-31 17:25:07 +0100725 if (__split_large_page(cpa, kpte, address, base))
Wen Congyangae9aae92013-02-22 16:33:04 -0800726 __free_page(base);
727
728 return 0;
729}
730
Borislav Petkov52a628f2013-10-31 17:25:06 +0100731static bool try_to_free_pte_page(pte_t *pte)
732{
733 int i;
734
735 for (i = 0; i < PTRS_PER_PTE; i++)
736 if (!pte_none(pte[i]))
737 return false;
738
739 free_page((unsigned long)pte);
740 return true;
741}
742
743static bool try_to_free_pmd_page(pmd_t *pmd)
744{
745 int i;
746
747 for (i = 0; i < PTRS_PER_PMD; i++)
748 if (!pmd_none(pmd[i]))
749 return false;
750
751 free_page((unsigned long)pmd);
752 return true;
753}
754
Borislav Petkov42a54772014-01-18 12:48:16 +0100755static bool try_to_free_pud_page(pud_t *pud)
756{
757 int i;
758
759 for (i = 0; i < PTRS_PER_PUD; i++)
760 if (!pud_none(pud[i]))
761 return false;
762
763 free_page((unsigned long)pud);
764 return true;
765}
766
Borislav Petkov52a628f2013-10-31 17:25:06 +0100767static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
768{
769 pte_t *pte = pte_offset_kernel(pmd, start);
770
771 while (start < end) {
772 set_pte(pte, __pte(0));
773
774 start += PAGE_SIZE;
775 pte++;
776 }
777
778 if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
779 pmd_clear(pmd);
780 return true;
781 }
782 return false;
783}
784
785static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
786 unsigned long start, unsigned long end)
787{
788 if (unmap_pte_range(pmd, start, end))
789 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
790 pud_clear(pud);
791}
792
793static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
794{
795 pmd_t *pmd = pmd_offset(pud, start);
796
797 /*
798 * Not on a 2MB page boundary?
799 */
800 if (start & (PMD_SIZE - 1)) {
801 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
802 unsigned long pre_end = min_t(unsigned long, end, next_page);
803
804 __unmap_pmd_range(pud, pmd, start, pre_end);
805
806 start = pre_end;
807 pmd++;
808 }
809
810 /*
811 * Try to unmap in 2M chunks.
812 */
813 while (end - start >= PMD_SIZE) {
814 if (pmd_large(*pmd))
815 pmd_clear(pmd);
816 else
817 __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
818
819 start += PMD_SIZE;
820 pmd++;
821 }
822
823 /*
824 * 4K leftovers?
825 */
826 if (start < end)
827 return __unmap_pmd_range(pud, pmd, start, end);
828
829 /*
830 * Try again to free the PMD page if haven't succeeded above.
831 */
832 if (!pud_none(*pud))
833 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
834 pud_clear(pud);
835}
Borislav Petkov0bb8aee2013-10-31 17:25:05 +0100836
837static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
838{
839 pud_t *pud = pud_offset(pgd, start);
840
841 /*
842 * Not on a GB page boundary?
843 */
844 if (start & (PUD_SIZE - 1)) {
845 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
846 unsigned long pre_end = min_t(unsigned long, end, next_page);
847
848 unmap_pmd_range(pud, start, pre_end);
849
850 start = pre_end;
851 pud++;
852 }
853
854 /*
855 * Try to unmap in 1G chunks?
856 */
857 while (end - start >= PUD_SIZE) {
858
859 if (pud_large(*pud))
860 pud_clear(pud);
861 else
862 unmap_pmd_range(pud, start, start + PUD_SIZE);
863
864 start += PUD_SIZE;
865 pud++;
866 }
867
868 /*
869 * 2M leftovers?
870 */
871 if (start < end)
872 unmap_pmd_range(pud, start, end);
873
874 /*
875 * No need to try to free the PUD page because we'll free it in
876 * populate_pgd's error path
877 */
878}
879
Borislav Petkov42a54772014-01-18 12:48:16 +0100880static void unmap_pgd_range(pgd_t *root, unsigned long addr, unsigned long end)
881{
882 pgd_t *pgd_entry = root + pgd_index(addr);
883
884 unmap_pud_range(pgd_entry, addr, end);
885
886 if (try_to_free_pud_page((pud_t *)pgd_page_vaddr(*pgd_entry)))
887 pgd_clear(pgd_entry);
888}
889
Borislav Petkovf900a4b2013-10-31 17:25:03 +0100890static int alloc_pte_page(pmd_t *pmd)
891{
892 pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
893 if (!pte)
894 return -1;
895
896 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
897 return 0;
898}
899
Borislav Petkov4b235382013-10-31 17:25:02 +0100900static int alloc_pmd_page(pud_t *pud)
901{
902 pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
903 if (!pmd)
904 return -1;
905
906 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
907 return 0;
908}
909
Borislav Petkovc6b6f362013-10-31 17:25:04 +0100910static void populate_pte(struct cpa_data *cpa,
911 unsigned long start, unsigned long end,
912 unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
913{
914 pte_t *pte;
915
916 pte = pte_offset_kernel(pmd, start);
917
918 while (num_pages-- && start < end) {
919
920 /* deal with the NX bit */
921 if (!(pgprot_val(pgprot) & _PAGE_NX))
922 cpa->pfn &= ~_PAGE_NX;
923
924 set_pte(pte, pfn_pte(cpa->pfn >> PAGE_SHIFT, pgprot));
925
926 start += PAGE_SIZE;
927 cpa->pfn += PAGE_SIZE;
928 pte++;
929 }
930}
Borislav Petkovf900a4b2013-10-31 17:25:03 +0100931
932static int populate_pmd(struct cpa_data *cpa,
933 unsigned long start, unsigned long end,
934 unsigned num_pages, pud_t *pud, pgprot_t pgprot)
935{
936 unsigned int cur_pages = 0;
937 pmd_t *pmd;
Juergen Grossf5b28312014-11-03 14:02:02 +0100938 pgprot_t pmd_pgprot;
Borislav Petkovf900a4b2013-10-31 17:25:03 +0100939
940 /*
941 * Not on a 2M boundary?
942 */
943 if (start & (PMD_SIZE - 1)) {
944 unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
945 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
946
947 pre_end = min_t(unsigned long, pre_end, next_page);
948 cur_pages = (pre_end - start) >> PAGE_SHIFT;
949 cur_pages = min_t(unsigned int, num_pages, cur_pages);
950
951 /*
952 * Need a PTE page?
953 */
954 pmd = pmd_offset(pud, start);
955 if (pmd_none(*pmd))
956 if (alloc_pte_page(pmd))
957 return -1;
958
959 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
960
961 start = pre_end;
962 }
963
964 /*
965 * We mapped them all?
966 */
967 if (num_pages == cur_pages)
968 return cur_pages;
969
Juergen Grossf5b28312014-11-03 14:02:02 +0100970 pmd_pgprot = pgprot_4k_2_large(pgprot);
971
Borislav Petkovf900a4b2013-10-31 17:25:03 +0100972 while (end - start >= PMD_SIZE) {
973
974 /*
975 * We cannot use a 1G page so allocate a PMD page if needed.
976 */
977 if (pud_none(*pud))
978 if (alloc_pmd_page(pud))
979 return -1;
980
981 pmd = pmd_offset(pud, start);
982
Juergen Grossf5b28312014-11-03 14:02:02 +0100983 set_pmd(pmd, __pmd(cpa->pfn | _PAGE_PSE |
984 massage_pgprot(pmd_pgprot)));
Borislav Petkovf900a4b2013-10-31 17:25:03 +0100985
986 start += PMD_SIZE;
987 cpa->pfn += PMD_SIZE;
988 cur_pages += PMD_SIZE >> PAGE_SHIFT;
989 }
990
991 /*
992 * Map trailing 4K pages.
993 */
994 if (start < end) {
995 pmd = pmd_offset(pud, start);
996 if (pmd_none(*pmd))
997 if (alloc_pte_page(pmd))
998 return -1;
999
1000 populate_pte(cpa, start, end, num_pages - cur_pages,
1001 pmd, pgprot);
1002 }
1003 return num_pages;
1004}
Borislav Petkov4b235382013-10-31 17:25:02 +01001005
1006static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,
1007 pgprot_t pgprot)
1008{
1009 pud_t *pud;
1010 unsigned long end;
1011 int cur_pages = 0;
Juergen Grossf5b28312014-11-03 14:02:02 +01001012 pgprot_t pud_pgprot;
Borislav Petkov4b235382013-10-31 17:25:02 +01001013
1014 end = start + (cpa->numpages << PAGE_SHIFT);
1015
1016 /*
1017 * Not on a Gb page boundary? => map everything up to it with
1018 * smaller pages.
1019 */
1020 if (start & (PUD_SIZE - 1)) {
1021 unsigned long pre_end;
1022 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1023
1024 pre_end = min_t(unsigned long, end, next_page);
1025 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1026 cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1027
1028 pud = pud_offset(pgd, start);
1029
1030 /*
1031 * Need a PMD page?
1032 */
1033 if (pud_none(*pud))
1034 if (alloc_pmd_page(pud))
1035 return -1;
1036
1037 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1038 pud, pgprot);
1039 if (cur_pages < 0)
1040 return cur_pages;
1041
1042 start = pre_end;
1043 }
1044
1045 /* We mapped them all? */
1046 if (cpa->numpages == cur_pages)
1047 return cur_pages;
1048
1049 pud = pud_offset(pgd, start);
Juergen Grossf5b28312014-11-03 14:02:02 +01001050 pud_pgprot = pgprot_4k_2_large(pgprot);
Borislav Petkov4b235382013-10-31 17:25:02 +01001051
1052 /*
1053 * Map everything starting from the Gb boundary, possibly with 1G pages
1054 */
1055 while (end - start >= PUD_SIZE) {
Juergen Grossf5b28312014-11-03 14:02:02 +01001056 set_pud(pud, __pud(cpa->pfn | _PAGE_PSE |
1057 massage_pgprot(pud_pgprot)));
Borislav Petkov4b235382013-10-31 17:25:02 +01001058
1059 start += PUD_SIZE;
1060 cpa->pfn += PUD_SIZE;
1061 cur_pages += PUD_SIZE >> PAGE_SHIFT;
1062 pud++;
1063 }
1064
1065 /* Map trailing leftover */
1066 if (start < end) {
1067 int tmp;
1068
1069 pud = pud_offset(pgd, start);
1070 if (pud_none(*pud))
1071 if (alloc_pmd_page(pud))
1072 return -1;
1073
1074 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1075 pud, pgprot);
1076 if (tmp < 0)
1077 return cur_pages;
1078
1079 cur_pages += tmp;
1080 }
1081 return cur_pages;
1082}
Borislav Petkovf3f72962013-10-31 17:25:01 +01001083
1084/*
1085 * Restrictions for kernel page table do not necessarily apply when mapping in
1086 * an alternate PGD.
1087 */
1088static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1089{
1090 pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
Borislav Petkovf3f72962013-10-31 17:25:01 +01001091 pud_t *pud = NULL; /* shut up gcc */
Borislav Petkov42a54772014-01-18 12:48:16 +01001092 pgd_t *pgd_entry;
Borislav Petkovf3f72962013-10-31 17:25:01 +01001093 int ret;
1094
1095 pgd_entry = cpa->pgd + pgd_index(addr);
1096
1097 /*
1098 * Allocate a PUD page and hand it down for mapping.
1099 */
1100 if (pgd_none(*pgd_entry)) {
1101 pud = (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
1102 if (!pud)
1103 return -1;
1104
1105 set_pgd(pgd_entry, __pgd(__pa(pud) | _KERNPG_TABLE));
Borislav Petkovf3f72962013-10-31 17:25:01 +01001106 }
1107
1108 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1109 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set);
1110
1111 ret = populate_pud(cpa, addr, pgd_entry, pgprot);
Borislav Petkov0bb8aee2013-10-31 17:25:05 +01001112 if (ret < 0) {
Borislav Petkov42a54772014-01-18 12:48:16 +01001113 unmap_pgd_range(cpa->pgd, addr,
Borislav Petkov0bb8aee2013-10-31 17:25:05 +01001114 addr + (cpa->numpages << PAGE_SHIFT));
Borislav Petkov0bb8aee2013-10-31 17:25:05 +01001115 return ret;
1116 }
Borislav Petkov42a54772014-01-18 12:48:16 +01001117
Borislav Petkovf3f72962013-10-31 17:25:01 +01001118 cpa->numpages = ret;
1119 return 0;
1120}
1121
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001122static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1123 int primary)
1124{
Borislav Petkov82f07122013-10-31 17:25:07 +01001125 if (cpa->pgd)
1126 return populate_pgd(cpa, vaddr);
1127
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001128 /*
1129 * Ignore all non primary paths.
1130 */
Jan Beulich405e11332016-02-10 02:03:00 -07001131 if (!primary) {
1132 cpa->numpages = 1;
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001133 return 0;
Jan Beulich405e11332016-02-10 02:03:00 -07001134 }
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001135
1136 /*
1137 * Ignore the NULL PTE for kernel identity mapping, as it is expected
1138 * to have holes.
1139 * Also set numpages to '1' indicating that we processed cpa req for
1140 * one virtual address page and its pfn. TBD: numpages can be set based
1141 * on the initial value and the level returned by lookup_address().
1142 */
1143 if (within(vaddr, PAGE_OFFSET,
1144 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
1145 cpa->numpages = 1;
1146 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1147 return 0;
1148 } else {
1149 WARN(1, KERN_WARNING "CPA: called for zero pte. "
1150 "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1151 *cpa->vaddr);
1152
1153 return -EFAULT;
1154 }
1155}
1156
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001157static int __change_page_attr(struct cpa_data *cpa, int primary)
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001158{
Shaohua Lid75586a2008-08-21 10:46:06 +08001159 unsigned long address;
Harvey Harrisonda7bfc52008-02-09 23:24:08 +01001160 int do_split, err;
1161 unsigned int level;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001162 pte_t *kpte, old_pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163
Thomas Hellstrom8523acf2009-08-03 09:25:45 +02001164 if (cpa->flags & CPA_PAGES_ARRAY) {
1165 struct page *page = cpa->pages[cpa->curpage];
1166 if (unlikely(PageHighMem(page)))
1167 return 0;
1168 address = (unsigned long)page_address(page);
1169 } else if (cpa->flags & CPA_ARRAY)
Shaohua Lid75586a2008-08-21 10:46:06 +08001170 address = cpa->vaddr[cpa->curpage];
1171 else
1172 address = *cpa->vaddr;
Ingo Molnar97f99fe2008-01-30 13:33:55 +01001173repeat:
Borislav Petkov82f07122013-10-31 17:25:07 +01001174 kpte = _lookup_address_cpa(cpa, address, &level);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 if (!kpte)
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001176 return __cpa_process_fault(cpa, address, primary);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001177
1178 old_pte = *kpte;
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001179 if (!pte_val(old_pte))
1180 return __cpa_process_fault(cpa, address, primary);
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001181
Thomas Gleixner30551bb2008-01-30 13:34:04 +01001182 if (level == PG_LEVEL_4K) {
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001183 pte_t new_pte;
Arjan van de Ven626c2c92008-02-04 16:48:05 +01001184 pgprot_t new_prot = pte_pgprot(old_pte);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001185 unsigned long pfn = pte_pfn(old_pte);
Thomas Gleixnera72a08a2008-01-30 13:34:07 +01001186
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001187 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1188 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
Ingo Molnar86f03982008-01-30 13:34:09 +01001189
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001190 new_prot = static_protections(new_prot, address, pfn);
Ingo Molnar86f03982008-01-30 13:34:09 +01001191
Arjan van de Ven626c2c92008-02-04 16:48:05 +01001192 /*
Andrea Arcangelia8aed3e2013-02-22 15:11:51 -08001193 * Set the GLOBAL flags only if the PRESENT flag is
1194 * set otherwise pte_present will return true even on
1195 * a non present pte. The canon_pgprot will clear
1196 * _PAGE_GLOBAL for the ancient hardware that doesn't
1197 * support it.
1198 */
1199 if (pgprot_val(new_prot) & _PAGE_PRESENT)
1200 pgprot_val(new_prot) |= _PAGE_GLOBAL;
1201 else
1202 pgprot_val(new_prot) &= ~_PAGE_GLOBAL;
1203
1204 /*
Arjan van de Ven626c2c92008-02-04 16:48:05 +01001205 * We need to keep the pfn from the existing PTE,
1206 * after all we're only going to change it's attributes
1207 * not the memory it points to
1208 */
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001209 new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
1210 cpa->pfn = pfn;
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +01001211 /*
1212 * Do we really change anything ?
1213 */
1214 if (pte_val(old_pte) != pte_val(new_pte)) {
1215 set_pte_atomic(kpte, new_pte);
Shaohua Lid75586a2008-08-21 10:46:06 +08001216 cpa->flags |= CPA_FLUSHTLB;
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +01001217 }
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +01001218 cpa->numpages = 1;
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001219 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 }
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001221
1222 /*
1223 * Check, whether we can keep the large page intact
1224 * and just change the pte:
1225 */
Ingo Molnarbeaff632008-02-04 16:48:09 +01001226 do_split = try_preserve_large_page(kpte, address, cpa);
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001227 /*
1228 * When the range fits into the existing large page,
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +01001229 * return. cp->numpages and cpa->tlbflush have been updated in
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001230 * try_large_page:
1231 */
Ingo Molnar87f7f8f2008-02-04 16:48:10 +01001232 if (do_split <= 0)
1233 return do_split;
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001234
1235 /*
1236 * We have to split the large page:
1237 */
Borislav Petkov82f07122013-10-31 17:25:07 +01001238 err = split_large_page(cpa, kpte, address);
Ingo Molnar87f7f8f2008-02-04 16:48:10 +01001239 if (!err) {
Suresh Siddhaad5ca552008-09-23 14:00:42 -07001240 /*
1241 * Do a global flush tlb after splitting the large page
1242 * and before we do the actual change page attribute in the PTE.
1243 *
1244 * With out this, we violate the TLB application note, that says
1245 * "The TLBs may contain both ordinary and large-page
1246 * translations for a 4-KByte range of linear addresses. This
1247 * may occur if software modifies the paging structures so that
1248 * the page size used for the address range changes. If the two
1249 * translations differ with respect to page frame or attributes
1250 * (e.g., permissions), processor behavior is undefined and may
1251 * be implementation-specific."
1252 *
1253 * We do this global tlb flush inside the cpa_lock, so that we
1254 * don't allow any other cpu, with stale tlb entries change the
1255 * page attribute in parallel, that also falls into the
1256 * just split large page entry.
1257 */
1258 flush_tlb_all();
Ingo Molnar87f7f8f2008-02-04 16:48:10 +01001259 goto repeat;
1260 }
Ingo Molnarbeaff632008-02-04 16:48:09 +01001261
Ingo Molnar87f7f8f2008-02-04 16:48:10 +01001262 return err;
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001263}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001265static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1266
1267static int cpa_process_alias(struct cpa_data *cpa)
Ingo Molnar44af6c42008-01-30 13:34:03 +01001268{
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001269 struct cpa_data alias_cpa;
Tejun Heo992f4c12009-06-22 11:56:24 +09001270 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
Tejun Heoe933a732009-08-14 15:00:53 +09001271 unsigned long vaddr;
Tejun Heo992f4c12009-06-22 11:56:24 +09001272 int ret;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001273
Yinghai Lu8eb57792012-11-16 19:38:49 -08001274 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001275 return 0;
1276
Thomas Gleixnerf34b4392008-02-15 22:17:57 +01001277 /*
1278 * No need to redo, when the primary call touched the direct
1279 * mapping already:
1280 */
Thomas Hellstrom8523acf2009-08-03 09:25:45 +02001281 if (cpa->flags & CPA_PAGES_ARRAY) {
1282 struct page *page = cpa->pages[cpa->curpage];
1283 if (unlikely(PageHighMem(page)))
1284 return 0;
1285 vaddr = (unsigned long)page_address(page);
1286 } else if (cpa->flags & CPA_ARRAY)
Shaohua Lid75586a2008-08-21 10:46:06 +08001287 vaddr = cpa->vaddr[cpa->curpage];
1288 else
1289 vaddr = *cpa->vaddr;
1290
1291 if (!(within(vaddr, PAGE_OFFSET,
Suresh Siddhaa1e46212009-01-20 14:20:21 -08001292 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001293
Thomas Gleixnerf34b4392008-02-15 22:17:57 +01001294 alias_cpa = *cpa;
Tejun Heo992f4c12009-06-22 11:56:24 +09001295 alias_cpa.vaddr = &laddr;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001296 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
Shaohua Lid75586a2008-08-21 10:46:06 +08001297
Thomas Gleixnerf34b4392008-02-15 22:17:57 +01001298 ret = __change_page_attr_set_clr(&alias_cpa, 0);
Tejun Heo992f4c12009-06-22 11:56:24 +09001299 if (ret)
1300 return ret;
Thomas Gleixnerf34b4392008-02-15 22:17:57 +01001301 }
Ingo Molnar44af6c42008-01-30 13:34:03 +01001302
Arjan van de Ven488fd992008-01-30 13:34:07 +01001303#ifdef CONFIG_X86_64
Thomas Gleixner08797502008-01-30 13:34:09 +01001304 /*
Tejun Heo992f4c12009-06-22 11:56:24 +09001305 * If the primary call didn't touch the high mapping already
1306 * and the physical address is inside the kernel map, we need
Thomas Gleixner08797502008-01-30 13:34:09 +01001307 * to touch the high mapped kernel as well:
1308 */
Tejun Heo992f4c12009-06-22 11:56:24 +09001309 if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1310 within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) {
1311 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1312 __START_KERNEL_map - phys_base;
1313 alias_cpa = *cpa;
1314 alias_cpa.vaddr = &temp_cpa_vaddr;
1315 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
Thomas Gleixner08797502008-01-30 13:34:09 +01001316
Tejun Heo992f4c12009-06-22 11:56:24 +09001317 /*
1318 * The high mapping range is imprecise, so ignore the
1319 * return value.
1320 */
1321 __change_page_attr_set_clr(&alias_cpa, 0);
1322 }
Thomas Gleixner08797502008-01-30 13:34:09 +01001323#endif
Tejun Heo992f4c12009-06-22 11:56:24 +09001324
1325 return 0;
Ingo Molnar44af6c42008-01-30 13:34:03 +01001326}
1327
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001328static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
Thomas Gleixnerff314522008-01-30 13:34:08 +01001329{
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001330 int ret, numpages = cpa->numpages;
Thomas Gleixnerff314522008-01-30 13:34:08 +01001331
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001332 while (numpages) {
1333 /*
1334 * Store the remaining nr of pages for the large page
1335 * preservation check.
1336 */
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +01001337 cpa->numpages = numpages;
Shaohua Lid75586a2008-08-21 10:46:06 +08001338 /* for array changes, we can't use large page */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001339 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
Shaohua Lid75586a2008-08-21 10:46:06 +08001340 cpa->numpages = 1;
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001341
Suresh Siddhaad5ca552008-09-23 14:00:42 -07001342 if (!debug_pagealloc)
1343 spin_lock(&cpa_lock);
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001344 ret = __change_page_attr(cpa, checkalias);
Suresh Siddhaad5ca552008-09-23 14:00:42 -07001345 if (!debug_pagealloc)
1346 spin_unlock(&cpa_lock);
Thomas Gleixnerff314522008-01-30 13:34:08 +01001347 if (ret)
1348 return ret;
Thomas Gleixnerff314522008-01-30 13:34:08 +01001349
Thomas Gleixnerc31c7d42008-02-18 20:54:14 +01001350 if (checkalias) {
1351 ret = cpa_process_alias(cpa);
1352 if (ret)
1353 return ret;
1354 }
1355
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001356 /*
1357 * Adjust the number of pages with the result of the
1358 * CPA operation. Either a large page has been
1359 * preserved or a single page update happened.
1360 */
Matt Fleming74256372016-01-29 11:36:10 +00001361 BUG_ON(cpa->numpages > numpages || !cpa->numpages);
Rafael J. Wysocki9b5cf482008-03-03 01:17:37 +01001362 numpages -= cpa->numpages;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001363 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
Shaohua Lid75586a2008-08-21 10:46:06 +08001364 cpa->curpage++;
1365 else
1366 *cpa->vaddr += cpa->numpages * PAGE_SIZE;
1367
Thomas Gleixner65e074d2008-02-04 16:48:07 +01001368 }
Thomas Gleixnerff314522008-01-30 13:34:08 +01001369 return 0;
1370}
1371
Shaohua Lid75586a2008-08-21 10:46:06 +08001372static int change_page_attr_set_clr(unsigned long *addr, int numpages,
Andi Kleenc9caa022008-03-12 03:53:29 +01001373 pgprot_t mask_set, pgprot_t mask_clr,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001374 int force_split, int in_flag,
1375 struct page **pages)
Thomas Gleixnerff314522008-01-30 13:34:08 +01001376{
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001377 struct cpa_data cpa;
Ingo Molnarcacf8902008-08-21 13:46:33 +02001378 int ret, cache, checkalias;
Jack Steinerfa526d02009-09-03 12:56:02 -05001379 unsigned long baddr = 0;
Thomas Gleixner331e4062008-02-04 16:48:06 +01001380
Borislav Petkov82f07122013-10-31 17:25:07 +01001381 memset(&cpa, 0, sizeof(cpa));
1382
Thomas Gleixner331e4062008-02-04 16:48:06 +01001383 /*
1384 * Check, if we are requested to change a not supported
1385 * feature:
1386 */
1387 mask_set = canon_pgprot(mask_set);
1388 mask_clr = canon_pgprot(mask_clr);
Andi Kleenc9caa022008-03-12 03:53:29 +01001389 if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
Thomas Gleixner331e4062008-02-04 16:48:06 +01001390 return 0;
1391
Thomas Gleixner69b14152008-02-13 11:04:50 +01001392 /* Ensure we are PAGE_SIZE aligned */
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001393 if (in_flag & CPA_ARRAY) {
Shaohua Lid75586a2008-08-21 10:46:06 +08001394 int i;
1395 for (i = 0; i < numpages; i++) {
1396 if (addr[i] & ~PAGE_MASK) {
1397 addr[i] &= PAGE_MASK;
1398 WARN_ON_ONCE(1);
1399 }
1400 }
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001401 } else if (!(in_flag & CPA_PAGES_ARRAY)) {
1402 /*
1403 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
1404 * No need to cehck in that case
1405 */
1406 if (*addr & ~PAGE_MASK) {
1407 *addr &= PAGE_MASK;
1408 /*
1409 * People should not be passing in unaligned addresses:
1410 */
1411 WARN_ON_ONCE(1);
1412 }
Jack Steinerfa526d02009-09-03 12:56:02 -05001413 /*
1414 * Save address for cache flush. *addr is modified in the call
1415 * to __change_page_attr_set_clr() below.
1416 */
1417 baddr = *addr;
Thomas Gleixner69b14152008-02-13 11:04:50 +01001418 }
1419
Nick Piggin5843d9a2008-08-01 03:15:21 +02001420 /* Must avoid aliasing mappings in the highmem code */
1421 kmap_flush_unused();
1422
Nick Piggindb64fe02008-10-18 20:27:03 -07001423 vm_unmap_aliases();
1424
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001425 cpa.vaddr = addr;
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001426 cpa.pages = pages;
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001427 cpa.numpages = numpages;
1428 cpa.mask_set = mask_set;
1429 cpa.mask_clr = mask_clr;
Shaohua Lid75586a2008-08-21 10:46:06 +08001430 cpa.flags = 0;
1431 cpa.curpage = 0;
Andi Kleenc9caa022008-03-12 03:53:29 +01001432 cpa.force_split = force_split;
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001433
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001434 if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
1435 cpa.flags |= in_flag;
Shaohua Lid75586a2008-08-21 10:46:06 +08001436
Thomas Gleixneraf96e442008-02-15 21:49:46 +01001437 /* No alias checking for _NX bit modifications */
1438 checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
1439
1440 ret = __change_page_attr_set_clr(&cpa, checkalias);
Thomas Gleixnerff314522008-01-30 13:34:08 +01001441
Thomas Gleixner57a6a462008-01-30 13:34:08 +01001442 /*
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +01001443 * Check whether we really changed something:
1444 */
Shaohua Lid75586a2008-08-21 10:46:06 +08001445 if (!(cpa.flags & CPA_FLUSHTLB))
Shaohua Li1ac2f7d2008-08-04 14:51:24 +08001446 goto out;
Ingo Molnarcacf8902008-08-21 13:46:33 +02001447
Thomas Gleixnerf4ae5da2008-02-04 16:48:07 +01001448 /*
Andi Kleen6bb83832008-02-04 16:48:06 +01001449 * No need to flush, when we did not set any of the caching
1450 * attributes:
1451 */
Juergen Grossc06814d2014-11-03 14:01:57 +01001452 cache = !!pgprot2cachemode(mask_set);
Andi Kleen6bb83832008-02-04 16:48:06 +01001453
1454 /*
Borislav Petkovb82ad3d2014-03-12 15:13:04 +01001455 * On success we use CLFLUSH, when the CPU supports it to
1456 * avoid the WBINVD. If the CPU does not support it and in the
H. Peter Anvinf026cfa2012-08-14 09:53:38 -07001457 * error case we fall back to cpa_flush_all (which uses
Borislav Petkovb82ad3d2014-03-12 15:13:04 +01001458 * WBINVD):
Thomas Gleixner57a6a462008-01-30 13:34:08 +01001459 */
H. Peter Anvinf026cfa2012-08-14 09:53:38 -07001460 if (!ret && cpu_has_clflush) {
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001461 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
1462 cpa_flush_array(addr, numpages, cache,
1463 cpa.flags, pages);
1464 } else
Jack Steinerfa526d02009-09-03 12:56:02 -05001465 cpa_flush_range(baddr, numpages, cache);
Shaohua Lid75586a2008-08-21 10:46:06 +08001466 } else
Andi Kleen6bb83832008-02-04 16:48:06 +01001467 cpa_flush_all(cache);
Ingo Molnarcacf8902008-08-21 13:46:33 +02001468
Thomas Gleixner76ebd052008-02-09 23:24:09 +01001469out:
Thomas Gleixnerff314522008-01-30 13:34:08 +01001470 return ret;
1471}
1472
Shaohua Lid75586a2008-08-21 10:46:06 +08001473static inline int change_page_attr_set(unsigned long *addr, int numpages,
1474 pgprot_t mask, int array)
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001475{
Shaohua Lid75586a2008-08-21 10:46:06 +08001476 return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001477 (array ? CPA_ARRAY : 0), NULL);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001478}
1479
Shaohua Lid75586a2008-08-21 10:46:06 +08001480static inline int change_page_attr_clear(unsigned long *addr, int numpages,
1481 pgprot_t mask, int array)
Thomas Gleixner72932c72008-01-30 13:34:08 +01001482{
Shaohua Lid75586a2008-08-21 10:46:06 +08001483 return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001484 (array ? CPA_ARRAY : 0), NULL);
Thomas Gleixner72932c72008-01-30 13:34:08 +01001485}
1486
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001487static inline int cpa_set_pages_array(struct page **pages, int numpages,
1488 pgprot_t mask)
1489{
1490 return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
1491 CPA_PAGES_ARRAY, pages);
1492}
1493
1494static inline int cpa_clear_pages_array(struct page **pages, int numpages,
1495 pgprot_t mask)
1496{
1497 return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
1498 CPA_PAGES_ARRAY, pages);
1499}
1500
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001501int _set_memory_uc(unsigned long addr, int numpages)
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001502{
Suresh Siddhade33c442008-04-25 17:07:22 -07001503 /*
1504 * for now UC MINUS. see comments in ioremap_nocache()
Luis R. Rodrigueze4b6be332015-05-11 10:15:53 +02001505 * If you really need strong UC use ioremap_uc(), but note
1506 * that you cannot override IO areas with set_memory_*() as
1507 * these helpers cannot work with IO memory.
Suresh Siddhade33c442008-04-25 17:07:22 -07001508 */
Shaohua Lid75586a2008-08-21 10:46:06 +08001509 return change_page_attr_set(&addr, numpages,
Juergen Grossc06814d2014-11-03 14:01:57 +01001510 cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1511 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001512}
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001513
1514int set_memory_uc(unsigned long addr, int numpages)
1515{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001516 int ret;
1517
Suresh Siddhade33c442008-04-25 17:07:22 -07001518 /*
1519 * for now UC MINUS. see comments in ioremap_nocache()
1520 */
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001521 ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
Juergen Grosse00c8cc2014-11-03 14:01:59 +01001522 _PAGE_CACHE_MODE_UC_MINUS, NULL);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001523 if (ret)
1524 goto out_err;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001525
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001526 ret = _set_memory_uc(addr, numpages);
1527 if (ret)
1528 goto out_free;
1529
1530 return 0;
1531
1532out_free:
1533 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1534out_err:
1535 return ret;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001536}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001537EXPORT_SYMBOL(set_memory_uc);
1538
H Hartley Sweeten2d070ef2011-11-15 14:49:00 -08001539static int _set_memory_array(unsigned long *addr, int addrinarray,
Juergen Grossc06814d2014-11-03 14:01:57 +01001540 enum page_cache_mode new_type)
Shaohua Lid75586a2008-08-21 10:46:06 +08001541{
Toshi Kani623dffb2015-06-04 18:55:20 +02001542 enum page_cache_mode set_type;
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001543 int i, j;
1544 int ret;
1545
Shaohua Lid75586a2008-08-21 10:46:06 +08001546 for (i = 0; i < addrinarray; i++) {
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001547 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
Pauli Nieminen4f646252010-04-01 12:45:01 +00001548 new_type, NULL);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001549 if (ret)
1550 goto out_free;
Shaohua Lid75586a2008-08-21 10:46:06 +08001551 }
1552
Toshi Kani623dffb2015-06-04 18:55:20 +02001553 /* If WC, set to UC- first and then WC */
1554 set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1555 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1556
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001557 ret = change_page_attr_set(addr, addrinarray,
Toshi Kani623dffb2015-06-04 18:55:20 +02001558 cachemode2pgprot(set_type), 1);
Pauli Nieminen4f646252010-04-01 12:45:01 +00001559
Juergen Grossc06814d2014-11-03 14:01:57 +01001560 if (!ret && new_type == _PAGE_CACHE_MODE_WC)
Pauli Nieminen4f646252010-04-01 12:45:01 +00001561 ret = change_page_attr_set_clr(addr, addrinarray,
Juergen Grossc06814d2014-11-03 14:01:57 +01001562 cachemode2pgprot(
1563 _PAGE_CACHE_MODE_WC),
Pauli Nieminen4f646252010-04-01 12:45:01 +00001564 __pgprot(_PAGE_CACHE_MASK),
1565 0, CPA_ARRAY, NULL);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001566 if (ret)
1567 goto out_free;
Rene Hermanc5e147c2008-08-22 01:02:20 +02001568
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001569 return 0;
1570
1571out_free:
1572 for (j = 0; j < i; j++)
1573 free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
1574
1575 return ret;
Shaohua Lid75586a2008-08-21 10:46:06 +08001576}
Pauli Nieminen4f646252010-04-01 12:45:01 +00001577
1578int set_memory_array_uc(unsigned long *addr, int addrinarray)
1579{
Juergen Grossc06814d2014-11-03 14:01:57 +01001580 return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
Pauli Nieminen4f646252010-04-01 12:45:01 +00001581}
Shaohua Lid75586a2008-08-21 10:46:06 +08001582EXPORT_SYMBOL(set_memory_array_uc);
1583
Pauli Nieminen4f646252010-04-01 12:45:01 +00001584int set_memory_array_wc(unsigned long *addr, int addrinarray)
1585{
Juergen Grossc06814d2014-11-03 14:01:57 +01001586 return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC);
Pauli Nieminen4f646252010-04-01 12:45:01 +00001587}
1588EXPORT_SYMBOL(set_memory_array_wc);
1589
Toshi Kani623dffb2015-06-04 18:55:20 +02001590int set_memory_array_wt(unsigned long *addr, int addrinarray)
1591{
1592 return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WT);
1593}
1594EXPORT_SYMBOL_GPL(set_memory_array_wt);
1595
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001596int _set_memory_wc(unsigned long addr, int numpages)
1597{
venkatesh.pallipadi@intel.com3869c4a2009-04-09 14:26:50 -07001598 int ret;
Pallipadi, Venkateshbdc63402009-07-30 14:43:19 -07001599 unsigned long addr_copy = addr;
1600
venkatesh.pallipadi@intel.com3869c4a2009-04-09 14:26:50 -07001601 ret = change_page_attr_set(&addr, numpages,
Juergen Grossc06814d2014-11-03 14:01:57 +01001602 cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1603 0);
venkatesh.pallipadi@intel.com3869c4a2009-04-09 14:26:50 -07001604 if (!ret) {
Pallipadi, Venkateshbdc63402009-07-30 14:43:19 -07001605 ret = change_page_attr_set_clr(&addr_copy, numpages,
Juergen Grossc06814d2014-11-03 14:01:57 +01001606 cachemode2pgprot(
1607 _PAGE_CACHE_MODE_WC),
Pallipadi, Venkateshbdc63402009-07-30 14:43:19 -07001608 __pgprot(_PAGE_CACHE_MASK),
1609 0, 0, NULL);
venkatesh.pallipadi@intel.com3869c4a2009-04-09 14:26:50 -07001610 }
1611 return ret;
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001612}
1613
1614int set_memory_wc(unsigned long addr, int numpages)
1615{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001616 int ret;
1617
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001618 ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
Juergen Grosse00c8cc2014-11-03 14:01:59 +01001619 _PAGE_CACHE_MODE_WC, NULL);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001620 if (ret)
Toshi Kani623dffb2015-06-04 18:55:20 +02001621 return ret;
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001622
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001623 ret = _set_memory_wc(addr, numpages);
1624 if (ret)
Toshi Kani623dffb2015-06-04 18:55:20 +02001625 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001626
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001627 return ret;
venkatesh.pallipadi@intel.comef354af2008-03-18 17:00:23 -07001628}
1629EXPORT_SYMBOL(set_memory_wc);
1630
Toshi Kani623dffb2015-06-04 18:55:20 +02001631int _set_memory_wt(unsigned long addr, int numpages)
1632{
1633 return change_page_attr_set(&addr, numpages,
1634 cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
1635}
1636
1637int set_memory_wt(unsigned long addr, int numpages)
1638{
1639 int ret;
1640
1641 ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1642 _PAGE_CACHE_MODE_WT, NULL);
1643 if (ret)
1644 return ret;
1645
1646 ret = _set_memory_wt(addr, numpages);
1647 if (ret)
1648 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1649
1650 return ret;
1651}
1652EXPORT_SYMBOL_GPL(set_memory_wt);
1653
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001654int _set_memory_wb(unsigned long addr, int numpages)
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001655{
Juergen Grossc06814d2014-11-03 14:01:57 +01001656 /* WB cache mode is hard wired to all cache attribute bits being 0 */
Shaohua Lid75586a2008-08-21 10:46:06 +08001657 return change_page_attr_clear(&addr, numpages,
1658 __pgprot(_PAGE_CACHE_MASK), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001659}
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001660
1661int set_memory_wb(unsigned long addr, int numpages)
1662{
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001663 int ret;
1664
1665 ret = _set_memory_wb(addr, numpages);
1666 if (ret)
1667 return ret;
1668
venkatesh.pallipadi@intel.comc15238d2008-08-20 16:45:51 -07001669 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001670 return 0;
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -07001671}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001672EXPORT_SYMBOL(set_memory_wb);
1673
Shaohua Lid75586a2008-08-21 10:46:06 +08001674int set_memory_array_wb(unsigned long *addr, int addrinarray)
1675{
1676 int i;
venkatesh.pallipadi@intel.coma5593e02009-04-09 14:26:48 -07001677 int ret;
1678
Juergen Grossc06814d2014-11-03 14:01:57 +01001679 /* WB cache mode is hard wired to all cache attribute bits being 0 */
venkatesh.pallipadi@intel.coma5593e02009-04-09 14:26:48 -07001680 ret = change_page_attr_clear(addr, addrinarray,
1681 __pgprot(_PAGE_CACHE_MASK), 1);
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001682 if (ret)
1683 return ret;
Shaohua Lid75586a2008-08-21 10:46:06 +08001684
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001685 for (i = 0; i < addrinarray; i++)
1686 free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
Rene Hermanc5e147c2008-08-22 01:02:20 +02001687
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001688 return 0;
Shaohua Lid75586a2008-08-21 10:46:06 +08001689}
1690EXPORT_SYMBOL(set_memory_array_wb);
1691
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001692int set_memory_x(unsigned long addr, int numpages)
1693{
H. Peter Anvin583140a2009-11-13 15:28:15 -08001694 if (!(__supported_pte_mask & _PAGE_NX))
1695 return 0;
1696
Shaohua Lid75586a2008-08-21 10:46:06 +08001697 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001698}
1699EXPORT_SYMBOL(set_memory_x);
1700
1701int set_memory_nx(unsigned long addr, int numpages)
1702{
H. Peter Anvin583140a2009-11-13 15:28:15 -08001703 if (!(__supported_pte_mask & _PAGE_NX))
1704 return 0;
1705
Shaohua Lid75586a2008-08-21 10:46:06 +08001706 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001707}
1708EXPORT_SYMBOL(set_memory_nx);
1709
1710int set_memory_ro(unsigned long addr, int numpages)
1711{
Shaohua Lid75586a2008-08-21 10:46:06 +08001712 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001713}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001714
1715int set_memory_rw(unsigned long addr, int numpages)
1716{
Shaohua Lid75586a2008-08-21 10:46:06 +08001717 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001718}
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001719
1720int set_memory_np(unsigned long addr, int numpages)
1721{
Shaohua Lid75586a2008-08-21 10:46:06 +08001722 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001723}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001724
Andi Kleenc9caa022008-03-12 03:53:29 +01001725int set_memory_4k(unsigned long addr, int numpages)
1726{
Shaohua Lid75586a2008-08-21 10:46:06 +08001727 return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
venkatesh.pallipadi@intel.com9ae28472009-03-19 14:51:14 -07001728 __pgprot(0), 1, 0, NULL);
Andi Kleenc9caa022008-03-12 03:53:29 +01001729}
1730
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001731int set_pages_uc(struct page *page, int numpages)
1732{
1733 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001734
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001735 return set_memory_uc(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001736}
1737EXPORT_SYMBOL(set_pages_uc);
1738
Pauli Nieminen4f646252010-04-01 12:45:01 +00001739static int _set_pages_array(struct page **pages, int addrinarray,
Juergen Grossc06814d2014-11-03 14:01:57 +01001740 enum page_cache_mode new_type)
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001741{
1742 unsigned long start;
1743 unsigned long end;
Toshi Kani623dffb2015-06-04 18:55:20 +02001744 enum page_cache_mode set_type;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001745 int i;
1746 int free_idx;
Pauli Nieminen4f646252010-04-01 12:45:01 +00001747 int ret;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001748
1749 for (i = 0; i < addrinarray; i++) {
Thomas Hellstrom8523acf2009-08-03 09:25:45 +02001750 if (PageHighMem(pages[i]))
1751 continue;
1752 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001753 end = start + PAGE_SIZE;
Pauli Nieminen4f646252010-04-01 12:45:01 +00001754 if (reserve_memtype(start, end, new_type, NULL))
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001755 goto err_out;
1756 }
1757
Toshi Kani623dffb2015-06-04 18:55:20 +02001758 /* If WC, set to UC- first and then WC */
1759 set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1760 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1761
Pauli Nieminen4f646252010-04-01 12:45:01 +00001762 ret = cpa_set_pages_array(pages, addrinarray,
Toshi Kani623dffb2015-06-04 18:55:20 +02001763 cachemode2pgprot(set_type));
Juergen Grossc06814d2014-11-03 14:01:57 +01001764 if (!ret && new_type == _PAGE_CACHE_MODE_WC)
Pauli Nieminen4f646252010-04-01 12:45:01 +00001765 ret = change_page_attr_set_clr(NULL, addrinarray,
Juergen Grossc06814d2014-11-03 14:01:57 +01001766 cachemode2pgprot(
1767 _PAGE_CACHE_MODE_WC),
Pauli Nieminen4f646252010-04-01 12:45:01 +00001768 __pgprot(_PAGE_CACHE_MASK),
1769 0, CPA_PAGES_ARRAY, pages);
1770 if (ret)
1771 goto err_out;
1772 return 0; /* Success */
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001773err_out:
1774 free_idx = i;
1775 for (i = 0; i < free_idx; i++) {
Thomas Hellstrom8523acf2009-08-03 09:25:45 +02001776 if (PageHighMem(pages[i]))
1777 continue;
1778 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001779 end = start + PAGE_SIZE;
1780 free_memtype(start, end);
1781 }
1782 return -EINVAL;
1783}
Pauli Nieminen4f646252010-04-01 12:45:01 +00001784
1785int set_pages_array_uc(struct page **pages, int addrinarray)
1786{
Juergen Grossc06814d2014-11-03 14:01:57 +01001787 return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
Pauli Nieminen4f646252010-04-01 12:45:01 +00001788}
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001789EXPORT_SYMBOL(set_pages_array_uc);
1790
Pauli Nieminen4f646252010-04-01 12:45:01 +00001791int set_pages_array_wc(struct page **pages, int addrinarray)
1792{
Juergen Grossc06814d2014-11-03 14:01:57 +01001793 return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC);
Pauli Nieminen4f646252010-04-01 12:45:01 +00001794}
1795EXPORT_SYMBOL(set_pages_array_wc);
1796
Toshi Kani623dffb2015-06-04 18:55:20 +02001797int set_pages_array_wt(struct page **pages, int addrinarray)
1798{
1799 return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WT);
1800}
1801EXPORT_SYMBOL_GPL(set_pages_array_wt);
1802
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001803int set_pages_wb(struct page *page, int numpages)
1804{
1805 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001806
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001807 return set_memory_wb(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001808}
1809EXPORT_SYMBOL(set_pages_wb);
1810
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001811int set_pages_array_wb(struct page **pages, int addrinarray)
1812{
1813 int retval;
1814 unsigned long start;
1815 unsigned long end;
1816 int i;
1817
Juergen Grossc06814d2014-11-03 14:01:57 +01001818 /* WB cache mode is hard wired to all cache attribute bits being 0 */
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001819 retval = cpa_clear_pages_array(pages, addrinarray,
1820 __pgprot(_PAGE_CACHE_MASK));
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001821 if (retval)
1822 return retval;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001823
1824 for (i = 0; i < addrinarray; i++) {
Thomas Hellstrom8523acf2009-08-03 09:25:45 +02001825 if (PageHighMem(pages[i]))
1826 continue;
1827 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001828 end = start + PAGE_SIZE;
1829 free_memtype(start, end);
1830 }
1831
venkatesh.pallipadi@intel.com9fa3ab32009-04-09 14:26:49 -07001832 return 0;
venkatesh.pallipadi@intel.com0f350752009-03-19 14:51:15 -07001833}
1834EXPORT_SYMBOL(set_pages_array_wb);
1835
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001836int set_pages_x(struct page *page, int numpages)
1837{
1838 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001839
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001840 return set_memory_x(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001841}
1842EXPORT_SYMBOL(set_pages_x);
1843
1844int set_pages_nx(struct page *page, int numpages)
1845{
1846 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001847
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001848 return set_memory_nx(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001849}
1850EXPORT_SYMBOL(set_pages_nx);
1851
1852int set_pages_ro(struct page *page, int numpages)
1853{
1854 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001855
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001856 return set_memory_ro(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001857}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001858
1859int set_pages_rw(struct page *page, int numpages)
1860{
1861 unsigned long addr = (unsigned long)page_address(page);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001862
Thomas Gleixnerd7c8f212008-01-30 13:34:07 +01001863 return set_memory_rw(addr, numpages);
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001864}
Arjan van de Ven75cbade2008-01-30 13:34:06 +01001865
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866#ifdef CONFIG_DEBUG_PAGEALLOC
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001867
1868static int __set_pages_p(struct page *page, int numpages)
1869{
Shaohua Lid75586a2008-08-21 10:46:06 +08001870 unsigned long tempaddr = (unsigned long) page_address(page);
1871 struct cpa_data cpa = { .vaddr = &tempaddr,
Borislav Petkov82f07122013-10-31 17:25:07 +01001872 .pgd = NULL,
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001873 .numpages = numpages,
1874 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
Shaohua Lid75586a2008-08-21 10:46:06 +08001875 .mask_clr = __pgprot(0),
1876 .flags = 0};
Thomas Gleixner72932c72008-01-30 13:34:08 +01001877
Suresh Siddha55121b42008-09-23 14:00:40 -07001878 /*
1879 * No alias checking needed for setting present flag. otherwise,
1880 * we may need to break large pages for 64-bit kernel text
1881 * mappings (this adds to complexity if we want to do this from
1882 * atomic context especially). Let's keep it simple!
1883 */
1884 return __change_page_attr_set_clr(&cpa, 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001885}
1886
1887static int __set_pages_np(struct page *page, int numpages)
1888{
Shaohua Lid75586a2008-08-21 10:46:06 +08001889 unsigned long tempaddr = (unsigned long) page_address(page);
1890 struct cpa_data cpa = { .vaddr = &tempaddr,
Borislav Petkov82f07122013-10-31 17:25:07 +01001891 .pgd = NULL,
Thomas Gleixner72e458d2008-02-04 16:48:07 +01001892 .numpages = numpages,
1893 .mask_set = __pgprot(0),
Shaohua Lid75586a2008-08-21 10:46:06 +08001894 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
1895 .flags = 0};
Thomas Gleixner72932c72008-01-30 13:34:08 +01001896
Suresh Siddha55121b42008-09-23 14:00:40 -07001897 /*
1898 * No alias checking needed for setting not present flag. otherwise,
1899 * we may need to break large pages for 64-bit kernel text
1900 * mappings (this adds to complexity if we want to do this from
1901 * atomic context especially). Let's keep it simple!
1902 */
1903 return __change_page_attr_set_clr(&cpa, 0);
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001904}
1905
Joonsoo Kim031bc572014-12-12 16:55:52 -08001906void __kernel_map_pages(struct page *page, int numpages, int enable)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907{
1908 if (PageHighMem(page))
1909 return;
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001910 if (!enable) {
Ingo Molnarf9b84042006-06-27 02:54:49 -07001911 debug_check_no_locks_freed(page_address(page),
1912 numpages * PAGE_SIZE);
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001913 }
Ingo Molnarde5097c2006-01-09 15:59:21 -08001914
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001915 /*
Ingo Molnarf8d84062008-02-13 14:09:53 +01001916 * The return value is ignored as the calls cannot fail.
Suresh Siddha55121b42008-09-23 14:00:40 -07001917 * Large pages for identity mappings are not used at boot time
1918 * and hence no memory allocations during large page split.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919 */
Ingo Molnarf62d0f02008-01-30 13:34:07 +01001920 if (enable)
1921 __set_pages_p(page, numpages);
1922 else
1923 __set_pages_np(page, numpages);
Ingo Molnar9f4c8152008-01-30 13:33:41 +01001924
1925 /*
Ingo Molnare4b71dc2008-01-30 13:34:04 +01001926 * We should perform an IPI and flush all tlbs,
1927 * but that can deadlock->flush only current cpu:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 */
1929 __flush_tlb_all();
Boris Ostrovsky26564602013-04-11 13:59:52 -04001930
1931 arch_flush_lazy_mmu_mode();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932}
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01001933
1934#ifdef CONFIG_HIBERNATION
1935
1936bool kernel_page_present(struct page *page)
1937{
1938 unsigned int level;
1939 pte_t *pte;
1940
1941 if (PageHighMem(page))
1942 return false;
1943
1944 pte = lookup_address((unsigned long)page_address(page), &level);
1945 return (pte_val(*pte) & _PAGE_PRESENT);
1946}
1947
1948#endif /* CONFIG_HIBERNATION */
1949
1950#endif /* CONFIG_DEBUG_PAGEALLOC */
Arjan van de Vend1028a12008-01-30 13:34:07 +01001951
Borislav Petkov82f07122013-10-31 17:25:07 +01001952int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
1953 unsigned numpages, unsigned long page_flags)
1954{
1955 int retval = -EINVAL;
1956
1957 struct cpa_data cpa = {
1958 .vaddr = &address,
1959 .pfn = pfn,
1960 .pgd = pgd,
1961 .numpages = numpages,
1962 .mask_set = __pgprot(0),
1963 .mask_clr = __pgprot(0),
1964 .flags = 0,
1965 };
1966
1967 if (!(__supported_pte_mask & _PAGE_NX))
1968 goto out;
1969
1970 if (!(page_flags & _PAGE_NX))
1971 cpa.mask_clr = __pgprot(_PAGE_NX);
1972
1973 cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
1974
1975 retval = __change_page_attr_set_clr(&cpa, 0);
1976 __flush_tlb_all();
1977
1978out:
1979 return retval;
1980}
1981
Borislav Petkov42a54772014-01-18 12:48:16 +01001982void kernel_unmap_pages_in_pgd(pgd_t *root, unsigned long address,
1983 unsigned numpages)
1984{
1985 unmap_pgd_range(root, address, address + (numpages << PAGE_SHIFT));
1986}
1987
Arjan van de Vend1028a12008-01-30 13:34:07 +01001988/*
1989 * The testcases use internal knowledge of the implementation that shouldn't
1990 * be exposed to the rest of the kernel. Include these directly here.
1991 */
1992#ifdef CONFIG_CPA_DEBUG
1993#include "pageattr-test.c"
1994#endif