blob: bb6e8a267bfeb9b75fd62277add0fce45758d836 [file] [log] [blame]
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -07001/*
2 * Handle caching attributes in page tables (PAT)
3 *
4 * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 * Suresh B Siddha <suresh.b.siddha@intel.com>
6 *
7 * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
8 */
9
10#include <linux/mm.h>
11#include <linux/kernel.h>
12#include <linux/gfp.h>
13#include <linux/fs.h>
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -070014#include <linux/bootmem.h>
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -070015#include <linux/debugfs.h>
16#include <linux/seq_file.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070017
18#include <asm/msr.h>
19#include <asm/tlbflush.h>
20#include <asm/processor.h>
Venki Pallipadi0124cec2008-04-26 11:32:12 -070021#include <asm/page.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070022#include <asm/pgtable.h>
23#include <asm/pat.h>
24#include <asm/e820.h>
25#include <asm/cacheflush.h>
26#include <asm/fcntl.h>
27#include <asm/mtrr.h>
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -070028#include <asm/io.h>
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070029
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020030#ifdef CONFIG_X86_PAT
Andreas Herrmann499f8f82008-06-10 16:06:21 +020031int __read_mostly pat_enabled = 1;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070032
Avi Kivity31f4d872008-05-14 12:20:32 +030033void __cpuinit pat_disable(char *reason)
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020034{
Andreas Herrmann499f8f82008-06-10 16:06:21 +020035 pat_enabled = 0;
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020036 printk(KERN_INFO "%s\n", reason);
37}
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070038
Andrew Mortonbe524fb2008-05-29 00:01:28 -070039static int __init nopat(char *str)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070040{
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020041 pat_disable("PAT support disabled.");
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070042 return 0;
43}
44early_param("nopat", nopat);
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020045#endif
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070046
Venki Pallipadi77b52b42008-05-05 19:09:10 -070047
48static int debug_enable;
49static int __init pat_debug_setup(char *str)
50{
51 debug_enable = 1;
52 return 0;
53}
54__setup("debugpat", pat_debug_setup);
55
56#define dprintk(fmt, arg...) \
57 do { if (debug_enable) printk(KERN_INFO fmt, ##arg); } while (0)
58
59
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020060static u64 __read_mostly boot_pat_state;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070061
62enum {
63 PAT_UC = 0, /* uncached */
64 PAT_WC = 1, /* Write combining */
65 PAT_WT = 4, /* Write Through */
66 PAT_WP = 5, /* Write Protected */
67 PAT_WB = 6, /* Write Back (default) */
68 PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */
69};
70
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +020071#define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070072
73void pat_init(void)
74{
75 u64 pat;
76
Andreas Herrmann499f8f82008-06-10 16:06:21 +020077 if (!pat_enabled)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070078 return;
79
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020080 /* Paranoia check. */
Andreas Herrmann97cfab62008-06-10 16:05:18 +020081 if (!cpu_has_pat && boot_pat_state) {
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020082 /*
Andreas Herrmann97cfab62008-06-10 16:05:18 +020083 * If this happens we are on a secondary CPU, but
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020084 * switched to PAT on the boot CPU. We have no way to
85 * undo PAT.
Andreas Herrmann97cfab62008-06-10 16:05:18 +020086 */
87 printk(KERN_ERR "PAT enabled, "
88 "but not supported by secondary CPU\n");
89 BUG();
Thomas Gleixner8d4a4302008-05-08 09:18:43 +020090 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -070091
92 /* Set PWT to Write-Combining. All other bits stay the same */
93 /*
94 * PTE encoding used in Linux:
95 * PAT
96 * |PCD
97 * ||PWT
98 * |||
99 * 000 WB _PAGE_CACHE_WB
100 * 001 WC _PAGE_CACHE_WC
101 * 010 UC- _PAGE_CACHE_UC_MINUS
102 * 011 UC _PAGE_CACHE_UC
103 * PAT bit unused
104 */
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200105 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
106 PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700107
108 /* Boot CPU check */
Thomas Gleixner8d4a4302008-05-08 09:18:43 +0200109 if (!boot_pat_state)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700110 rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700111
112 wrmsrl(MSR_IA32_CR_PAT, pat);
113 printk(KERN_INFO "x86 PAT enabled: cpu %d, old 0x%Lx, new 0x%Lx\n",
114 smp_processor_id(), boot_pat_state, pat);
115}
116
117#undef PAT
118
119static char *cattr_name(unsigned long flags)
120{
121 switch (flags & _PAGE_CACHE_MASK) {
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200122 case _PAGE_CACHE_UC: return "uncached";
123 case _PAGE_CACHE_UC_MINUS: return "uncached-minus";
124 case _PAGE_CACHE_WB: return "write-back";
125 case _PAGE_CACHE_WC: return "write-combining";
126 default: return "broken";
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700127 }
128}
129
130/*
131 * The global memtype list keeps track of memory type for specific
132 * physical memory areas. Conflicting memory types in different
133 * mappings can cause CPU cache corruption. To avoid this we keep track.
134 *
135 * The list is sorted based on starting address and can contain multiple
136 * entries for each address (this allows reference counting for overlapping
137 * areas). All the aliases have the same cache attributes of course.
138 * Zero attributes are represented as holes.
139 *
140 * Currently the data structure is a list because the number of mappings
141 * are expected to be relatively small. If this should be a problem
142 * it could be changed to a rbtree or similar.
143 *
144 * memtype_lock protects the whole list.
145 */
146
147struct memtype {
148 u64 start;
149 u64 end;
150 unsigned long type;
151 struct list_head nd;
152};
153
154static LIST_HEAD(memtype_list);
155static DEFINE_SPINLOCK(memtype_lock); /* protects memtype list */
156
157/*
158 * Does intersection of PAT memory type and MTRR memory type and returns
159 * the resulting memory type as PAT understands it.
160 * (Type in pat and mtrr will not have same value)
161 * The intersection is based on "Effective Memory Type" tables in IA-32
162 * SDM vol 3a
163 */
Hugh Dickins6cf514f2008-06-16 18:42:43 +0100164static unsigned long pat_x_mtrr_type(u64 start, u64 end, unsigned long req_type)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700165{
Venki Pallipadic26421d2008-05-29 12:01:44 -0700166 /*
167 * Look for MTRR hint to get the effective type in case where PAT
168 * request is for WB.
169 */
Andreas Herrmanndd0c7c42008-06-18 15:38:57 +0200170 if (req_type == _PAGE_CACHE_WB) {
171 u8 mtrr_type;
172
173 mtrr_type = mtrr_type_lookup(start, end);
174 if (mtrr_type == MTRR_TYPE_UNCACHABLE)
175 return _PAGE_CACHE_UC;
176 if (mtrr_type == MTRR_TYPE_WRCOMB)
177 return _PAGE_CACHE_WC;
178 }
179
180 return req_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700181}
182
Andreas Herrmann64fe44c2008-06-20 22:07:09 +0200183static int chk_conflict(struct memtype *new, struct memtype *entry,
184 unsigned long *type)
185{
186 if (new->type != entry->type) {
187 if (type) {
188 new->type = entry->type;
189 *type = entry->type;
190 } else
191 goto conflict;
192 }
193
194 /* check overlaps with more than one entry in the list */
195 list_for_each_entry_continue(entry, &memtype_list, nd) {
196 if (new->end <= entry->start)
197 break;
198 else if (new->type != entry->type)
199 goto conflict;
200 }
201 return 0;
202
203 conflict:
204 printk(KERN_INFO "%s:%d conflicting memory types "
205 "%Lx-%Lx %s<->%s\n", current->comm, current->pid, new->start,
206 new->end, cattr_name(new->type), cattr_name(entry->type));
207 return -EBUSY;
208}
209
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700210static struct memtype *cached_entry;
211static u64 cached_start;
212
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700213/*
214 * req_type typically has one of the:
215 * - _PAGE_CACHE_WB
216 * - _PAGE_CACHE_WC
217 * - _PAGE_CACHE_UC_MINUS
218 * - _PAGE_CACHE_UC
219 *
220 * req_type will have a special case value '-1', when requester want to inherit
221 * the memory type from mtrr (if WB), existing PAT, defaulting to UC_MINUS.
222 *
Andreas Herrmannac979912008-06-20 22:01:49 +0200223 * If new_type is NULL, function will return an error if it cannot reserve the
224 * region with req_type. If new_type is non-NULL, function will return
225 * available type in new_type in case of no error. In case of any error
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700226 * it will return a negative return value.
227 */
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700228int reserve_memtype(u64 start, u64 end, unsigned long req_type,
Andreas Herrmannac979912008-06-20 22:01:49 +0200229 unsigned long *new_type)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700230{
Andreas Herrmannac979912008-06-20 22:01:49 +0200231 struct memtype *new, *entry;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700232 unsigned long actual_type;
Andreas Herrmannf6887262008-06-20 22:05:37 +0200233 struct list_head *where;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700234 int err = 0;
235
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200236 BUG_ON(start >= end); /* end is exclusive */
237
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200238 if (!pat_enabled) {
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700239 /* This is identical to page table setting without PAT */
Andreas Herrmannac979912008-06-20 22:01:49 +0200240 if (new_type) {
241 if (req_type == -1)
242 *new_type = _PAGE_CACHE_WB;
243 else
244 *new_type = req_type & _PAGE_CACHE_MASK;
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700245 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700246 return 0;
247 }
248
249 /* Low ISA region is always mapped WB in page table. No need to track */
Andreas Herrmannbcc643d2008-06-20 21:58:46 +0200250 if (is_ISA_range(start, end - 1)) {
Andreas Herrmannac979912008-06-20 22:01:49 +0200251 if (new_type)
252 *new_type = _PAGE_CACHE_WB;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700253 return 0;
254 }
255
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700256 if (req_type == -1) {
257 /*
Venki Pallipadic26421d2008-05-29 12:01:44 -0700258 * Call mtrr_lookup to get the type hint. This is an
259 * optimization for /dev/mem mmap'ers into WB memory (BIOS
260 * tools and ACPI tools). Use WB request for WB memory and use
261 * UC_MINUS otherwise.
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700262 */
263 u8 mtrr_type = mtrr_type_lookup(start, end);
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700264
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200265 if (mtrr_type == MTRR_TYPE_WRBACK)
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700266 actual_type = _PAGE_CACHE_WB;
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200267 else
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700268 actual_type = _PAGE_CACHE_UC_MINUS;
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200269 } else
270 actual_type = pat_x_mtrr_type(start, end,
271 req_type & _PAGE_CACHE_MASK);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700272
Andreas Herrmannac979912008-06-20 22:01:49 +0200273 new = kmalloc(sizeof(struct memtype), GFP_KERNEL);
274 if (!new)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700275 return -ENOMEM;
276
Andreas Herrmannac979912008-06-20 22:01:49 +0200277 new->start = start;
278 new->end = end;
279 new->type = actual_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700280
Andreas Herrmannac979912008-06-20 22:01:49 +0200281 if (new_type)
282 *new_type = actual_type;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700283
284 spin_lock(&memtype_lock);
285
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700286 if (cached_entry && start >= cached_start)
287 entry = cached_entry;
288 else
289 entry = list_entry(&memtype_list, struct memtype, nd);
290
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700291 /* Search for existing mapping that overlaps the current range */
Andreas Herrmannf6887262008-06-20 22:05:37 +0200292 where = NULL;
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700293 list_for_each_entry_continue(entry, &memtype_list, nd) {
Andreas Herrmann33af9032008-06-20 22:08:37 +0200294 if (end <= entry->start) {
Andreas Herrmannf6887262008-06-20 22:05:37 +0200295 where = entry->nd.prev;
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700296 cached_entry = list_entry(where, struct memtype, nd);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700297 break;
Andreas Herrmann33af9032008-06-20 22:08:37 +0200298 } else if (start <= entry->start) { /* end > entry->start */
Andreas Herrmann64fe44c2008-06-20 22:07:09 +0200299 err = chk_conflict(new, entry, new_type);
Andreas Herrmann33af9032008-06-20 22:08:37 +0200300 if (!err) {
301 dprintk("Overlap at 0x%Lx-0x%Lx\n",
302 entry->start, entry->end);
303 where = entry->nd.prev;
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700304 cached_entry = list_entry(where,
305 struct memtype, nd);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700306 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700307 break;
Andreas Herrmann33af9032008-06-20 22:08:37 +0200308 } else if (start < entry->end) { /* start > entry->start */
Andreas Herrmann64fe44c2008-06-20 22:07:09 +0200309 err = chk_conflict(new, entry, new_type);
Andreas Herrmann33af9032008-06-20 22:08:37 +0200310 if (!err) {
311 dprintk("Overlap at 0x%Lx-0x%Lx\n",
312 entry->start, entry->end);
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700313 cached_entry = list_entry(entry->nd.prev,
314 struct memtype, nd);
315
316 /*
317 * Move to right position in the linked
318 * list to add this new entry
319 */
320 list_for_each_entry_continue(entry,
321 &memtype_list, nd) {
322 if (start <= entry->start) {
323 where = entry->nd.prev;
324 break;
325 }
326 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700327 }
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700328 break;
329 }
330 }
331
332 if (err) {
Andreas Herrmann3e9c83b2008-06-20 22:04:02 +0200333 printk(KERN_INFO "reserve_memtype failed 0x%Lx-0x%Lx, "
334 "track %s, req %s\n",
335 start, end, cattr_name(new->type), cattr_name(req_type));
Andreas Herrmannac979912008-06-20 22:01:49 +0200336 kfree(new);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700337 spin_unlock(&memtype_lock);
338 return err;
339 }
340
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700341 cached_start = start;
342
Andreas Herrmannf6887262008-06-20 22:05:37 +0200343 if (where)
344 list_add(&new->nd, where);
345 else
Andreas Herrmannac979912008-06-20 22:01:49 +0200346 list_add_tail(&new->nd, &memtype_list);
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700347
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700348 spin_unlock(&memtype_lock);
Andreas Herrmann3e9c83b2008-06-20 22:04:02 +0200349
350 dprintk("reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
351 start, end, cattr_name(new->type), cattr_name(req_type),
352 new_type ? cattr_name(*new_type) : "-");
353
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700354 return err;
355}
356
357int free_memtype(u64 start, u64 end)
358{
Andreas Herrmannac979912008-06-20 22:01:49 +0200359 struct memtype *entry;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700360 int err = -EINVAL;
361
Andreas Herrmann69e26be2008-06-20 22:03:06 +0200362 if (!pat_enabled)
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700363 return 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700364
365 /* Low ISA region is always mapped WB. No need to track */
Andreas Herrmannbcc643d2008-06-20 21:58:46 +0200366 if (is_ISA_range(start, end - 1))
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700367 return 0;
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700368
369 spin_lock(&memtype_lock);
Andreas Herrmannac979912008-06-20 22:01:49 +0200370 list_for_each_entry(entry, &memtype_list, nd) {
371 if (entry->start == start && entry->end == end) {
Venki Pallipadi80c5e732008-08-19 16:28:01 -0700372 if (cached_entry == entry || cached_start == start)
373 cached_entry = NULL;
374
Andreas Herrmannac979912008-06-20 22:01:49 +0200375 list_del(&entry->nd);
376 kfree(entry);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700377 err = 0;
378 break;
379 }
380 }
381 spin_unlock(&memtype_lock);
382
383 if (err) {
Ingo Molnar28eb559b2008-04-03 10:14:33 +0200384 printk(KERN_INFO "%s:%d freeing invalid memtype %Lx-%Lx\n",
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700385 current->comm, current->pid, start, end);
386 }
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700387
Venki Pallipadi77b52b42008-05-05 19:09:10 -0700388 dprintk("free_memtype request 0x%Lx-0x%Lx\n", start, end);
venkatesh.pallipadi@intel.com2e5d9c82008-03-18 17:00:14 -0700389 return err;
390}
391
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700392
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700393/*
394 * /dev/mem mmap interface. The memtype used for mapping varies:
395 * - Use UC for mappings with O_SYNC flag
396 * - Without O_SYNC flag, if there is any conflict in reserve_memtype,
397 * inherit the memtype from existing mapping.
398 * - Else use UC_MINUS memtype (for backward compatibility with existing
399 * X drivers.
400 */
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700401pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
402 unsigned long size, pgprot_t vma_prot)
403{
404 return vma_prot;
405}
406
Ingo Molnard0926332008-07-18 00:26:59 +0200407#ifdef CONFIG_STRICT_DEVMEM
408/* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM*/
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700409static inline int range_is_allowed(unsigned long pfn, unsigned long size)
410{
411 return 1;
412}
413#else
414static inline int range_is_allowed(unsigned long pfn, unsigned long size)
415{
416 u64 from = ((u64)pfn) << PAGE_SHIFT;
417 u64 to = from + size;
418 u64 cursor = from;
419
420 while (cursor < to) {
421 if (!devmem_is_allowed(pfn)) {
422 printk(KERN_INFO
423 "Program %s tried to access /dev/mem between %Lx->%Lx.\n",
424 current->comm, from, to);
425 return 0;
426 }
427 cursor += PAGE_SIZE;
428 pfn++;
429 }
430 return 1;
431}
Ingo Molnard0926332008-07-18 00:26:59 +0200432#endif /* CONFIG_STRICT_DEVMEM */
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700433
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700434int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
435 unsigned long size, pgprot_t *vma_prot)
436{
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700437 u64 offset = ((u64) pfn) << PAGE_SHIFT;
438 unsigned long flags = _PAGE_CACHE_UC_MINUS;
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700439 int retval;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700440
Venki Pallipadi0124cec2008-04-26 11:32:12 -0700441 if (!range_is_allowed(pfn, size))
442 return 0;
443
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700444 if (file->f_flags & O_SYNC) {
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700445 flags = _PAGE_CACHE_UC;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700446 }
447
448#ifdef CONFIG_X86_32
449 /*
450 * On the PPro and successors, the MTRRs are used to set
451 * memory types for physical addresses outside main memory,
452 * so blindly setting UC or PWT on those pages is wrong.
453 * For Pentiums and earlier, the surround logic should disable
454 * caching for the high addresses through the KEN pin, but
455 * we maintain the tradition of paranoia in this code.
456 */
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200457 if (!pat_enabled &&
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200458 !(boot_cpu_has(X86_FEATURE_MTRR) ||
459 boot_cpu_has(X86_FEATURE_K6_MTRR) ||
460 boot_cpu_has(X86_FEATURE_CYRIX_ARR) ||
461 boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) &&
462 (pfn << PAGE_SHIFT) >= __pa(high_memory)) {
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700463 flags = _PAGE_CACHE_UC;
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700464 }
465#endif
466
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700467 /*
468 * With O_SYNC, we can only take UC mapping. Fail if we cannot.
469 * Without O_SYNC, we want to get
470 * - WB for WB-able memory and no other conflicting mappings
471 * - UC_MINUS for non-WB-able memory with no other conflicting mappings
472 * - Inherit from confliting mappings otherwise
473 */
474 if (flags != _PAGE_CACHE_UC_MINUS) {
475 retval = reserve_memtype(offset, offset + size, flags, NULL);
476 } else {
Ingo Molnarf022bfd2008-03-21 15:42:28 +0100477 retval = reserve_memtype(offset, offset + size, -1, &flags);
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700478 }
479
480 if (retval < 0)
481 return 0;
482
Yinghai Lu965194c2008-07-12 14:31:28 -0700483 if (((pfn < max_low_pfn_mapped) ||
484 (pfn >= (1UL<<(32 - PAGE_SHIFT)) && pfn < max_pfn_mapped)) &&
Andreas Herrmanncd7a4e92008-06-10 16:05:39 +0200485 ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) {
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700486 free_memtype(offset, offset + size);
Ingo Molnar28eb559b2008-04-03 10:14:33 +0200487 printk(KERN_INFO
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700488 "%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n",
489 current->comm, current->pid,
490 cattr_name(flags),
Pranith Kumarafc85342008-05-12 14:52:26 +0530491 offset, (unsigned long long)(offset + size));
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700492 return 0;
493 }
494
495 *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
496 flags);
venkatesh.pallipadi@intel.comf0970c12008-03-18 17:00:20 -0700497 return 1;
498}
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700499
500void map_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot)
501{
502 u64 addr = (u64)pfn << PAGE_SHIFT;
503 unsigned long flags;
504 unsigned long want_flags = (pgprot_val(vma_prot) & _PAGE_CACHE_MASK);
505
506 reserve_memtype(addr, addr + size, want_flags, &flags);
507 if (flags != want_flags) {
Ingo Molnar28eb559b2008-04-03 10:14:33 +0200508 printk(KERN_INFO
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700509 "%s:%d /dev/mem expected mapping type %s for %Lx-%Lx, got %s\n",
510 current->comm, current->pid,
511 cattr_name(want_flags),
Pranith Kumarafc85342008-05-12 14:52:26 +0530512 addr, (unsigned long long)(addr + size),
venkatesh.pallipadi@intel.come7f260a2008-03-18 17:00:21 -0700513 cattr_name(flags));
514 }
515}
516
517void unmap_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot)
518{
519 u64 addr = (u64)pfn << PAGE_SHIFT;
520
521 free_memtype(addr, addr + size);
522}
523
venkatesh.pallipadi@intel.comfec09622008-07-18 16:08:14 -0700524#if defined(CONFIG_DEBUG_FS)
525
526/* get Nth element of the linked list */
527static struct memtype *memtype_get_idx(loff_t pos)
528{
529 struct memtype *list_node, *print_entry;
530 int i = 1;
531
532 print_entry = kmalloc(sizeof(struct memtype), GFP_KERNEL);
533 if (!print_entry)
534 return NULL;
535
536 spin_lock(&memtype_lock);
537 list_for_each_entry(list_node, &memtype_list, nd) {
538 if (pos == i) {
539 *print_entry = *list_node;
540 spin_unlock(&memtype_lock);
541 return print_entry;
542 }
543 ++i;
544 }
545 spin_unlock(&memtype_lock);
546 kfree(print_entry);
547 return NULL;
548}
549
550static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
551{
552 if (*pos == 0) {
553 ++*pos;
554 seq_printf(seq, "PAT memtype list:\n");
555 }
556
557 return memtype_get_idx(*pos);
558}
559
560static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
561{
562 ++*pos;
563 return memtype_get_idx(*pos);
564}
565
566static void memtype_seq_stop(struct seq_file *seq, void *v)
567{
568}
569
570static int memtype_seq_show(struct seq_file *seq, void *v)
571{
572 struct memtype *print_entry = (struct memtype *)v;
573
574 seq_printf(seq, "%s @ 0x%Lx-0x%Lx\n", cattr_name(print_entry->type),
575 print_entry->start, print_entry->end);
576 kfree(print_entry);
577 return 0;
578}
579
580static struct seq_operations memtype_seq_ops = {
581 .start = memtype_seq_start,
582 .next = memtype_seq_next,
583 .stop = memtype_seq_stop,
584 .show = memtype_seq_show,
585};
586
587static int memtype_seq_open(struct inode *inode, struct file *file)
588{
589 return seq_open(file, &memtype_seq_ops);
590}
591
592static const struct file_operations memtype_fops = {
593 .open = memtype_seq_open,
594 .read = seq_read,
595 .llseek = seq_lseek,
596 .release = seq_release,
597};
598
599static int __init pat_memtype_list_init(void)
600{
601 debugfs_create_file("pat_memtype_list", S_IRUSR, arch_debugfs_dir,
602 NULL, &memtype_fops);
603 return 0;
604}
605
606late_initcall(pat_memtype_list_init);
607
608#endif /* CONFIG_DEBUG_FS */