blob: 0215e2c563ef5bbbaebd0e4f2666023f0afdef1d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 */
8
Thomas Gleixnere9332ca2008-01-30 13:34:05 +01009#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/init.h>
Haavard Skinnemoena148ecf2006-09-30 23:29:17 -070011#include <linux/io.h>
Thomas Gleixner3cbd09e2008-01-30 13:34:05 +010012#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/vmalloc.h>
Pekka Paalanend61fc442008-05-12 21:20:57 +020015#include <linux/mmiotrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
Thomas Gleixner3cbd09e2008-01-30 13:34:05 +010017#include <asm/cacheflush.h>
18#include <asm/e820.h>
19#include <asm/fixmap.h>
20#include <asm/pgtable.h>
21#include <asm/tlbflush.h>
Jeremy Fitzhardingef6df72e2008-01-30 13:34:11 +010022#include <asm/pgalloc.h>
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -070023#include <asm/pat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Jeremy Fitzhardinge78c86e52009-09-10 10:09:38 -070025#include "physaddr.h"
Thomas Gleixner240d3a72008-01-30 13:34:05 +010026
Linus Torvalds1da177e2005-04-16 15:20:36 -070027/*
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010028 * Fix up the linear direct mapping of the kernel to avoid cache attribute
29 * conflicts.
30 */
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -070031int ioremap_change_attr(unsigned long vaddr, unsigned long size,
32 unsigned long prot_val)
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010033{
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010034 unsigned long nrpages = size >> PAGE_SHIFT;
Harvey Harrison93809be2008-02-01 17:49:43 +010035 int err;
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010036
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -070037 switch (prot_val) {
38 case _PAGE_CACHE_UC:
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010039 default:
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -070040 err = _set_memory_uc(vaddr, nrpages);
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010041 break;
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -070042 case _PAGE_CACHE_WC:
43 err = _set_memory_wc(vaddr, nrpages);
44 break;
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -070045 case _PAGE_CACHE_WB:
venkatesh.pallipadi@intel.com12193332008-03-18 17:00:18 -070046 err = _set_memory_wb(vaddr, nrpages);
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010047 break;
48 }
Thomas Gleixnere9332ca2008-01-30 13:34:05 +010049
50 return err;
51}
52
53/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 * Remap an arbitrary physical address space into the kernel virtual
55 * address space. Needed when the kernel wants to access high addresses
56 * directly.
57 *
58 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
59 * have to convert them into an offset in a page-aligned mapping, but the
60 * caller shouldn't need to know that small detail.
61 */
Christoph Lameter23016962008-04-28 02:12:42 -070062static void __iomem *__ioremap_caller(resource_size_t phys_addr,
63 unsigned long size, unsigned long prot_val, void *caller)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064{
Kenji Kaneshigeffa71f32010-06-18 12:22:40 +090065 unsigned long offset, vaddr;
66 resource_size_t pfn, last_pfn, last_addr;
Pekka Paalanen87e547f2008-05-12 21:21:03 +020067 const resource_size_t unaligned_phys_addr = phys_addr;
68 const unsigned long unaligned_size = size;
Thomas Gleixner91eebf42008-01-30 13:34:05 +010069 struct vm_struct *area;
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -070070 unsigned long new_prot_val;
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +010071 pgprot_t prot;
Venki Pallipadidee7cbb2008-03-24 14:39:55 -070072 int retval;
Pekka Paalanend61fc442008-05-12 21:20:57 +020073 void __iomem *ret_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070074
75 /* Don't allow wraparound or zero size */
76 last_addr = phys_addr + size - 1;
77 if (!size || last_addr < phys_addr)
78 return NULL;
79
Thomas Gleixnere3100c82008-02-27 20:57:40 +010080 if (!phys_addr_valid(phys_addr)) {
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -070081 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
Randy Dunlap4c8337a2008-04-10 15:09:50 -070082 (unsigned long long)phys_addr);
Thomas Gleixnere3100c82008-02-27 20:57:40 +010083 WARN_ON_ONCE(1);
84 return NULL;
85 }
86
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 /*
88 * Don't remap the low PCI/ISA area, it's always mapped..
89 */
Andreas Herrmannbcc643d2008-06-20 21:58:46 +020090 if (is_ISA_range(phys_addr, last_addr))
Thomas Gleixner4b40fce2008-01-30 13:34:05 +010091 return (__force void __iomem *)phys_to_virt(phys_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
93 /*
94 * Don't allow anybody to remap normal RAM that we're using..
95 */
Kenji Kaneshigeffa71f32010-06-18 12:22:40 +090096 last_pfn = last_addr >> PAGE_SHIFT;
Kenji Kaneshige35be1b72010-06-18 12:23:57 +090097 for (pfn = phys_addr >> PAGE_SHIFT; pfn <= last_pfn; pfn++) {
Ingo Molnarba748d22008-03-03 09:37:41 +010098 int is_ram = page_is_ram(pfn);
99
100 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
Thomas Gleixner266b9f82008-01-30 13:34:06 +0100101 return NULL;
Ingo Molnarba748d22008-03-03 09:37:41 +0100102 WARN_ON_ONCE(is_ram);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 }
104
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700105 /*
106 * Mappings have to be page-aligned
107 */
108 offset = phys_addr & ~PAGE_MASK;
Kenji Kaneshigeffa71f32010-06-18 12:22:40 +0900109 phys_addr &= PHYSICAL_PAGE_MASK;
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700110 size = PAGE_ALIGN(last_addr+1) - phys_addr;
111
Andi Kleene213e872008-08-15 18:12:47 +0200112 retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
Venki Pallipadidee7cbb2008-03-24 14:39:55 -0700113 prot_val, &new_prot_val);
114 if (retval) {
Venkatesh Pallipadi279e6692009-07-10 09:57:33 -0700115 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
Venki Pallipadidee7cbb2008-03-24 14:39:55 -0700116 return NULL;
117 }
118
119 if (prot_val != new_prot_val) {
H. Peter Anvinb8551922009-08-26 17:17:51 -0700120 if (!is_new_memtype_allowed(phys_addr, size,
121 prot_val, new_prot_val)) {
Venkatesh Pallipadi279e6692009-07-10 09:57:33 -0700122 printk(KERN_ERR
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700123 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
Randy Dunlap4c8337a2008-04-10 15:09:50 -0700124 (unsigned long long)phys_addr,
125 (unsigned long long)(phys_addr + size),
venkatesh.pallipadi@intel.com6997ab42008-03-18 17:00:25 -0700126 prot_val, new_prot_val);
Xiaotian Fengde2a47c2009-11-05 10:43:51 +0800127 goto err_free_memtype;
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700128 }
129 prot_val = new_prot_val;
130 }
131
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700132 switch (prot_val) {
133 case _PAGE_CACHE_UC:
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100134 default:
Jeremy Fitzhardingebe43d722008-09-07 15:21:13 -0700135 prot = PAGE_KERNEL_IO_NOCACHE;
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100136 break;
Suresh Siddhade33c442008-04-25 17:07:22 -0700137 case _PAGE_CACHE_UC_MINUS:
Jeremy Fitzhardingebe43d722008-09-07 15:21:13 -0700138 prot = PAGE_KERNEL_IO_UC_MINUS;
Suresh Siddhade33c442008-04-25 17:07:22 -0700139 break;
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700140 case _PAGE_CACHE_WC:
Jeremy Fitzhardingebe43d722008-09-07 15:21:13 -0700141 prot = PAGE_KERNEL_IO_WC;
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700142 break;
venkatesh.pallipadi@intel.com3a96ce82008-03-18 17:00:16 -0700143 case _PAGE_CACHE_WB:
Jeremy Fitzhardingebe43d722008-09-07 15:21:13 -0700144 prot = PAGE_KERNEL_IO;
Thomas Gleixnerd806e5e2008-01-30 13:34:06 +0100145 break;
146 }
Haavard Skinnemoena148ecf2006-09-30 23:29:17 -0700147
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 * Ok, go for it..
150 */
Christoph Lameter23016962008-04-28 02:12:42 -0700151 area = get_vm_area_caller(size, VM_IOREMAP, caller);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 if (!area)
Xiaotian Fengde2a47c2009-11-05 10:43:51 +0800153 goto err_free_memtype;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 area->phys_addr = phys_addr;
Thomas Gleixnere66aadb2008-02-04 16:48:05 +0100155 vaddr = (unsigned long) area->addr;
Suresh Siddha43a432b2009-04-09 14:26:47 -0700156
Xiaotian Fengde2a47c2009-11-05 10:43:51 +0800157 if (kernel_map_sync_memtype(phys_addr, size, prot_val))
158 goto err_free_area;
Thomas Gleixnere9332ca2008-01-30 13:34:05 +0100159
Xiaotian Fengde2a47c2009-11-05 10:43:51 +0800160 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
161 goto err_free_area;
Thomas Gleixnere9332ca2008-01-30 13:34:05 +0100162
Pekka Paalanend61fc442008-05-12 21:20:57 +0200163 ret_addr = (void __iomem *) (vaddr + offset);
Pekka Paalanen87e547f2008-05-12 21:21:03 +0200164 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
Pekka Paalanend61fc442008-05-12 21:20:57 +0200165
Tim Gardnerc7a7b8142011-04-28 11:00:30 -0600166 /*
167 * Check if the request spans more than any BAR in the iomem resource
168 * tree.
169 */
170 WARN_ONCE(iomem_map_sanity_check(unaligned_phys_addr, unaligned_size),
171 KERN_INFO "Info: mapping multiple BARs. Your kernel is fine.");
172
Pekka Paalanend61fc442008-05-12 21:20:57 +0200173 return ret_addr;
Xiaotian Fengde2a47c2009-11-05 10:43:51 +0800174err_free_area:
175 free_vm_area(area);
176err_free_memtype:
177 free_memtype(phys_addr, phys_addr + size);
178 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
181/**
182 * ioremap_nocache - map bus memory into CPU space
Wanpeng Li9efc31b2012-06-10 10:50:52 +0800183 * @phys_addr: bus address of the memory
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 * @size: size of the resource to map
185 *
186 * ioremap_nocache performs a platform specific sequence of operations to
187 * make bus memory CPU accessible via the readb/readw/readl/writeb/
188 * writew/writel functions and the other mmio helpers. The returned
189 * address is not guaranteed to be usable directly as a virtual
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100190 * address.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191 *
192 * This version of ioremap ensures that the memory is marked uncachable
193 * on the CPU as well as honouring existing caching rules from things like
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100194 * the PCI bus. Note that there are other caches and buffers on many
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 * busses. In particular driver authors should read up on PCI writes
196 *
197 * It's useful if some control registers are in such an area and
198 * write combining or read caching is not desirable:
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100199 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 * Must be freed with iounmap.
201 */
Linus Torvaldsb9e76a02008-03-24 11:22:39 -0700202void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203{
Suresh Siddhade33c442008-04-25 17:07:22 -0700204 /*
205 * Ideally, this should be:
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200206 * pat_enabled ? _PAGE_CACHE_UC : _PAGE_CACHE_UC_MINUS;
Suresh Siddhade33c442008-04-25 17:07:22 -0700207 *
208 * Till we fix all X drivers to use ioremap_wc(), we will use
209 * UC MINUS.
210 */
211 unsigned long val = _PAGE_CACHE_UC_MINUS;
212
213 return __ioremap_caller(phys_addr, size, val,
Christoph Lameter23016962008-04-28 02:12:42 -0700214 __builtin_return_address(0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215}
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700216EXPORT_SYMBOL(ioremap_nocache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700218/**
219 * ioremap_wc - map memory into CPU space write combined
Wanpeng Li9efc31b2012-06-10 10:50:52 +0800220 * @phys_addr: bus address of the memory
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700221 * @size: size of the resource to map
222 *
223 * This version of ioremap ensures that the memory is marked write combining.
224 * Write combining allows faster writes to some hardware devices.
225 *
226 * Must be freed with iounmap.
227 */
venkatesh.pallipadi@intel.comd639bab2009-01-09 16:13:13 -0800228void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700229{
Andreas Herrmann499f8f82008-06-10 16:06:21 +0200230 if (pat_enabled)
Christoph Lameter23016962008-04-28 02:12:42 -0700231 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
232 __builtin_return_address(0));
venkatesh.pallipadi@intel.comb310f381d2008-03-18 17:00:24 -0700233 else
234 return ioremap_nocache(phys_addr, size);
235}
236EXPORT_SYMBOL(ioremap_wc);
237
Linus Torvaldsb9e76a02008-03-24 11:22:39 -0700238void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
Thomas Gleixner5f868152008-01-30 13:34:06 +0100239{
Christoph Lameter23016962008-04-28 02:12:42 -0700240 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
241 __builtin_return_address(0));
Thomas Gleixner5f868152008-01-30 13:34:06 +0100242}
243EXPORT_SYMBOL(ioremap_cache);
244
Rik van Riel28b2ee22008-07-23 21:27:05 -0700245void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
246 unsigned long prot_val)
247{
248 return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK),
249 __builtin_return_address(0));
250}
251EXPORT_SYMBOL(ioremap_prot);
252
Andi Kleenbf5421c2005-12-12 22:17:09 -0800253/**
254 * iounmap - Free a IO remapping
255 * @addr: virtual address from ioremap_*
256 *
257 * Caller must ensure there is only one unmapping for the same pointer.
258 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259void iounmap(volatile void __iomem *addr)
260{
Andi Kleenbf5421c2005-12-12 22:17:09 -0800261 struct vm_struct *p, *o;
Andrew Mortonc23a4e962005-07-07 17:56:02 -0700262
263 if ((void __force *)addr <= high_memory)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 return;
265
266 /*
267 * __ioremap special-cases the PCI/ISA range by not instantiating a
268 * vm_area and by simply returning an address into the kernel mapping
269 * of ISA space. So handle that here.
270 */
Thomas Gleixner6e92a5a2008-05-12 15:43:35 +0200271 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
272 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 return;
274
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100275 addr = (volatile void __iomem *)
276 (PAGE_MASK & (unsigned long __force)addr);
Andi Kleenbf5421c2005-12-12 22:17:09 -0800277
Pekka Paalanend61fc442008-05-12 21:20:57 +0200278 mmiotrace_iounmap(addr);
279
Andi Kleenbf5421c2005-12-12 22:17:09 -0800280 /* Use the vm area unlocked, assuming the caller
281 ensures there isn't another iounmap for the same address
282 in parallel. Reuse of the virtual address is prevented by
283 leaving it in the global lists until we're done with it.
284 cpa takes care of the direct mappings. */
Joonsoo Kimef932472013-04-29 15:07:27 -0700285 p = find_vm_area((void __force *)addr);
Andi Kleenbf5421c2005-12-12 22:17:09 -0800286
287 if (!p) {
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100288 printk(KERN_ERR "iounmap: bad address %p\n", addr);
Andrew Mortonc23a4e962005-07-07 17:56:02 -0700289 dump_stack();
Andi Kleenbf5421c2005-12-12 22:17:09 -0800290 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 }
292
venkatesh.pallipadi@intel.comd7677d42008-03-18 17:00:17 -0700293 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
294
Andi Kleenbf5421c2005-12-12 22:17:09 -0800295 /* Finally remove it */
Thomas Gleixner6e92a5a2008-05-12 15:43:35 +0200296 o = remove_vm_area((void __force *)addr);
Andi Kleenbf5421c2005-12-12 22:17:09 -0800297 BUG_ON(p != o || o == NULL);
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100298 kfree(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299}
Alexey Dobriyan129f6942005-06-23 00:08:33 -0700300EXPORT_SYMBOL(iounmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301
venkatesh.pallipadi@intel.come045fb22008-03-18 17:00:15 -0700302/*
303 * Convert a physical pointer to a virtual kernel pointer for /dev/mem
304 * access
305 */
306void *xlate_dev_mem_ptr(unsigned long phys)
307{
308 void *addr;
309 unsigned long start = phys & PAGE_MASK;
310
311 /* If page is RAM, we can use __va. Otherwise ioremap and unmap. */
312 if (page_is_ram(start >> PAGE_SHIFT))
313 return __va(phys);
314
Xiaotian Feng2fb8f4e2009-11-10 17:23:25 +0800315 addr = (void __force *)ioremap_cache(start, PAGE_SIZE);
venkatesh.pallipadi@intel.come045fb22008-03-18 17:00:15 -0700316 if (addr)
317 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
318
319 return addr;
320}
321
322void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
323{
324 if (page_is_ram(phys >> PAGE_SHIFT))
325 return;
326
327 iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
328 return;
329}
330
Jaswinder Singh4b6e9f22008-07-23 17:39:16 +0530331static int __initdata early_ioremap_debug;
Ingo Molnard18d6d62008-01-30 13:33:45 +0100332
333static int __init early_ioremap_debug_setup(char *str)
334{
335 early_ioremap_debug = 1;
336
Huang, Ying793b24a2008-01-30 13:33:45 +0100337 return 0;
Ingo Molnard18d6d62008-01-30 13:33:45 +0100338}
Huang, Ying793b24a2008-01-30 13:33:45 +0100339early_param("early_ioremap_debug", early_ioremap_debug_setup);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100340
Huang, Ying0947b2f2008-01-30 13:33:44 +0100341static __initdata int after_paging_init;
Jeremy Fitzhardinge45c7b282009-03-20 17:53:34 -0700342static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100343
Ian Campbell551889a2008-02-09 23:24:09 +0100344static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100345{
Jeremy Fitzhardinge37cc8d72008-02-13 16:20:35 +0100346 /* Don't assume we're using swapper_pg_dir at this point */
347 pgd_t *base = __va(read_cr3());
348 pgd_t *pgd = &base[pgd_index(addr)];
Ian Campbell551889a2008-02-09 23:24:09 +0100349 pud_t *pud = pud_offset(pgd, addr);
350 pmd_t *pmd = pmd_offset(pud, addr);
351
352 return pmd;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100353}
354
Ian Campbell551889a2008-02-09 23:24:09 +0100355static inline pte_t * __init early_ioremap_pte(unsigned long addr)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100356{
Ian Campbell551889a2008-02-09 23:24:09 +0100357 return &bm_pte[pte_index(addr)];
Huang, Ying0947b2f2008-01-30 13:33:44 +0100358}
359
Jeremy Fitzhardingefef5ba72010-10-13 16:02:24 -0700360bool __init is_early_ioremap_ptep(pte_t *ptep)
361{
362 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
363}
364
Wang Chen88272472009-03-07 13:34:19 +0800365static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
366
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100367void __init early_ioremap_init(void)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100368{
Ian Campbell551889a2008-02-09 23:24:09 +0100369 pmd_t *pmd;
Wang Chen88272472009-03-07 13:34:19 +0800370 int i;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100371
Ingo Molnard18d6d62008-01-30 13:33:45 +0100372 if (early_ioremap_debug)
Ingo Molnaradafdf62008-01-30 13:34:08 +0100373 printk(KERN_INFO "early_ioremap_init()\n");
Ingo Molnard18d6d62008-01-30 13:33:45 +0100374
Wang Chen88272472009-03-07 13:34:19 +0800375 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
Wang Chen9f4f25c2009-03-25 14:07:11 +0100376 slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
Wang Chen88272472009-03-07 13:34:19 +0800377
Ian Campbell551889a2008-02-09 23:24:09 +0100378 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
Jeremy Fitzhardinge45c7b282009-03-20 17:53:34 -0700379 memset(bm_pte, 0, sizeof(bm_pte));
380 pmd_populate_kernel(&init_mm, pmd, bm_pte);
Ian Campbell551889a2008-02-09 23:24:09 +0100381
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100382 /*
Ian Campbell551889a2008-02-09 23:24:09 +0100383 * The boot-ioremap range spans multiple pmds, for which
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100384 * we are not prepared:
385 */
Jan Beulich499a5f12009-12-18 16:05:51 +0000386#define __FIXADDR_TOP (-PAGE_SIZE)
387 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
388 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
389#undef __FIXADDR_TOP
Ian Campbell551889a2008-02-09 23:24:09 +0100390 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100391 WARN_ON(1);
Ian Campbell551889a2008-02-09 23:24:09 +0100392 printk(KERN_WARNING "pmd %p != %p\n",
393 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100394 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
Ian Campbell551889a2008-02-09 23:24:09 +0100395 fix_to_virt(FIX_BTMAP_BEGIN));
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100396 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
Ian Campbell551889a2008-02-09 23:24:09 +0100397 fix_to_virt(FIX_BTMAP_END));
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100398
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100399 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
400 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
401 FIX_BTMAP_BEGIN);
Ingo Molnar0e3a9542008-01-30 13:33:49 +0100402 }
Huang, Ying0947b2f2008-01-30 13:33:44 +0100403}
404
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100405void __init early_ioremap_reset(void)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100406{
Huang, Ying0947b2f2008-01-30 13:33:44 +0100407 after_paging_init = 1;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100408}
409
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100410static void __init __early_set_fixmap(enum fixed_addresses idx,
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700411 phys_addr_t phys, pgprot_t flags)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100412{
Ian Campbell551889a2008-02-09 23:24:09 +0100413 unsigned long addr = __fix_to_virt(idx);
414 pte_t *pte;
Huang, Ying0947b2f2008-01-30 13:33:44 +0100415
416 if (idx >= __end_of_fixed_addresses) {
417 BUG();
418 return;
419 }
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100420 pte = early_ioremap_pte(addr);
Jeremy Fitzhardinge4583ed52008-06-25 00:19:03 -0400421
Huang, Ying0947b2f2008-01-30 13:33:44 +0100422 if (pgprot_val(flags))
Ian Campbell551889a2008-02-09 23:24:09 +0100423 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
Huang, Ying0947b2f2008-01-30 13:33:44 +0100424 else
Jeremy Fitzhardinge4f9c11d2008-06-25 00:19:19 -0400425 pte_clear(&init_mm, addr, pte);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100426 __flush_tlb_one(addr);
427}
428
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100429static inline void __init early_set_fixmap(enum fixed_addresses idx,
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700430 phys_addr_t phys, pgprot_t prot)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100431{
432 if (after_paging_init)
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700433 __set_fixmap(idx, phys, prot);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100434 else
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700435 __early_set_fixmap(idx, phys, prot);
Huang, Ying0947b2f2008-01-30 13:33:44 +0100436}
437
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100438static inline void __init early_clear_fixmap(enum fixed_addresses idx)
Huang, Ying0947b2f2008-01-30 13:33:44 +0100439{
440 if (after_paging_init)
441 clear_fixmap(idx);
442 else
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100443 __early_set_fixmap(idx, 0, __pgprot(0));
Huang, Ying0947b2f2008-01-30 13:33:44 +0100444}
445
Harvey Harrison1d6cf1f2008-10-28 22:46:04 -0700446static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700447static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
Wang Chen88272472009-03-07 13:34:19 +0800448
Liang Lie67a8072010-04-30 18:01:51 +0800449void __init fixup_early_ioremap(void)
450{
451 int i;
452
453 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
454 if (prev_map[i]) {
455 WARN_ON(1);
456 break;
457 }
458 }
459
460 early_ioremap_init();
461}
462
Ingo Molnard690b2a2008-01-30 13:33:47 +0100463static int __init check_early_ioremap_leak(void)
464{
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700465 int count = 0;
466 int i;
467
468 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
469 if (prev_map[i])
470 count++;
471
472 if (!count)
Ingo Molnard690b2a2008-01-30 13:33:47 +0100473 return 0;
Arjan van de Ven0c072bb2008-07-08 09:50:22 -0700474 WARN(1, KERN_WARNING
Thomas Gleixner91eebf42008-01-30 13:34:05 +0100475 "Debug warning: early ioremap leak of %d areas detected.\n",
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700476 count);
Ingo Molnard690b2a2008-01-30 13:33:47 +0100477 printk(KERN_WARNING
Arjan van de Ven0c072bb2008-07-08 09:50:22 -0700478 "please boot with early_ioremap_debug and report the dmesg.\n");
Ingo Molnard690b2a2008-01-30 13:33:47 +0100479
480 return 1;
481}
482late_initcall(check_early_ioremap_leak);
483
Wang Chen88272472009-03-07 13:34:19 +0800484static void __init __iomem *
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700485__early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700487 unsigned long offset;
488 resource_size_t last_addr;
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700489 unsigned int nrpages;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100490 enum fixed_addresses idx0, idx;
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700491 int i, slot;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100492
493 WARN_ON(system_state != SYSTEM_BOOTING);
494
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700495 slot = -1;
496 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
497 if (!prev_map[i]) {
498 slot = i;
499 break;
500 }
501 }
502
503 if (slot < 0) {
Borislav Petkov4f4319a2013-06-27 23:53:16 +0200504 printk(KERN_INFO "%s(%08llx, %08lx) not found slot\n",
505 __func__, (u64)phys_addr, size);
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700506 WARN_ON(1);
507 return NULL;
508 }
509
Ingo Molnard18d6d62008-01-30 13:33:45 +0100510 if (early_ioremap_debug) {
Borislav Petkov4f4319a2013-06-27 23:53:16 +0200511 printk(KERN_INFO "%s(%08llx, %08lx) [%d] => ",
512 __func__, (u64)phys_addr, size, slot);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100513 dump_stack();
514 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515
516 /* Don't allow wraparound or zero size */
517 last_addr = phys_addr + size - 1;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100518 if (!size || last_addr < phys_addr) {
519 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 return NULL;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100521 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700523 prev_size[slot] = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 /*
525 * Mappings have to be page-aligned
526 */
527 offset = phys_addr & ~PAGE_MASK;
528 phys_addr &= PAGE_MASK;
Alan Coxc613ec12008-10-10 10:46:45 +0100529 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531 /*
532 * Mappings have to fit in the FIX_BTMAP area.
533 */
534 nrpages = size >> PAGE_SHIFT;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100535 if (nrpages > NR_FIX_BTMAPS) {
536 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 return NULL;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100538 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
540 /*
541 * Ok, go for it..
542 */
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700543 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100544 idx = idx0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 while (nrpages > 0) {
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700546 early_set_fixmap(idx, phys_addr, prot);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 phys_addr += PAGE_SIZE;
548 --idx;
549 --nrpages;
550 }
Ingo Molnard18d6d62008-01-30 13:33:45 +0100551 if (early_ioremap_debug)
Wang Chen88272472009-03-07 13:34:19 +0800552 printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]);
Ingo Molnar1b42f512008-01-30 13:33:45 +0100553
Wang Chen88272472009-03-07 13:34:19 +0800554 prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700555 return prev_map[slot];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556}
557
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700558/* Remap an IO device */
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700559void __init __iomem *
560early_ioremap(resource_size_t phys_addr, unsigned long size)
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700561{
562 return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO);
563}
564
565/* Remap memory */
Masami Hiramatsu9b987ae2009-04-09 10:55:33 -0700566void __init __iomem *
567early_memremap(resource_size_t phys_addr, unsigned long size)
Jeremy Fitzhardinge14941772008-09-07 15:21:15 -0700568{
569 return __early_ioremap(phys_addr, size, PAGE_KERNEL);
570}
571
Harvey Harrison1d6cf1f2008-10-28 22:46:04 -0700572void __init early_iounmap(void __iomem *addr, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573{
574 unsigned long virt_addr;
575 unsigned long offset;
576 unsigned int nrpages;
577 enum fixed_addresses idx;
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700578 int i, slot;
Ingo Molnar1b42f512008-01-30 13:33:45 +0100579
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700580 slot = -1;
581 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
582 if (prev_map[i] == addr) {
583 slot = i;
584 break;
585 }
586 }
587
588 if (slot < 0) {
589 printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n",
590 addr, size);
591 WARN_ON(1);
Ingo Molnar226e9a92008-05-27 09:56:49 +0200592 return;
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700593 }
594
595 if (prev_size[slot] != size) {
596 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
597 addr, size, slot, prev_size[slot]);
598 WARN_ON(1);
599 return;
600 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601
Ingo Molnard18d6d62008-01-30 13:33:45 +0100602 if (early_ioremap_debug) {
Ingo Molnaradafdf62008-01-30 13:34:08 +0100603 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700604 size, slot);
Ingo Molnard18d6d62008-01-30 13:33:45 +0100605 dump_stack();
606 }
607
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 virt_addr = (unsigned long)addr;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100609 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
610 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 return;
Ingo Molnarbd796ed2008-01-30 13:33:45 +0100612 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 offset = virt_addr & ~PAGE_MASK;
Florian Zumbiehl468c30f2010-07-20 15:19:47 -0700614 nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
Yinghai Luc1a2f4b2008-09-14 02:33:12 -0700616 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 while (nrpages > 0) {
Huang, Yingbeacfaa2008-01-30 13:33:44 +0100618 early_clear_fixmap(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 --idx;
620 --nrpages;
621 }
Harvey Harrison1d6cf1f2008-10-28 22:46:04 -0700622 prev_map[slot] = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623}