blob: 303a7ff6bfd2eeb3120164d9cc185e0973d600de [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/arm/mm/ioremap.c
3 *
4 * Re-map IO memory to kernel address space so that we can access it.
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 *
8 * Hacked for ARM by Phil Blundell <philb@gnu.org>
9 * Hacked to allow all architectures to build, and various cleanups
10 * by Russell King
11 *
12 * This allows a driver to remap an arbitrary region of bus memory into
13 * virtual space. One should *only* use readl, writel, memcpy_toio and
14 * so on with such remapped areas.
15 *
16 * Because the ARM only has a 32-bit address space we can't address the
17 * whole of the (physical) PCI space at once. PCI huge-mode addressing
18 * allows us to circumvent this restriction by splitting PCI space into
19 * two 2GB chunks and mapping only one at a time into processor memory.
20 * We use MMU protection domains to trap any attempt to access the bank
21 * that is not currently mapped. (This isn't fully implemented yet.)
22 */
23#include <linux/module.h>
24#include <linux/errno.h>
25#include <linux/mm.h>
26#include <linux/vmalloc.h>
27
28#include <asm/cacheflush.h>
29#include <asm/io.h>
Russell Kingff0daca2006-06-29 20:17:15 +010030#include <asm/mmu_context.h>
31#include <asm/pgalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <asm/tlbflush.h>
Russell Kingff0daca2006-06-29 20:17:15 +010033#include <asm/sizes.h>
34
Russell Kingb29e9f52007-04-21 10:47:29 +010035#include <asm/mach/map.h>
36#include "mm.h"
37
Russell Kingff0daca2006-06-29 20:17:15 +010038/*
Lennert Buytenheka069c892006-07-01 19:58:20 +010039 * Used by ioremap() and iounmap() code to mark (super)section-mapped
40 * I/O regions in vm_struct->flags field.
Russell Kingff0daca2006-06-29 20:17:15 +010041 */
42#define VM_ARM_SECTION_MAPPING 0x80000000
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
Russell Kingda2c12a2006-12-13 14:35:58 +000044static int remap_area_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
Russell Kingb29e9f52007-04-21 10:47:29 +010045 unsigned long phys_addr, const struct mem_type *type)
Linus Torvalds1da177e2005-04-16 15:20:36 -070046{
Russell Kingb29e9f52007-04-21 10:47:29 +010047 pgprot_t prot = __pgprot(type->prot_pte);
Russell Kingda2c12a2006-12-13 14:35:58 +000048 pte_t *pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
Russell Kingda2c12a2006-12-13 14:35:58 +000050 pte = pte_alloc_kernel(pmd, addr);
51 if (!pte)
52 return -ENOMEM;
53
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 do {
55 if (!pte_none(*pte))
56 goto bad;
57
Russell Kingc172cc92007-04-21 10:52:32 +010058 set_pte_ext(pte, pfn_pte(phys_addr >> PAGE_SHIFT, prot),
59 type->prot_pte_ext);
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 phys_addr += PAGE_SIZE;
Russell Kingda2c12a2006-12-13 14:35:58 +000061 } while (pte++, addr += PAGE_SIZE, addr != end);
62 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
64 bad:
Russell Kingda2c12a2006-12-13 14:35:58 +000065 printk(KERN_CRIT "remap_area_pte: page already exists\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 BUG();
67}
68
Russell Kingda2c12a2006-12-13 14:35:58 +000069static inline int remap_area_pmd(pgd_t *pgd, unsigned long addr,
70 unsigned long end, unsigned long phys_addr,
Russell Kingb29e9f52007-04-21 10:47:29 +010071 const struct mem_type *type)
Linus Torvalds1da177e2005-04-16 15:20:36 -070072{
Russell Kingda2c12a2006-12-13 14:35:58 +000073 unsigned long next;
74 pmd_t *pmd;
75 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
Russell Kingda2c12a2006-12-13 14:35:58 +000077 pmd = pmd_alloc(&init_mm, pgd, addr);
78 if (!pmd)
79 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 do {
Russell Kingda2c12a2006-12-13 14:35:58 +000082 next = pmd_addr_end(addr, end);
Russell Kingb29e9f52007-04-21 10:47:29 +010083 ret = remap_area_pte(pmd, addr, next, phys_addr, type);
Russell Kingda2c12a2006-12-13 14:35:58 +000084 if (ret)
85 return ret;
86 phys_addr += next - addr;
87 } while (pmd++, addr = next, addr != end);
88 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089}
90
Russell Kingda2c12a2006-12-13 14:35:58 +000091static int remap_area_pages(unsigned long start, unsigned long pfn,
Russell Kingb29e9f52007-04-21 10:47:29 +010092 size_t size, const struct mem_type *type)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093{
Russell Kingda2c12a2006-12-13 14:35:58 +000094 unsigned long addr = start;
95 unsigned long next, end = start + size;
Deepak Saxena9d4ae722006-01-09 19:23:11 +000096 unsigned long phys_addr = __pfn_to_phys(pfn);
Russell Kingda2c12a2006-12-13 14:35:58 +000097 pgd_t *pgd;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
Russell Kingda2c12a2006-12-13 14:35:58 +0000100 BUG_ON(addr >= end);
101 pgd = pgd_offset_k(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 do {
Russell Kingda2c12a2006-12-13 14:35:58 +0000103 next = pgd_addr_end(addr, end);
Russell Kingb29e9f52007-04-21 10:47:29 +0100104 err = remap_area_pmd(pgd, addr, next, phys_addr, type);
Russell Kingda2c12a2006-12-13 14:35:58 +0000105 if (err)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 break;
Russell Kingda2c12a2006-12-13 14:35:58 +0000107 phys_addr += next - addr;
108 } while (pgd++, addr = next, addr != end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 return err;
111}
112
Russell Kingff0daca2006-06-29 20:17:15 +0100113
114void __check_kvm_seq(struct mm_struct *mm)
115{
116 unsigned int seq;
117
118 do {
119 seq = init_mm.context.kvm_seq;
120 memcpy(pgd_offset(mm, VMALLOC_START),
121 pgd_offset_k(VMALLOC_START),
122 sizeof(pgd_t) * (pgd_index(VMALLOC_END) -
123 pgd_index(VMALLOC_START)));
124 mm->context.kvm_seq = seq;
125 } while (seq != init_mm.context.kvm_seq);
126}
127
128#ifndef CONFIG_SMP
129/*
130 * Section support is unsafe on SMP - If you iounmap and ioremap a region,
131 * the other CPUs will not see this change until their next context switch.
132 * Meanwhile, (eg) if an interrupt comes in on one of those other CPUs
133 * which requires the new ioremap'd region to be referenced, the CPU will
134 * reference the _old_ region.
135 *
136 * Note that get_vm_area() allocates a guard 4K page, so we need to mask
137 * the size back to 1MB aligned or we will overflow in the loop below.
138 */
139static void unmap_area_sections(unsigned long virt, unsigned long size)
140{
141 unsigned long addr = virt, end = virt + (size & ~SZ_1M);
142 pgd_t *pgd;
143
144 flush_cache_vunmap(addr, end);
145 pgd = pgd_offset_k(addr);
146 do {
147 pmd_t pmd, *pmdp = pmd_offset(pgd, addr);
148
149 pmd = *pmdp;
150 if (!pmd_none(pmd)) {
151 /*
152 * Clear the PMD from the page table, and
153 * increment the kvm sequence so others
154 * notice this change.
155 *
156 * Note: this is still racy on SMP machines.
157 */
158 pmd_clear(pmdp);
159 init_mm.context.kvm_seq++;
160
161 /*
162 * Free the page table, if there was one.
163 */
164 if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE)
Benjamin Herrenschmidt5e541972008-02-04 22:29:14 -0800165 pte_free_kernel(&init_mm, pmd_page_vaddr(pmd));
Russell Kingff0daca2006-06-29 20:17:15 +0100166 }
167
168 addr += PGDIR_SIZE;
169 pgd++;
170 } while (addr < end);
171
172 /*
173 * Ensure that the active_mm is up to date - we want to
174 * catch any use-after-iounmap cases.
175 */
176 if (current->active_mm->context.kvm_seq != init_mm.context.kvm_seq)
177 __check_kvm_seq(current->active_mm);
178
179 flush_tlb_kernel_range(virt, end);
180}
181
182static int
183remap_area_sections(unsigned long virt, unsigned long pfn,
Russell Kingb29e9f52007-04-21 10:47:29 +0100184 size_t size, const struct mem_type *type)
Russell Kingff0daca2006-06-29 20:17:15 +0100185{
Russell Kingb29e9f52007-04-21 10:47:29 +0100186 unsigned long addr = virt, end = virt + size;
Russell Kingff0daca2006-06-29 20:17:15 +0100187 pgd_t *pgd;
188
189 /*
190 * Remove and free any PTE-based mapping, and
191 * sync the current kernel mapping.
192 */
193 unmap_area_sections(virt, size);
194
Russell Kingff0daca2006-06-29 20:17:15 +0100195 pgd = pgd_offset_k(addr);
196 do {
197 pmd_t *pmd = pmd_offset(pgd, addr);
198
Russell Kingb29e9f52007-04-21 10:47:29 +0100199 pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
Russell Kingff0daca2006-06-29 20:17:15 +0100200 pfn += SZ_1M >> PAGE_SHIFT;
Russell Kingb29e9f52007-04-21 10:47:29 +0100201 pmd[1] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
Russell Kingff0daca2006-06-29 20:17:15 +0100202 pfn += SZ_1M >> PAGE_SHIFT;
203 flush_pmd_entry(pmd);
204
205 addr += PGDIR_SIZE;
206 pgd++;
207 } while (addr < end);
208
209 return 0;
210}
Lennert Buytenheka069c892006-07-01 19:58:20 +0100211
212static int
213remap_area_supersections(unsigned long virt, unsigned long pfn,
Russell Kingb29e9f52007-04-21 10:47:29 +0100214 size_t size, const struct mem_type *type)
Lennert Buytenheka069c892006-07-01 19:58:20 +0100215{
Russell Kingb29e9f52007-04-21 10:47:29 +0100216 unsigned long addr = virt, end = virt + size;
Lennert Buytenheka069c892006-07-01 19:58:20 +0100217 pgd_t *pgd;
218
219 /*
220 * Remove and free any PTE-based mapping, and
221 * sync the current kernel mapping.
222 */
223 unmap_area_sections(virt, size);
224
Lennert Buytenheka069c892006-07-01 19:58:20 +0100225 pgd = pgd_offset_k(virt);
226 do {
227 unsigned long super_pmd_val, i;
228
Russell Kingb29e9f52007-04-21 10:47:29 +0100229 super_pmd_val = __pfn_to_phys(pfn) | type->prot_sect |
230 PMD_SECT_SUPER;
Lennert Buytenheka069c892006-07-01 19:58:20 +0100231 super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20;
232
233 for (i = 0; i < 8; i++) {
234 pmd_t *pmd = pmd_offset(pgd, addr);
235
236 pmd[0] = __pmd(super_pmd_val);
237 pmd[1] = __pmd(super_pmd_val);
238 flush_pmd_entry(pmd);
239
240 addr += PGDIR_SIZE;
241 pgd++;
242 }
243
244 pfn += SUPERSECTION_SIZE >> PAGE_SHIFT;
245 } while (addr < end);
246
247 return 0;
248}
Russell Kingff0daca2006-06-29 20:17:15 +0100249#endif
250
251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252/*
253 * Remap an arbitrary physical address space into the kernel virtual
254 * address space. Needed when the kernel wants to access high addresses
255 * directly.
256 *
257 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
258 * have to convert them into an offset in a page-aligned mapping, but the
259 * caller shouldn't need to know that small detail.
260 *
261 * 'flags' are the extra L_PTE_ flags that you want to specify for this
262 * mapping. See include/asm-arm/proc-armv/pgtable.h for more information.
263 */
264void __iomem *
Russell King3603ab22007-05-05 20:59:27 +0100265__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
266 unsigned int mtype)
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000267{
Russell Kingb29e9f52007-04-21 10:47:29 +0100268 const struct mem_type *type;
Russell Kingff0daca2006-06-29 20:17:15 +0100269 int err;
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000270 unsigned long addr;
271 struct vm_struct * area;
Lennert Buytenheka069c892006-07-01 19:58:20 +0100272
273 /*
274 * High mappings must be supersection aligned
275 */
276 if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
277 return NULL;
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000278
Russell King3603ab22007-05-05 20:59:27 +0100279 type = get_mem_type(mtype);
280 if (!type)
281 return NULL;
Russell Kingb29e9f52007-04-21 10:47:29 +0100282
Russell King6d78b5f2007-06-03 19:26:04 +0100283 /*
284 * Page align the mapping size, taking account of any offset.
285 */
286 size = PAGE_ALIGN(offset + size);
Russell Kingc924aff2006-12-17 23:29:57 +0000287
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000288 area = get_vm_area(size, VM_IOREMAP);
289 if (!area)
290 return NULL;
291 addr = (unsigned long)area->addr;
Russell Kingff0daca2006-06-29 20:17:15 +0100292
293#ifndef CONFIG_SMP
Catalin Marinas412489a2007-01-25 14:16:47 +0100294 if (DOMAIN_IO == 0 &&
295 (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
Russell King4a56c1e2007-04-21 10:16:48 +0100296 cpu_is_xsc3()) && pfn >= 0x100000 &&
Lennert Buytenheka069c892006-07-01 19:58:20 +0100297 !((__pfn_to_phys(pfn) | size | addr) & ~SUPERSECTION_MASK)) {
298 area->flags |= VM_ARM_SECTION_MAPPING;
Russell Kingb29e9f52007-04-21 10:47:29 +0100299 err = remap_area_supersections(addr, pfn, size, type);
Lennert Buytenheka069c892006-07-01 19:58:20 +0100300 } else if (!((__pfn_to_phys(pfn) | size | addr) & ~PMD_MASK)) {
Russell Kingff0daca2006-06-29 20:17:15 +0100301 area->flags |= VM_ARM_SECTION_MAPPING;
Russell Kingb29e9f52007-04-21 10:47:29 +0100302 err = remap_area_sections(addr, pfn, size, type);
Russell Kingff0daca2006-06-29 20:17:15 +0100303 } else
304#endif
Russell Kingb29e9f52007-04-21 10:47:29 +0100305 err = remap_area_pages(addr, pfn, size, type);
Russell Kingff0daca2006-06-29 20:17:15 +0100306
307 if (err) {
Catalin Marinas478922c2006-05-16 11:30:26 +0100308 vunmap((void *)addr);
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000309 return NULL;
310 }
Russell Kingff0daca2006-06-29 20:17:15 +0100311
312 flush_cache_vmap(addr, addr + size);
313 return (void __iomem *) (offset + addr);
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000314}
Russell King3603ab22007-05-05 20:59:27 +0100315EXPORT_SYMBOL(__arm_ioremap_pfn);
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000316
317void __iomem *
Russell King3603ab22007-05-05 20:59:27 +0100318__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319{
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000320 unsigned long last_addr;
321 unsigned long offset = phys_addr & ~PAGE_MASK;
322 unsigned long pfn = __phys_to_pfn(phys_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323
Deepak Saxena9d4ae722006-01-09 19:23:11 +0000324 /*
325 * Don't allow wraparound or zero size
326 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 last_addr = phys_addr + size - 1;
328 if (!size || last_addr < phys_addr)
329 return NULL;
330
Russell King3603ab22007-05-05 20:59:27 +0100331 return __arm_ioremap_pfn(pfn, offset, size, mtype);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332}
Russell King3603ab22007-05-05 20:59:27 +0100333EXPORT_SYMBOL(__arm_ioremap);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334
Al Viro16226052006-10-09 02:09:49 +0100335void __iounmap(volatile void __iomem *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336{
Catalin Marinasceaccbd2006-07-29 08:29:30 +0100337#ifndef CONFIG_SMP
Russell Kingff0daca2006-06-29 20:17:15 +0100338 struct vm_struct **p, *tmp;
Catalin Marinasceaccbd2006-07-29 08:29:30 +0100339#endif
Russell Kingff0daca2006-06-29 20:17:15 +0100340 unsigned int section_mapping = 0;
341
Al Viro16226052006-10-09 02:09:49 +0100342 addr = (volatile void __iomem *)(PAGE_MASK & (unsigned long)addr);
Russell Kingff0daca2006-06-29 20:17:15 +0100343
Lennert Buytenhek7cddc392006-07-03 12:26:02 +0100344#ifndef CONFIG_SMP
Russell Kingff0daca2006-06-29 20:17:15 +0100345 /*
346 * If this is a section based mapping we need to handle it
Simon Arlott6cbdc8c2007-05-11 20:40:30 +0100347 * specially as the VM subsystem does not know how to handle
Russell Kingff0daca2006-06-29 20:17:15 +0100348 * such a beast. We need the lock here b/c we need to clear
349 * all the mappings before the area can be reclaimed
350 * by someone else.
351 */
352 write_lock(&vmlist_lock);
353 for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) {
354 if((tmp->flags & VM_IOREMAP) && (tmp->addr == addr)) {
355 if (tmp->flags & VM_ARM_SECTION_MAPPING) {
356 *p = tmp->next;
357 unmap_area_sections((unsigned long)tmp->addr,
358 tmp->size);
359 kfree(tmp);
360 section_mapping = 1;
361 }
362 break;
363 }
364 }
365 write_unlock(&vmlist_lock);
Lennert Buytenhek7cddc392006-07-03 12:26:02 +0100366#endif
Russell Kingff0daca2006-06-29 20:17:15 +0100367
368 if (!section_mapping)
Al Viro16226052006-10-09 02:09:49 +0100369 vunmap((void __force *)addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370}
371EXPORT_SYMBOL(__iounmap);