Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/arm/mm/mm-armv.c |
| 3 | * |
Russell King | 9007205 | 2005-10-28 14:48:37 +0100 | [diff] [blame] | 4 | * Copyright (C) 1998-2005 Russell King |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | * |
| 10 | * Page table sludge for ARM v3 and v4 processor architectures. |
| 11 | */ |
| 12 | #include <linux/config.h> |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/mm.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/bootmem.h> |
| 17 | #include <linux/highmem.h> |
| 18 | #include <linux/nodemask.h> |
| 19 | |
| 20 | #include <asm/pgalloc.h> |
| 21 | #include <asm/page.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | #include <asm/setup.h> |
| 23 | #include <asm/tlbflush.h> |
| 24 | |
| 25 | #include <asm/mach/map.h> |
| 26 | |
| 27 | #define CPOLICY_UNCACHED 0 |
| 28 | #define CPOLICY_BUFFERED 1 |
| 29 | #define CPOLICY_WRITETHROUGH 2 |
| 30 | #define CPOLICY_WRITEBACK 3 |
| 31 | #define CPOLICY_WRITEALLOC 4 |
| 32 | |
| 33 | static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; |
| 34 | static unsigned int ecc_mask __initdata = 0; |
| 35 | pgprot_t pgprot_kernel; |
| 36 | |
| 37 | EXPORT_SYMBOL(pgprot_kernel); |
| 38 | |
Russell King | c4e1f6f | 2005-05-10 10:40:19 +0100 | [diff] [blame] | 39 | pmd_t *top_pmd; |
| 40 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | struct cachepolicy { |
| 42 | const char policy[16]; |
| 43 | unsigned int cr_mask; |
| 44 | unsigned int pmd; |
| 45 | unsigned int pte; |
| 46 | }; |
| 47 | |
| 48 | static struct cachepolicy cache_policies[] __initdata = { |
| 49 | { |
| 50 | .policy = "uncached", |
| 51 | .cr_mask = CR_W|CR_C, |
| 52 | .pmd = PMD_SECT_UNCACHED, |
| 53 | .pte = 0, |
| 54 | }, { |
| 55 | .policy = "buffered", |
| 56 | .cr_mask = CR_C, |
| 57 | .pmd = PMD_SECT_BUFFERED, |
| 58 | .pte = PTE_BUFFERABLE, |
| 59 | }, { |
| 60 | .policy = "writethrough", |
| 61 | .cr_mask = 0, |
| 62 | .pmd = PMD_SECT_WT, |
| 63 | .pte = PTE_CACHEABLE, |
| 64 | }, { |
| 65 | .policy = "writeback", |
| 66 | .cr_mask = 0, |
| 67 | .pmd = PMD_SECT_WB, |
| 68 | .pte = PTE_BUFFERABLE|PTE_CACHEABLE, |
| 69 | }, { |
| 70 | .policy = "writealloc", |
| 71 | .cr_mask = 0, |
| 72 | .pmd = PMD_SECT_WBWA, |
| 73 | .pte = PTE_BUFFERABLE|PTE_CACHEABLE, |
| 74 | } |
| 75 | }; |
| 76 | |
| 77 | /* |
| 78 | * These are useful for identifing cache coherency |
| 79 | * problems by allowing the cache or the cache and |
| 80 | * writebuffer to be turned off. (Note: the write |
| 81 | * buffer should not be on and the cache off). |
| 82 | */ |
| 83 | static void __init early_cachepolicy(char **p) |
| 84 | { |
| 85 | int i; |
| 86 | |
| 87 | for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { |
| 88 | int len = strlen(cache_policies[i].policy); |
| 89 | |
| 90 | if (memcmp(*p, cache_policies[i].policy, len) == 0) { |
| 91 | cachepolicy = i; |
| 92 | cr_alignment &= ~cache_policies[i].cr_mask; |
| 93 | cr_no_alignment &= ~cache_policies[i].cr_mask; |
| 94 | *p += len; |
| 95 | break; |
| 96 | } |
| 97 | } |
| 98 | if (i == ARRAY_SIZE(cache_policies)) |
| 99 | printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n"); |
| 100 | flush_cache_all(); |
| 101 | set_cr(cr_alignment); |
| 102 | } |
| 103 | |
| 104 | static void __init early_nocache(char **__unused) |
| 105 | { |
| 106 | char *p = "buffered"; |
| 107 | printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p); |
| 108 | early_cachepolicy(&p); |
| 109 | } |
| 110 | |
| 111 | static void __init early_nowrite(char **__unused) |
| 112 | { |
| 113 | char *p = "uncached"; |
| 114 | printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p); |
| 115 | early_cachepolicy(&p); |
| 116 | } |
| 117 | |
| 118 | static void __init early_ecc(char **p) |
| 119 | { |
| 120 | if (memcmp(*p, "on", 2) == 0) { |
| 121 | ecc_mask = PMD_PROTECTION; |
| 122 | *p += 2; |
| 123 | } else if (memcmp(*p, "off", 3) == 0) { |
| 124 | ecc_mask = 0; |
| 125 | *p += 3; |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | __early_param("nocache", early_nocache); |
| 130 | __early_param("nowb", early_nowrite); |
| 131 | __early_param("cachepolicy=", early_cachepolicy); |
| 132 | __early_param("ecc=", early_ecc); |
| 133 | |
| 134 | static int __init noalign_setup(char *__unused) |
| 135 | { |
| 136 | cr_alignment &= ~CR_A; |
| 137 | cr_no_alignment &= ~CR_A; |
| 138 | set_cr(cr_alignment); |
| 139 | return 1; |
| 140 | } |
| 141 | |
| 142 | __setup("noalign", noalign_setup); |
| 143 | |
| 144 | #define FIRST_KERNEL_PGD_NR (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD) |
| 145 | |
Russell King | 155bb14 | 2005-05-09 20:52:51 +0100 | [diff] [blame] | 146 | static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) |
| 147 | { |
| 148 | return pmd_offset(pgd, virt); |
| 149 | } |
| 150 | |
| 151 | static inline pmd_t *pmd_off_k(unsigned long virt) |
| 152 | { |
| 153 | return pmd_off(pgd_offset_k(virt), virt); |
| 154 | } |
| 155 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | /* |
| 157 | * need to get a 16k page for level 1 |
| 158 | */ |
| 159 | pgd_t *get_pgd_slow(struct mm_struct *mm) |
| 160 | { |
| 161 | pgd_t *new_pgd, *init_pgd; |
| 162 | pmd_t *new_pmd, *init_pmd; |
| 163 | pte_t *new_pte, *init_pte; |
| 164 | |
| 165 | new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2); |
| 166 | if (!new_pgd) |
| 167 | goto no_pgd; |
| 168 | |
| 169 | memzero(new_pgd, FIRST_KERNEL_PGD_NR * sizeof(pgd_t)); |
| 170 | |
Russell King | a343e60 | 2005-06-27 14:08:56 +0100 | [diff] [blame] | 171 | /* |
| 172 | * Copy over the kernel and IO PGD entries |
| 173 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | init_pgd = pgd_offset_k(0); |
Russell King | a343e60 | 2005-06-27 14:08:56 +0100 | [diff] [blame] | 175 | memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, |
| 176 | (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); |
| 177 | |
| 178 | clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | |
| 180 | if (!vectors_high()) { |
| 181 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | * On ARM, first page must always be allocated since it |
| 183 | * contains the machine vectors. |
| 184 | */ |
| 185 | new_pmd = pmd_alloc(mm, new_pgd, 0); |
| 186 | if (!new_pmd) |
| 187 | goto no_pmd; |
| 188 | |
| 189 | new_pte = pte_alloc_map(mm, new_pmd, 0); |
| 190 | if (!new_pte) |
| 191 | goto no_pte; |
| 192 | |
| 193 | init_pmd = pmd_offset(init_pgd, 0); |
| 194 | init_pte = pte_offset_map_nested(init_pmd, 0); |
| 195 | set_pte(new_pte, *init_pte); |
| 196 | pte_unmap_nested(init_pte); |
| 197 | pte_unmap(new_pte); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | } |
| 199 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | return new_pgd; |
| 201 | |
| 202 | no_pte: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | pmd_free(new_pmd); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | no_pmd: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | free_pages((unsigned long)new_pgd, 2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | no_pgd: |
| 207 | return NULL; |
| 208 | } |
| 209 | |
| 210 | void free_pgd_slow(pgd_t *pgd) |
| 211 | { |
| 212 | pmd_t *pmd; |
| 213 | struct page *pte; |
| 214 | |
| 215 | if (!pgd) |
| 216 | return; |
| 217 | |
| 218 | /* pgd is always present and good */ |
Russell King | 155bb14 | 2005-05-09 20:52:51 +0100 | [diff] [blame] | 219 | pmd = pmd_off(pgd, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | if (pmd_none(*pmd)) |
| 221 | goto free; |
| 222 | if (pmd_bad(*pmd)) { |
| 223 | pmd_ERROR(*pmd); |
| 224 | pmd_clear(pmd); |
| 225 | goto free; |
| 226 | } |
| 227 | |
| 228 | pte = pmd_page(*pmd); |
| 229 | pmd_clear(pmd); |
| 230 | dec_page_state(nr_page_table_pages); |
Hugh Dickins | 4c21e2f | 2005-10-29 18:16:40 -0700 | [diff] [blame] | 231 | pte_lock_deinit(pte); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 232 | pte_free(pte); |
| 233 | pmd_free(pmd); |
| 234 | free: |
| 235 | free_pages((unsigned long) pgd, 2); |
| 236 | } |
| 237 | |
| 238 | /* |
| 239 | * Create a SECTION PGD between VIRT and PHYS in domain |
| 240 | * DOMAIN with protection PROT. This operates on half- |
| 241 | * pgdir entry increments. |
| 242 | */ |
| 243 | static inline void |
| 244 | alloc_init_section(unsigned long virt, unsigned long phys, int prot) |
| 245 | { |
Russell King | 155bb14 | 2005-05-09 20:52:51 +0100 | [diff] [blame] | 246 | pmd_t *pmdp = pmd_off_k(virt); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | if (virt & (1 << 20)) |
| 249 | pmdp++; |
| 250 | |
| 251 | *pmdp = __pmd(phys | prot); |
| 252 | flush_pmd_entry(pmdp); |
| 253 | } |
| 254 | |
| 255 | /* |
| 256 | * Create a SUPER SECTION PGD between VIRT and PHYS with protection PROT |
| 257 | */ |
| 258 | static inline void |
| 259 | alloc_init_supersection(unsigned long virt, unsigned long phys, int prot) |
| 260 | { |
| 261 | int i; |
| 262 | |
| 263 | for (i = 0; i < 16; i += 1) { |
Deepak Saxena | 083bc6b | 2005-08-29 22:54:53 +0100 | [diff] [blame] | 264 | alloc_init_section(virt, phys, prot | PMD_SECT_SUPER); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 265 | |
| 266 | virt += (PGDIR_SIZE / 2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 267 | } |
| 268 | } |
| 269 | |
| 270 | /* |
| 271 | * Add a PAGE mapping between VIRT and PHYS in domain |
| 272 | * DOMAIN with protection PROT. Note that due to the |
| 273 | * way we map the PTEs, we must allocate two PTE_SIZE'd |
| 274 | * blocks - one for the Linux pte table, and one for |
| 275 | * the hardware pte table. |
| 276 | */ |
| 277 | static inline void |
| 278 | alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pgprot_t prot) |
| 279 | { |
Russell King | 155bb14 | 2005-05-09 20:52:51 +0100 | [diff] [blame] | 280 | pmd_t *pmdp = pmd_off_k(virt); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 281 | pte_t *ptep; |
| 282 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 283 | if (pmd_none(*pmdp)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 284 | ptep = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * |
| 285 | sizeof(pte_t)); |
| 286 | |
Russell King | 08f4ffb | 2005-09-01 14:45:18 +0100 | [diff] [blame] | 287 | __pmd_populate(pmdp, __pa(ptep) | prot_l1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | } |
| 289 | ptep = pte_offset_kernel(pmdp, virt); |
| 290 | |
| 291 | set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); |
| 292 | } |
| 293 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | struct mem_types { |
| 295 | unsigned int prot_pte; |
| 296 | unsigned int prot_l1; |
| 297 | unsigned int prot_sect; |
| 298 | unsigned int domain; |
| 299 | }; |
| 300 | |
| 301 | static struct mem_types mem_types[] __initdata = { |
| 302 | [MT_DEVICE] = { |
| 303 | .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | |
| 304 | L_PTE_WRITE, |
| 305 | .prot_l1 = PMD_TYPE_TABLE, |
| 306 | .prot_sect = PMD_TYPE_SECT | PMD_SECT_UNCACHED | |
| 307 | PMD_SECT_AP_WRITE, |
| 308 | .domain = DOMAIN_IO, |
| 309 | }, |
| 310 | [MT_CACHECLEAN] = { |
| 311 | .prot_sect = PMD_TYPE_SECT, |
| 312 | .domain = DOMAIN_KERNEL, |
| 313 | }, |
| 314 | [MT_MINICLEAN] = { |
| 315 | .prot_sect = PMD_TYPE_SECT | PMD_SECT_MINICACHE, |
| 316 | .domain = DOMAIN_KERNEL, |
| 317 | }, |
| 318 | [MT_LOW_VECTORS] = { |
| 319 | .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | |
| 320 | L_PTE_EXEC, |
| 321 | .prot_l1 = PMD_TYPE_TABLE, |
| 322 | .domain = DOMAIN_USER, |
| 323 | }, |
| 324 | [MT_HIGH_VECTORS] = { |
| 325 | .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | |
| 326 | L_PTE_USER | L_PTE_EXEC, |
| 327 | .prot_l1 = PMD_TYPE_TABLE, |
| 328 | .domain = DOMAIN_USER, |
| 329 | }, |
| 330 | [MT_MEMORY] = { |
| 331 | .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, |
| 332 | .domain = DOMAIN_KERNEL, |
| 333 | }, |
| 334 | [MT_ROM] = { |
| 335 | .prot_sect = PMD_TYPE_SECT, |
| 336 | .domain = DOMAIN_KERNEL, |
| 337 | }, |
| 338 | [MT_IXP2000_DEVICE] = { /* IXP2400 requires XCB=101 for on-chip I/O */ |
| 339 | .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | |
| 340 | L_PTE_WRITE, |
| 341 | .prot_l1 = PMD_TYPE_TABLE, |
| 342 | .prot_sect = PMD_TYPE_SECT | PMD_SECT_UNCACHED | |
| 343 | PMD_SECT_AP_WRITE | PMD_SECT_BUFFERABLE | |
| 344 | PMD_SECT_TEX(1), |
| 345 | .domain = DOMAIN_IO, |
George G. Davis | 7efb830 | 2006-01-26 15:21:28 +0000 | [diff] [blame] | 346 | }, |
| 347 | [MT_NONSHARED_DEVICE] = { |
| 348 | .prot_l1 = PMD_TYPE_TABLE, |
| 349 | .prot_sect = PMD_TYPE_SECT | PMD_SECT_NONSHARED_DEV | |
| 350 | PMD_SECT_AP_WRITE, |
| 351 | .domain = DOMAIN_IO, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | } |
| 353 | }; |
| 354 | |
| 355 | /* |
| 356 | * Adjust the PMD section entries according to the CPU in use. |
| 357 | */ |
Russell King | 9007205 | 2005-10-28 14:48:37 +0100 | [diff] [blame] | 358 | void __init build_mem_type_table(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 359 | { |
| 360 | struct cachepolicy *cp; |
| 361 | unsigned int cr = get_cr(); |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 362 | unsigned int user_pgprot, kern_pgprot; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | int cpu_arch = cpu_architecture(); |
| 364 | int i; |
| 365 | |
| 366 | #if defined(CONFIG_CPU_DCACHE_DISABLE) |
| 367 | if (cachepolicy > CPOLICY_BUFFERED) |
| 368 | cachepolicy = CPOLICY_BUFFERED; |
| 369 | #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH) |
| 370 | if (cachepolicy > CPOLICY_WRITETHROUGH) |
| 371 | cachepolicy = CPOLICY_WRITETHROUGH; |
| 372 | #endif |
| 373 | if (cpu_arch < CPU_ARCH_ARMv5) { |
| 374 | if (cachepolicy >= CPOLICY_WRITEALLOC) |
| 375 | cachepolicy = CPOLICY_WRITEBACK; |
| 376 | ecc_mask = 0; |
| 377 | } |
| 378 | |
Deepak Saxena | 8107338 | 2005-07-10 19:44:55 +0100 | [diff] [blame] | 379 | if (cpu_arch <= CPU_ARCH_ARMv5TEJ) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 380 | for (i = 0; i < ARRAY_SIZE(mem_types); i++) { |
| 381 | if (mem_types[i].prot_l1) |
| 382 | mem_types[i].prot_l1 |= PMD_BIT4; |
| 383 | if (mem_types[i].prot_sect) |
| 384 | mem_types[i].prot_sect |= PMD_BIT4; |
| 385 | } |
| 386 | } |
| 387 | |
Russell King | 6626a70 | 2005-08-10 16:18:35 +0100 | [diff] [blame] | 388 | cp = &cache_policies[cachepolicy]; |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 389 | kern_pgprot = user_pgprot = cp->pte; |
Russell King | 6626a70 | 2005-08-10 16:18:35 +0100 | [diff] [blame] | 390 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | /* |
| 392 | * ARMv6 and above have extended page tables. |
| 393 | */ |
| 394 | if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { |
| 395 | /* |
| 396 | * bit 4 becomes XN which we must clear for the |
| 397 | * kernel memory mapping. |
| 398 | */ |
| 399 | mem_types[MT_MEMORY].prot_sect &= ~PMD_BIT4; |
| 400 | mem_types[MT_ROM].prot_sect &= ~PMD_BIT4; |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 401 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 402 | /* |
George G. Davis | ca31515 | 2005-04-29 22:08:35 +0100 | [diff] [blame] | 403 | * Mark cache clean areas and XIP ROM read only |
| 404 | * from SVC mode and no access from userspace. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 405 | */ |
George G. Davis | ca31515 | 2005-04-29 22:08:35 +0100 | [diff] [blame] | 406 | mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; |
| 408 | mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; |
Russell King | 186efd5 | 2005-07-26 19:51:26 +0100 | [diff] [blame] | 409 | |
Russell King | 6626a70 | 2005-08-10 16:18:35 +0100 | [diff] [blame] | 410 | /* |
| 411 | * Mark the device area as "shared device" |
| 412 | */ |
Russell King | 186efd5 | 2005-07-26 19:51:26 +0100 | [diff] [blame] | 413 | mem_types[MT_DEVICE].prot_pte |= L_PTE_BUFFERABLE; |
| 414 | mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 415 | |
Russell King | 6626a70 | 2005-08-10 16:18:35 +0100 | [diff] [blame] | 416 | /* |
| 417 | * User pages need to be mapped with the ASID |
| 418 | * (iow, non-global) |
| 419 | */ |
| 420 | user_pgprot |= L_PTE_ASID; |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 421 | |
| 422 | #ifdef CONFIG_SMP |
| 423 | /* |
| 424 | * Mark memory with the "shared" attribute for SMP systems |
| 425 | */ |
| 426 | user_pgprot |= L_PTE_SHARED; |
| 427 | kern_pgprot |= L_PTE_SHARED; |
| 428 | mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; |
| 429 | #endif |
Russell King | 6626a70 | 2005-08-10 16:18:35 +0100 | [diff] [blame] | 430 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 431 | |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 432 | for (i = 0; i < 16; i++) { |
| 433 | unsigned long v = pgprot_val(protection_map[i]); |
| 434 | v = (v & ~(L_PTE_BUFFERABLE|L_PTE_CACHEABLE)) | user_pgprot; |
| 435 | protection_map[i] = __pgprot(v); |
| 436 | } |
| 437 | |
| 438 | mem_types[MT_LOW_VECTORS].prot_pte |= kern_pgprot; |
| 439 | mem_types[MT_HIGH_VECTORS].prot_pte |= kern_pgprot; |
| 440 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 441 | if (cpu_arch >= CPU_ARCH_ARMv5) { |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 442 | #ifndef CONFIG_SMP |
| 443 | /* |
| 444 | * Only use write-through for non-SMP systems |
| 445 | */ |
| 446 | mem_types[MT_LOW_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE; |
| 447 | mem_types[MT_HIGH_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE; |
| 448 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | } else { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 450 | mem_types[MT_MINICLEAN].prot_sect &= ~PMD_SECT_TEX(1); |
| 451 | } |
| 452 | |
Russell King | cd03adb | 2005-11-07 10:10:28 +0000 | [diff] [blame] | 453 | pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | |
| 454 | L_PTE_DIRTY | L_PTE_WRITE | |
| 455 | L_PTE_EXEC | kern_pgprot); |
| 456 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 457 | mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; |
| 458 | mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; |
| 459 | mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; |
| 460 | mem_types[MT_ROM].prot_sect |= cp->pmd; |
| 461 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 462 | switch (cp->pmd) { |
| 463 | case PMD_SECT_WT: |
| 464 | mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT; |
| 465 | break; |
| 466 | case PMD_SECT_WB: |
| 467 | case PMD_SECT_WBWA: |
| 468 | mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB; |
| 469 | break; |
| 470 | } |
| 471 | printk("Memory policy: ECC %sabled, Data cache %s\n", |
| 472 | ecc_mask ? "en" : "dis", cp->policy); |
| 473 | } |
| 474 | |
| 475 | #define vectors_base() (vectors_high() ? 0xffff0000 : 0) |
| 476 | |
| 477 | /* |
| 478 | * Create the page directory entries and any necessary |
| 479 | * page tables for the mapping specified by `md'. We |
| 480 | * are able to cope here with varying sizes and address |
| 481 | * offsets, and we take full advantage of sections and |
| 482 | * supersections. |
| 483 | */ |
Russell King | 9007205 | 2005-10-28 14:48:37 +0100 | [diff] [blame] | 484 | void __init create_mapping(struct map_desc *md) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 485 | { |
| 486 | unsigned long virt, length; |
| 487 | int prot_sect, prot_l1, domain; |
| 488 | pgprot_t prot_pte; |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 489 | unsigned long off = (u32)__pfn_to_phys(md->pfn); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | |
| 491 | if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) { |
| 492 | printk(KERN_WARNING "BUG: not creating mapping for " |
Nicolas Pitre | 24bcc2f | 2005-11-03 20:40:50 +0000 | [diff] [blame] | 493 | "0x%08llx at 0x%08lx in user region\n", |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 494 | __pfn_to_phys((u64)md->pfn), md->virtual); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 495 | return; |
| 496 | } |
| 497 | |
| 498 | if ((md->type == MT_DEVICE || md->type == MT_ROM) && |
| 499 | md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) { |
Nicolas Pitre | 24bcc2f | 2005-11-03 20:40:50 +0000 | [diff] [blame] | 500 | printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx " |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 501 | "overlaps vmalloc space\n", |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 502 | __pfn_to_phys((u64)md->pfn), md->virtual); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 503 | } |
| 504 | |
| 505 | domain = mem_types[md->type].domain; |
| 506 | prot_pte = __pgprot(mem_types[md->type].prot_pte); |
| 507 | prot_l1 = mem_types[md->type].prot_l1 | PMD_DOMAIN(domain); |
| 508 | prot_sect = mem_types[md->type].prot_sect | PMD_DOMAIN(domain); |
| 509 | |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 510 | /* |
| 511 | * Catch 36-bit addresses |
| 512 | */ |
| 513 | if(md->pfn >= 0x100000) { |
| 514 | if(domain) { |
| 515 | printk(KERN_ERR "MM: invalid domain in supersection " |
Nicolas Pitre | 24bcc2f | 2005-11-03 20:40:50 +0000 | [diff] [blame] | 516 | "mapping for 0x%08llx at 0x%08lx\n", |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 517 | __pfn_to_phys((u64)md->pfn), md->virtual); |
| 518 | return; |
| 519 | } |
| 520 | if((md->virtual | md->length | __pfn_to_phys(md->pfn)) |
| 521 | & ~SUPERSECTION_MASK) { |
| 522 | printk(KERN_ERR "MM: cannot create mapping for " |
Nicolas Pitre | 24bcc2f | 2005-11-03 20:40:50 +0000 | [diff] [blame] | 523 | "0x%08llx at 0x%08lx invalid alignment\n", |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 524 | __pfn_to_phys((u64)md->pfn), md->virtual); |
| 525 | return; |
| 526 | } |
| 527 | |
| 528 | /* |
| 529 | * Shift bits [35:32] of address into bits [23:20] of PMD |
| 530 | * (See ARMv6 spec). |
| 531 | */ |
| 532 | off |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20); |
| 533 | } |
| 534 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 535 | virt = md->virtual; |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 536 | off -= virt; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | length = md->length; |
| 538 | |
| 539 | if (mem_types[md->type].prot_l1 == 0 && |
| 540 | (virt & 0xfffff || (virt + off) & 0xfffff || (virt + length) & 0xfffff)) { |
| 541 | printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not " |
| 542 | "be mapped using pages, ignoring.\n", |
Deepak Saxena | 9769c24 | 2005-10-28 15:19:11 +0100 | [diff] [blame] | 543 | __pfn_to_phys(md->pfn), md->virtual); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 544 | return; |
| 545 | } |
| 546 | |
| 547 | while ((virt & 0xfffff || (virt + off) & 0xfffff) && length >= PAGE_SIZE) { |
| 548 | alloc_init_page(virt, virt + off, prot_l1, prot_pte); |
| 549 | |
| 550 | virt += PAGE_SIZE; |
| 551 | length -= PAGE_SIZE; |
| 552 | } |
| 553 | |
| 554 | /* N.B. ARMv6 supersections are only defined to work with domain 0. |
| 555 | * Since domain assignments can in fact be arbitrary, the |
| 556 | * 'domain == 0' check below is required to insure that ARMv6 |
| 557 | * supersections are only allocated for domain 0 regardless |
| 558 | * of the actual domain assignments in use. |
| 559 | */ |
Lennert Buytenhek | 23bdf86 | 2006-03-28 21:00:40 +0100 | [diff] [blame] | 560 | if ((cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3()) |
| 561 | && domain == 0) { |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 562 | /* |
| 563 | * Align to supersection boundary if !high pages. |
| 564 | * High pages have already been checked for proper |
| 565 | * alignment above and they will fail the SUPSERSECTION_MASK |
| 566 | * check because of the way the address is encoded into |
| 567 | * offset. |
| 568 | */ |
| 569 | if (md->pfn <= 0x100000) { |
| 570 | while ((virt & ~SUPERSECTION_MASK || |
| 571 | (virt + off) & ~SUPERSECTION_MASK) && |
| 572 | length >= (PGDIR_SIZE / 2)) { |
| 573 | alloc_init_section(virt, virt + off, prot_sect); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 574 | |
Deepak Saxena | 0b7cd62 | 2005-10-28 15:19:12 +0100 | [diff] [blame] | 575 | virt += (PGDIR_SIZE / 2); |
| 576 | length -= (PGDIR_SIZE / 2); |
| 577 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 578 | } |
| 579 | |
| 580 | while (length >= SUPERSECTION_SIZE) { |
| 581 | alloc_init_supersection(virt, virt + off, prot_sect); |
| 582 | |
| 583 | virt += SUPERSECTION_SIZE; |
| 584 | length -= SUPERSECTION_SIZE; |
| 585 | } |
| 586 | } |
| 587 | |
| 588 | /* |
| 589 | * A section mapping covers half a "pgdir" entry. |
| 590 | */ |
| 591 | while (length >= (PGDIR_SIZE / 2)) { |
| 592 | alloc_init_section(virt, virt + off, prot_sect); |
| 593 | |
| 594 | virt += (PGDIR_SIZE / 2); |
| 595 | length -= (PGDIR_SIZE / 2); |
| 596 | } |
| 597 | |
| 598 | while (length >= PAGE_SIZE) { |
| 599 | alloc_init_page(virt, virt + off, prot_l1, prot_pte); |
| 600 | |
| 601 | virt += PAGE_SIZE; |
| 602 | length -= PAGE_SIZE; |
| 603 | } |
| 604 | } |
| 605 | |
| 606 | /* |
| 607 | * In order to soft-boot, we need to insert a 1:1 mapping in place of |
| 608 | * the user-mode pages. This will then ensure that we have predictable |
| 609 | * results when turning the mmu off |
| 610 | */ |
| 611 | void setup_mm_for_reboot(char mode) |
| 612 | { |
Russell King | 103461a | 2005-09-01 14:51:59 +0100 | [diff] [blame] | 613 | unsigned long base_pmdval; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 614 | pgd_t *pgd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 615 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 616 | |
| 617 | if (current->mm && current->mm->pgd) |
| 618 | pgd = current->mm->pgd; |
| 619 | else |
| 620 | pgd = init_mm.pgd; |
| 621 | |
Russell King | 103461a | 2005-09-01 14:51:59 +0100 | [diff] [blame] | 622 | base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT; |
| 623 | if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ) |
| 624 | base_pmdval |= PMD_BIT4; |
| 625 | |
| 626 | for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++, pgd++) { |
| 627 | unsigned long pmdval = (i << PGDIR_SHIFT) | base_pmdval; |
| 628 | pmd_t *pmd; |
| 629 | |
Russell King | 155bb14 | 2005-05-09 20:52:51 +0100 | [diff] [blame] | 630 | pmd = pmd_off(pgd, i << PGDIR_SHIFT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 631 | pmd[0] = __pmd(pmdval); |
| 632 | pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1))); |
| 633 | flush_pmd_entry(pmd); |
| 634 | } |
| 635 | } |
| 636 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 637 | /* |
| 638 | * Create the architecture specific mappings |
| 639 | */ |
| 640 | void __init iotable_init(struct map_desc *io_desc, int nr) |
| 641 | { |
| 642 | int i; |
| 643 | |
| 644 | for (i = 0; i < nr; i++) |
| 645 | create_mapping(io_desc + i); |
| 646 | } |