Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Page table handling routines for radix page table. |
| 3 | * |
| 4 | * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | */ |
| 11 | #include <linux/sched.h> |
| 12 | #include <linux/memblock.h> |
| 13 | #include <linux/of_fdt.h> |
| 14 | |
| 15 | #include <asm/pgtable.h> |
| 16 | #include <asm/pgalloc.h> |
| 17 | #include <asm/dma.h> |
| 18 | #include <asm/machdep.h> |
| 19 | #include <asm/mmu.h> |
| 20 | #include <asm/firmware.h> |
| 21 | |
Aneesh Kumar K.V | bde3eb6 | 2016-04-29 23:26:30 +1000 | [diff] [blame] | 22 | #include <trace/events/thp.h> |
| 23 | |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 24 | static int native_update_partition_table(u64 patb1) |
| 25 | { |
| 26 | partition_tb->patb1 = cpu_to_be64(patb1); |
| 27 | return 0; |
| 28 | } |
| 29 | |
| 30 | static __ref void *early_alloc_pgtable(unsigned long size) |
| 31 | { |
| 32 | void *pt; |
| 33 | |
| 34 | pt = __va(memblock_alloc_base(size, size, MEMBLOCK_ALLOC_ANYWHERE)); |
| 35 | memset(pt, 0, size); |
| 36 | |
| 37 | return pt; |
| 38 | } |
| 39 | |
| 40 | int radix__map_kernel_page(unsigned long ea, unsigned long pa, |
| 41 | pgprot_t flags, |
| 42 | unsigned int map_page_size) |
| 43 | { |
| 44 | pgd_t *pgdp; |
| 45 | pud_t *pudp; |
| 46 | pmd_t *pmdp; |
| 47 | pte_t *ptep; |
| 48 | /* |
| 49 | * Make sure task size is correct as per the max adddr |
| 50 | */ |
| 51 | BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE); |
| 52 | if (slab_is_available()) { |
| 53 | pgdp = pgd_offset_k(ea); |
| 54 | pudp = pud_alloc(&init_mm, pgdp, ea); |
| 55 | if (!pudp) |
| 56 | return -ENOMEM; |
| 57 | if (map_page_size == PUD_SIZE) { |
| 58 | ptep = (pte_t *)pudp; |
| 59 | goto set_the_pte; |
| 60 | } |
| 61 | pmdp = pmd_alloc(&init_mm, pudp, ea); |
| 62 | if (!pmdp) |
| 63 | return -ENOMEM; |
| 64 | if (map_page_size == PMD_SIZE) { |
| 65 | ptep = (pte_t *)pudp; |
| 66 | goto set_the_pte; |
| 67 | } |
| 68 | ptep = pte_alloc_kernel(pmdp, ea); |
| 69 | if (!ptep) |
| 70 | return -ENOMEM; |
| 71 | } else { |
| 72 | pgdp = pgd_offset_k(ea); |
| 73 | if (pgd_none(*pgdp)) { |
| 74 | pudp = early_alloc_pgtable(PUD_TABLE_SIZE); |
| 75 | BUG_ON(pudp == NULL); |
| 76 | pgd_populate(&init_mm, pgdp, pudp); |
| 77 | } |
| 78 | pudp = pud_offset(pgdp, ea); |
| 79 | if (map_page_size == PUD_SIZE) { |
| 80 | ptep = (pte_t *)pudp; |
| 81 | goto set_the_pte; |
| 82 | } |
| 83 | if (pud_none(*pudp)) { |
| 84 | pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); |
| 85 | BUG_ON(pmdp == NULL); |
| 86 | pud_populate(&init_mm, pudp, pmdp); |
| 87 | } |
| 88 | pmdp = pmd_offset(pudp, ea); |
| 89 | if (map_page_size == PMD_SIZE) { |
| 90 | ptep = (pte_t *)pudp; |
| 91 | goto set_the_pte; |
| 92 | } |
| 93 | if (!pmd_present(*pmdp)) { |
| 94 | ptep = early_alloc_pgtable(PAGE_SIZE); |
| 95 | BUG_ON(ptep == NULL); |
| 96 | pmd_populate_kernel(&init_mm, pmdp, ptep); |
| 97 | } |
| 98 | ptep = pte_offset_kernel(pmdp, ea); |
| 99 | } |
| 100 | |
| 101 | set_the_pte: |
| 102 | set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, flags)); |
| 103 | smp_wmb(); |
| 104 | return 0; |
| 105 | } |
| 106 | |
| 107 | static void __init radix_init_pgtable(void) |
| 108 | { |
| 109 | int loop_count; |
| 110 | u64 base, end, start_addr; |
| 111 | unsigned long rts_field; |
| 112 | struct memblock_region *reg; |
| 113 | unsigned long linear_page_size; |
| 114 | |
| 115 | /* We don't support slb for radix */ |
| 116 | mmu_slb_size = 0; |
| 117 | /* |
| 118 | * Create the linear mapping, using standard page size for now |
| 119 | */ |
| 120 | loop_count = 0; |
| 121 | for_each_memblock(memory, reg) { |
| 122 | |
| 123 | start_addr = reg->base; |
| 124 | |
| 125 | redo: |
| 126 | if (loop_count < 1 && mmu_psize_defs[MMU_PAGE_1G].shift) |
| 127 | linear_page_size = PUD_SIZE; |
| 128 | else if (loop_count < 2 && mmu_psize_defs[MMU_PAGE_2M].shift) |
| 129 | linear_page_size = PMD_SIZE; |
| 130 | else |
| 131 | linear_page_size = PAGE_SIZE; |
| 132 | |
| 133 | base = _ALIGN_UP(start_addr, linear_page_size); |
| 134 | end = _ALIGN_DOWN(reg->base + reg->size, linear_page_size); |
| 135 | |
| 136 | pr_info("Mapping range 0x%lx - 0x%lx with 0x%lx\n", |
| 137 | (unsigned long)base, (unsigned long)end, |
| 138 | linear_page_size); |
| 139 | |
| 140 | while (base < end) { |
| 141 | radix__map_kernel_page((unsigned long)__va(base), |
| 142 | base, PAGE_KERNEL_X, |
| 143 | linear_page_size); |
| 144 | base += linear_page_size; |
| 145 | } |
| 146 | /* |
| 147 | * map the rest using lower page size |
| 148 | */ |
| 149 | if (end < reg->base + reg->size) { |
| 150 | start_addr = end; |
| 151 | loop_count++; |
| 152 | goto redo; |
| 153 | } |
| 154 | } |
| 155 | /* |
| 156 | * Allocate Partition table and process table for the |
| 157 | * host. |
| 158 | */ |
| 159 | BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 23), "Process table size too large."); |
| 160 | process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT); |
| 161 | /* |
| 162 | * Fill in the process table. |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 163 | */ |
Aneesh Kumar K.V | b23d9c5 | 2016-06-17 11:40:36 +0530 | [diff] [blame] | 164 | rts_field = radix__get_tree_size(); |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 165 | process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE); |
| 166 | /* |
| 167 | * Fill in the partition table. We are suppose to use effective address |
| 168 | * of process table here. But our linear mapping also enable us to use |
| 169 | * physical address here. |
| 170 | */ |
| 171 | ppc_md.update_partition_table(__pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR); |
| 172 | pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd); |
| 173 | } |
| 174 | |
| 175 | static void __init radix_init_partition_table(void) |
| 176 | { |
| 177 | unsigned long rts_field; |
Aneesh Kumar K.V | b23d9c5 | 2016-06-17 11:40:36 +0530 | [diff] [blame] | 178 | |
| 179 | rts_field = radix__get_tree_size(); |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 180 | |
| 181 | BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); |
| 182 | partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT); |
| 183 | partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | |
| 184 | RADIX_PGD_INDEX_SIZE | PATB_HR); |
| 185 | printk("Partition table %p\n", partition_tb); |
| 186 | |
| 187 | memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); |
| 188 | /* |
| 189 | * update partition table control register, |
| 190 | * 64 K size. |
| 191 | */ |
| 192 | mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); |
| 193 | } |
| 194 | |
| 195 | void __init radix_init_native(void) |
| 196 | { |
| 197 | ppc_md.update_partition_table = native_update_partition_table; |
| 198 | } |
| 199 | |
| 200 | static int __init get_idx_from_shift(unsigned int shift) |
| 201 | { |
| 202 | int idx = -1; |
| 203 | |
| 204 | switch (shift) { |
| 205 | case 0xc: |
| 206 | idx = MMU_PAGE_4K; |
| 207 | break; |
| 208 | case 0x10: |
| 209 | idx = MMU_PAGE_64K; |
| 210 | break; |
| 211 | case 0x15: |
| 212 | idx = MMU_PAGE_2M; |
| 213 | break; |
| 214 | case 0x1e: |
| 215 | idx = MMU_PAGE_1G; |
| 216 | break; |
| 217 | } |
| 218 | return idx; |
| 219 | } |
| 220 | |
| 221 | static int __init radix_dt_scan_page_sizes(unsigned long node, |
| 222 | const char *uname, int depth, |
| 223 | void *data) |
| 224 | { |
| 225 | int size = 0; |
| 226 | int shift, idx; |
| 227 | unsigned int ap; |
| 228 | const __be32 *prop; |
| 229 | const char *type = of_get_flat_dt_prop(node, "device_type", NULL); |
| 230 | |
| 231 | /* We are scanning "cpu" nodes only */ |
| 232 | if (type == NULL || strcmp(type, "cpu") != 0) |
| 233 | return 0; |
| 234 | |
| 235 | prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size); |
| 236 | if (!prop) |
| 237 | return 0; |
| 238 | |
| 239 | pr_info("Page sizes from device-tree:\n"); |
| 240 | for (; size >= 4; size -= 4, ++prop) { |
| 241 | |
| 242 | struct mmu_psize_def *def; |
| 243 | |
| 244 | /* top 3 bit is AP encoding */ |
| 245 | shift = be32_to_cpu(prop[0]) & ~(0xe << 28); |
| 246 | ap = be32_to_cpu(prop[0]) >> 29; |
| 247 | pr_info("Page size sift = %d AP=0x%x\n", shift, ap); |
| 248 | |
| 249 | idx = get_idx_from_shift(shift); |
| 250 | if (idx < 0) |
| 251 | continue; |
| 252 | |
| 253 | def = &mmu_psize_defs[idx]; |
| 254 | def->shift = shift; |
| 255 | def->ap = ap; |
| 256 | } |
| 257 | |
| 258 | /* needed ? */ |
| 259 | cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B; |
| 260 | return 1; |
| 261 | } |
| 262 | |
| 263 | static void __init radix_init_page_sizes(void) |
| 264 | { |
| 265 | int rc; |
| 266 | |
| 267 | /* |
| 268 | * Try to find the available page sizes in the device-tree |
| 269 | */ |
| 270 | rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL); |
| 271 | if (rc != 0) /* Found */ |
| 272 | goto found; |
| 273 | /* |
| 274 | * let's assume we have page 4k and 64k support |
| 275 | */ |
| 276 | mmu_psize_defs[MMU_PAGE_4K].shift = 12; |
| 277 | mmu_psize_defs[MMU_PAGE_4K].ap = 0x0; |
| 278 | |
| 279 | mmu_psize_defs[MMU_PAGE_64K].shift = 16; |
| 280 | mmu_psize_defs[MMU_PAGE_64K].ap = 0x5; |
| 281 | found: |
| 282 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
| 283 | if (mmu_psize_defs[MMU_PAGE_2M].shift) { |
| 284 | /* |
| 285 | * map vmemmap using 2M if available |
| 286 | */ |
| 287 | mmu_vmemmap_psize = MMU_PAGE_2M; |
| 288 | } |
| 289 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ |
| 290 | return; |
| 291 | } |
| 292 | |
| 293 | void __init radix__early_init_mmu(void) |
| 294 | { |
| 295 | unsigned long lpcr; |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 296 | |
| 297 | #ifdef CONFIG_PPC_64K_PAGES |
| 298 | /* PAGE_SIZE mappings */ |
| 299 | mmu_virtual_psize = MMU_PAGE_64K; |
| 300 | #else |
| 301 | mmu_virtual_psize = MMU_PAGE_4K; |
| 302 | #endif |
| 303 | |
| 304 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
| 305 | /* vmemmap mapping */ |
| 306 | mmu_vmemmap_psize = mmu_virtual_psize; |
| 307 | #endif |
| 308 | /* |
| 309 | * initialize page table size |
| 310 | */ |
| 311 | __pte_index_size = RADIX_PTE_INDEX_SIZE; |
| 312 | __pmd_index_size = RADIX_PMD_INDEX_SIZE; |
| 313 | __pud_index_size = RADIX_PUD_INDEX_SIZE; |
| 314 | __pgd_index_size = RADIX_PGD_INDEX_SIZE; |
| 315 | __pmd_cache_index = RADIX_PMD_INDEX_SIZE; |
| 316 | __pte_table_size = RADIX_PTE_TABLE_SIZE; |
| 317 | __pmd_table_size = RADIX_PMD_TABLE_SIZE; |
| 318 | __pud_table_size = RADIX_PUD_TABLE_SIZE; |
| 319 | __pgd_table_size = RADIX_PGD_TABLE_SIZE; |
| 320 | |
Aneesh Kumar K.V | a2f41eb | 2016-04-29 23:26:19 +1000 | [diff] [blame] | 321 | __pmd_val_bits = RADIX_PMD_VAL_BITS; |
| 322 | __pud_val_bits = RADIX_PUD_VAL_BITS; |
| 323 | __pgd_val_bits = RADIX_PGD_VAL_BITS; |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 324 | |
Aneesh Kumar K.V | d6a9996 | 2016-04-29 23:26:21 +1000 | [diff] [blame] | 325 | __kernel_virt_start = RADIX_KERN_VIRT_START; |
| 326 | __kernel_virt_size = RADIX_KERN_VIRT_SIZE; |
| 327 | __vmalloc_start = RADIX_VMALLOC_START; |
| 328 | __vmalloc_end = RADIX_VMALLOC_END; |
| 329 | vmemmap = (struct page *)RADIX_VMEMMAP_BASE; |
| 330 | ioremap_bot = IOREMAP_BASE; |
Darren Stevens | bfa3708 | 2016-06-29 21:06:28 +0100 | [diff] [blame^] | 331 | |
| 332 | #ifdef CONFIG_PCI |
| 333 | pci_io_base = ISA_IO_BASE; |
| 334 | #endif |
| 335 | |
Aneesh Kumar K.V | 5ed7ecd | 2016-04-29 23:26:23 +1000 | [diff] [blame] | 336 | /* |
| 337 | * For now radix also use the same frag size |
| 338 | */ |
| 339 | __pte_frag_nr = H_PTE_FRAG_NR; |
| 340 | __pte_frag_size_shift = H_PTE_FRAG_SIZE_SHIFT; |
Aneesh Kumar K.V | d6a9996 | 2016-04-29 23:26:21 +1000 | [diff] [blame] | 341 | |
Aneesh Kumar K.V | a2f41eb | 2016-04-29 23:26:19 +1000 | [diff] [blame] | 342 | radix_init_page_sizes(); |
Aneesh Kumar K.V | d6c8860 | 2016-05-31 11:56:29 +0530 | [diff] [blame] | 343 | if (!firmware_has_feature(FW_FEATURE_LPAR)) { |
| 344 | lpcr = mfspr(SPRN_LPCR); |
| 345 | mtspr(SPRN_LPCR, lpcr | LPCR_UPRT); |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 346 | radix_init_partition_table(); |
Aneesh Kumar K.V | d6c8860 | 2016-05-31 11:56:29 +0530 | [diff] [blame] | 347 | } |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 348 | |
| 349 | radix_init_pgtable(); |
| 350 | } |
| 351 | |
| 352 | void radix__early_init_mmu_secondary(void) |
| 353 | { |
| 354 | unsigned long lpcr; |
| 355 | /* |
Aneesh Kumar K.V | d6c8860 | 2016-05-31 11:56:29 +0530 | [diff] [blame] | 356 | * update partition table control register and UPRT |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 357 | */ |
Aneesh Kumar K.V | d6c8860 | 2016-05-31 11:56:29 +0530 | [diff] [blame] | 358 | if (!firmware_has_feature(FW_FEATURE_LPAR)) { |
| 359 | lpcr = mfspr(SPRN_LPCR); |
| 360 | mtspr(SPRN_LPCR, lpcr | LPCR_UPRT); |
| 361 | |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 362 | mtspr(SPRN_PTCR, |
| 363 | __pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); |
Aneesh Kumar K.V | d6c8860 | 2016-05-31 11:56:29 +0530 | [diff] [blame] | 364 | } |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 365 | } |
| 366 | |
| 367 | void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base, |
| 368 | phys_addr_t first_memblock_size) |
| 369 | { |
Aneesh Kumar K.V | 177ba7c | 2016-04-29 23:26:10 +1000 | [diff] [blame] | 370 | /* We don't currently support the first MEMBLOCK not mapping 0 |
| 371 | * physical on those processors |
| 372 | */ |
| 373 | BUG_ON(first_memblock_base != 0); |
| 374 | /* |
| 375 | * We limit the allocation that depend on ppc64_rma_size |
| 376 | * to first_memblock_size. We also clamp it to 1GB to |
| 377 | * avoid some funky things such as RTAS bugs. |
| 378 | * |
| 379 | * On radix config we really don't have a limitation |
| 380 | * on real mode access. But keeping it as above works |
| 381 | * well enough. |
| 382 | */ |
| 383 | ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000); |
| 384 | /* |
| 385 | * Finally limit subsequent allocations. We really don't want |
| 386 | * to limit the memblock allocations to rma_size. FIXME!! should |
| 387 | * we even limit at all ? |
| 388 | */ |
Aneesh Kumar K.V | 2bfd65e | 2016-04-29 23:25:58 +1000 | [diff] [blame] | 389 | memblock_set_current_limit(first_memblock_base + first_memblock_size); |
| 390 | } |
Aneesh Kumar K.V | d9225ad | 2016-04-29 23:26:00 +1000 | [diff] [blame] | 391 | |
| 392 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
| 393 | int __meminit radix__vmemmap_create_mapping(unsigned long start, |
| 394 | unsigned long page_size, |
| 395 | unsigned long phys) |
| 396 | { |
| 397 | /* Create a PTE encoding */ |
| 398 | unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW; |
| 399 | |
| 400 | BUG_ON(radix__map_kernel_page(start, phys, __pgprot(flags), page_size)); |
| 401 | return 0; |
| 402 | } |
| 403 | |
| 404 | #ifdef CONFIG_MEMORY_HOTPLUG |
| 405 | void radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size) |
| 406 | { |
| 407 | /* FIXME!! intel does more. We should free page tables mapping vmemmap ? */ |
| 408 | } |
| 409 | #endif |
| 410 | #endif |
Aneesh Kumar K.V | bde3eb6 | 2016-04-29 23:26:30 +1000 | [diff] [blame] | 411 | |
| 412 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 413 | |
| 414 | unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, |
| 415 | pmd_t *pmdp, unsigned long clr, |
| 416 | unsigned long set) |
| 417 | { |
| 418 | unsigned long old; |
| 419 | |
| 420 | #ifdef CONFIG_DEBUG_VM |
| 421 | WARN_ON(!radix__pmd_trans_huge(*pmdp)); |
| 422 | assert_spin_locked(&mm->page_table_lock); |
| 423 | #endif |
| 424 | |
| 425 | old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1); |
| 426 | trace_hugepage_update(addr, old, clr, set); |
| 427 | |
| 428 | return old; |
| 429 | } |
| 430 | |
| 431 | pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, |
| 432 | pmd_t *pmdp) |
| 433 | |
| 434 | { |
| 435 | pmd_t pmd; |
| 436 | |
| 437 | VM_BUG_ON(address & ~HPAGE_PMD_MASK); |
| 438 | VM_BUG_ON(radix__pmd_trans_huge(*pmdp)); |
| 439 | /* |
| 440 | * khugepaged calls this for normal pmd |
| 441 | */ |
| 442 | pmd = *pmdp; |
| 443 | pmd_clear(pmdp); |
| 444 | /*FIXME!! Verify whether we need this kick below */ |
| 445 | kick_all_cpus_sync(); |
| 446 | flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); |
| 447 | return pmd; |
| 448 | } |
| 449 | |
| 450 | /* |
| 451 | * For us pgtable_t is pte_t *. Inorder to save the deposisted |
| 452 | * page table, we consider the allocated page table as a list |
| 453 | * head. On withdraw we need to make sure we zero out the used |
| 454 | * list_head memory area. |
| 455 | */ |
| 456 | void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
| 457 | pgtable_t pgtable) |
| 458 | { |
| 459 | struct list_head *lh = (struct list_head *) pgtable; |
| 460 | |
| 461 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
| 462 | |
| 463 | /* FIFO */ |
| 464 | if (!pmd_huge_pte(mm, pmdp)) |
| 465 | INIT_LIST_HEAD(lh); |
| 466 | else |
| 467 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
| 468 | pmd_huge_pte(mm, pmdp) = pgtable; |
| 469 | } |
| 470 | |
| 471 | pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
| 472 | { |
| 473 | pte_t *ptep; |
| 474 | pgtable_t pgtable; |
| 475 | struct list_head *lh; |
| 476 | |
| 477 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
| 478 | |
| 479 | /* FIFO */ |
| 480 | pgtable = pmd_huge_pte(mm, pmdp); |
| 481 | lh = (struct list_head *) pgtable; |
| 482 | if (list_empty(lh)) |
| 483 | pmd_huge_pte(mm, pmdp) = NULL; |
| 484 | else { |
| 485 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
| 486 | list_del(lh); |
| 487 | } |
| 488 | ptep = (pte_t *) pgtable; |
| 489 | *ptep = __pte(0); |
| 490 | ptep++; |
| 491 | *ptep = __pte(0); |
| 492 | return pgtable; |
| 493 | } |
| 494 | |
| 495 | |
| 496 | pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm, |
| 497 | unsigned long addr, pmd_t *pmdp) |
| 498 | { |
| 499 | pmd_t old_pmd; |
| 500 | unsigned long old; |
| 501 | |
| 502 | old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0); |
| 503 | old_pmd = __pmd(old); |
| 504 | /* |
| 505 | * Serialize against find_linux_pte_or_hugepte which does lock-less |
| 506 | * lookup in page tables with local interrupts disabled. For huge pages |
| 507 | * it casts pmd_t to pte_t. Since format of pte_t is different from |
| 508 | * pmd_t we want to prevent transit from pmd pointing to page table |
| 509 | * to pmd pointing to huge page (and back) while interrupts are disabled. |
| 510 | * We clear pmd to possibly replace it with page table pointer in |
| 511 | * different code paths. So make sure we wait for the parallel |
| 512 | * find_linux_pte_or_hugepage to finish. |
| 513 | */ |
| 514 | kick_all_cpus_sync(); |
| 515 | return old_pmd; |
| 516 | } |
| 517 | |
| 518 | int radix__has_transparent_hugepage(void) |
| 519 | { |
| 520 | /* For radix 2M at PMD level means thp */ |
| 521 | if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT) |
| 522 | return 1; |
| 523 | return 0; |
| 524 | } |
| 525 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |