blob: b8c75e65157cc0a5b0229e5da99b941b6fdc0475 [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
2 * This file contains ioremap and related functions for 64-bit machines.
3 *
4 * Derived from arch/ppc64/mm/init.c
5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
6 *
7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
8 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
9 * Copyright (C) 1996 Paul Mackerras
Paul Mackerras14cf11a2005-09-26 16:04:21 +100010 *
11 * Derived from "arch/i386/mm/init.c"
12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
13 *
14 * Dave Engebretsen <engebret@us.ibm.com>
15 * Rework for PPC64 port.
16 *
17 * This program is free software; you can redistribute it and/or
18 * modify it under the terms of the GNU General Public License
19 * as published by the Free Software Foundation; either version
20 * 2 of the License, or (at your option) any later version.
21 *
22 */
23
Paul Mackerras14cf11a2005-09-26 16:04:21 +100024#include <linux/signal.h>
25#include <linux/sched.h>
26#include <linux/kernel.h>
27#include <linux/errno.h>
28#include <linux/string.h>
Paul Gortmaker66b15db2011-05-27 10:46:24 -040029#include <linux/export.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100030#include <linux/types.h>
31#include <linux/mman.h>
32#include <linux/mm.h>
33#include <linux/swap.h>
34#include <linux/stddef.h>
35#include <linux/vmalloc.h>
Yinghai Lu95f72d12010-07-12 14:36:09 +100036#include <linux/memblock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Aneesh Kumar K.V06743522014-11-05 21:57:39 +053038#include <linux/hugetlb.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100039
40#include <asm/pgalloc.h>
41#include <asm/page.h>
42#include <asm/prom.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100043#include <asm/io.h>
44#include <asm/mmu_context.h>
45#include <asm/pgtable.h>
46#include <asm/mmu.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100047#include <asm/smp.h>
48#include <asm/machdep.h>
49#include <asm/tlb.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100050#include <asm/processor.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100051#include <asm/cputable.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100052#include <asm/sections.h>
Stephen Rothwell5e203d62006-09-25 13:36:31 +100053#include <asm/firmware.h>
Anton Blanchard68cf0d62014-09-17 22:15:35 +100054#include <asm/dma.h>
David Gibson800fc3e2005-11-16 15:43:48 +110055
56#include "mmu_decl.h"
Paul Mackerras14cf11a2005-09-26 16:04:21 +100057
Aneesh Kumar K.V9e813302014-08-13 12:32:04 +053058#define CREATE_TRACE_POINTS
59#include <trace/events/thp.h>
60
Aneesh Kumar K.V78f1dbd2012-09-10 02:52:57 +000061#ifdef CONFIG_PPC_STD_MMU_64
Aneesh Kumar K.Vaf81d782013-03-13 03:34:55 +000062#if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
Aneesh Kumar K.V78f1dbd2012-09-10 02:52:57 +000063#error TASK_SIZE_USER64 exceeds user VSID range
64#endif
65#endif
66
Aneesh Kumar K.V50de5962016-04-29 23:25:43 +100067#ifdef CONFIG_PPC_BOOK3S_64
68/*
69 * partition table and process table for ISA 3.0
70 */
71struct prtb_entry *process_tb;
72struct patb_entry *partition_tb;
Aneesh Kumar K.Vdd1842a2016-04-29 23:25:49 +100073/*
74 * page table size
75 */
76unsigned long __pte_index_size;
77EXPORT_SYMBOL(__pte_index_size);
78unsigned long __pmd_index_size;
79EXPORT_SYMBOL(__pmd_index_size);
80unsigned long __pud_index_size;
81EXPORT_SYMBOL(__pud_index_size);
82unsigned long __pgd_index_size;
83EXPORT_SYMBOL(__pgd_index_size);
84unsigned long __pmd_cache_index;
85EXPORT_SYMBOL(__pmd_cache_index);
86unsigned long __pte_table_size;
87EXPORT_SYMBOL(__pte_table_size);
88unsigned long __pmd_table_size;
89EXPORT_SYMBOL(__pmd_table_size);
90unsigned long __pud_table_size;
91EXPORT_SYMBOL(__pud_table_size);
92unsigned long __pgd_table_size;
93EXPORT_SYMBOL(__pgd_table_size);
Aneesh Kumar K.Va2f41eb92016-04-29 23:26:19 +100094unsigned long __pmd_val_bits;
95EXPORT_SYMBOL(__pmd_val_bits);
96unsigned long __pud_val_bits;
97EXPORT_SYMBOL(__pud_val_bits);
98unsigned long __pgd_val_bits;
99EXPORT_SYMBOL(__pgd_val_bits);
Aneesh Kumar K.Vdd1842a2016-04-29 23:25:49 +1000100
Aneesh Kumar K.V50de5962016-04-29 23:25:43 +1000101#endif
Aneesh Kumar K.V78f1dbd2012-09-10 02:52:57 +0000102unsigned long ioremap_bot = IOREMAP_BASE;
Benjamin Herrenschmidta2450672009-07-23 23:15:16 +0000103
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000104/**
105 * __ioremap_at - Low level function to establish the page tables
106 * for an IO mapping
107 */
108void __iomem * __ioremap_at(phys_addr_t pa, void *ea, unsigned long size,
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000109 unsigned long flags)
110{
111 unsigned long i;
112
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700113 /* Make sure we have the base flags */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000114 if ((flags & _PAGE_PRESENT) == 0)
115 flags |= pgprot_val(PAGE_KERNEL);
116
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700117 /* We don't support the 4K PFN hack with ioremap */
Aneesh Kumar K.V945537d2016-04-29 23:25:45 +1000118 if (flags & H_PAGE_4K_PFN)
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700119 return NULL;
120
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000121 WARN_ON(pa & ~PAGE_MASK);
122 WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
123 WARN_ON(size & ~PAGE_MASK);
124
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000125 for (i = 0; i < size; i += PAGE_SIZE)
Benjamin Herrenschmidta2450672009-07-23 23:15:16 +0000126 if (map_kernel_page((unsigned long)ea+i, pa+i, flags))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000127 return NULL;
128
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000129 return (void __iomem *)ea;
130}
131
132/**
133 * __iounmap_from - Low level function to tear down the page tables
134 * for an IO mapping. This is used for mappings that
135 * are manipulated manually, like partial unmapping of
136 * PCI IOs or ISA space.
137 */
138void __iounmap_at(void *ea, unsigned long size)
139{
140 WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
141 WARN_ON(size & ~PAGE_MASK);
142
143 unmap_kernel_range((unsigned long)ea, size);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000144}
145
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000146void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size,
147 unsigned long flags, void *caller)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000148{
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000149 phys_addr_t paligned;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000150 void __iomem *ret;
151
152 /*
153 * Choose an address to map it to.
154 * Once the imalloc system is running, we use it.
155 * Before that, we map using addresses going
156 * up from ioremap_bot. imalloc will use
157 * the addresses from ioremap_bot through
158 * IMALLOC_END
159 *
160 */
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000161 paligned = addr & PAGE_MASK;
162 size = PAGE_ALIGN(addr + size) - paligned;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000163
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000164 if ((size == 0) || (paligned == 0))
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000165 return NULL;
166
Michael Ellermanf691fa12015-03-30 14:10:37 +1100167 if (slab_is_available()) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000168 struct vm_struct *area;
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000169
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000170 area = __get_vm_area_caller(size, VM_IOREMAP,
171 ioremap_bot, IOREMAP_END,
172 caller);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000173 if (area == NULL)
174 return NULL;
Michael Ellerman7a9d1252010-11-28 18:26:36 +0000175
176 area->phys_addr = paligned;
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000177 ret = __ioremap_at(paligned, area->addr, size, flags);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000178 if (!ret)
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000179 vunmap(area->addr);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000180 } else {
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000181 ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000182 if (ret)
183 ioremap_bot += size;
184 }
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000185
186 if (ret)
187 ret += addr & ~PAGE_MASK;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000188 return ret;
189}
190
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000191void __iomem * __ioremap(phys_addr_t addr, unsigned long size,
192 unsigned long flags)
193{
194 return __ioremap_caller(addr, size, flags, __builtin_return_address(0));
195}
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100196
Benjamin Herrenschmidt68a64352006-11-13 09:27:39 +1100197void __iomem * ioremap(phys_addr_t addr, unsigned long size)
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100198{
Aneesh Kumar K.V72176dd2016-04-29 23:25:37 +1000199 unsigned long flags = pgprot_val(pgprot_noncached(__pgprot(0)));
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000200 void *caller = __builtin_return_address(0);
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100201
202 if (ppc_md.ioremap)
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000203 return ppc_md.ioremap(addr, size, flags, caller);
204 return __ioremap_caller(addr, size, flags, caller);
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100205}
206
Anton Blanchardbe135f42011-05-08 21:41:59 +0000207void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size)
208{
Aneesh Kumar K.V72176dd2016-04-29 23:25:37 +1000209 unsigned long flags = pgprot_val(pgprot_noncached_wc(__pgprot(0)));
Anton Blanchardbe135f42011-05-08 21:41:59 +0000210 void *caller = __builtin_return_address(0);
211
212 if (ppc_md.ioremap)
213 return ppc_md.ioremap(addr, size, flags, caller);
214 return __ioremap_caller(addr, size, flags, caller);
215}
216
Anton Blanchard40f1ce72011-05-08 21:43:47 +0000217void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size,
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100218 unsigned long flags)
219{
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000220 void *caller = __builtin_return_address(0);
221
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700222 /* writeable implies dirty for kernel addresses */
Aneesh Kumar K.Vc7d54842016-04-29 23:25:30 +1000223 if (flags & _PAGE_WRITE)
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700224 flags |= _PAGE_DIRTY;
225
Aneesh Kumar K.Vac29c642016-04-29 23:25:34 +1000226 /* we don't want to let _PAGE_EXEC leak out */
227 flags &= ~_PAGE_EXEC;
228 /*
229 * Force kernel mapping.
230 */
231#if defined(CONFIG_PPC_BOOK3S_64)
232 flags |= _PAGE_PRIVILEGED;
233#else
234 flags &= ~_PAGE_USER;
235#endif
236
Benjamin Herrenschmidta1f242f2008-07-23 21:27:08 -0700237
Benjamin Herrenschmidt55052ee2010-04-07 14:39:36 +1000238#ifdef _PAGE_BAP_SR
239 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
240 * which means that we just cleared supervisor access... oops ;-) This
241 * restores it
242 */
243 flags |= _PAGE_BAP_SR;
244#endif
245
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100246 if (ppc_md.ioremap)
Benjamin Herrenschmidt1cdab552009-02-22 16:19:14 +0000247 return ppc_md.ioremap(addr, size, flags, caller);
248 return __ioremap_caller(addr, size, flags, caller);
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100249}
250
251
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000252/*
253 * Unmap an IO region and remove it from imalloc'd list.
254 * Access to IO memory should be serialized by driver.
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000255 */
Benjamin Herrenschmidt68a64352006-11-13 09:27:39 +1100256void __iounmap(volatile void __iomem *token)
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000257{
258 void *addr;
259
Michael Ellermanf691fa12015-03-30 14:10:37 +1100260 if (!slab_is_available())
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000261 return;
262
Benjamin Herrenschmidt3d5134e2007-06-04 15:15:36 +1000263 addr = (void *) ((unsigned long __force)
264 PCI_FIX_ADDR(token) & PAGE_MASK);
265 if ((unsigned long)addr < ioremap_bot) {
266 printk(KERN_WARNING "Attempt to iounmap early bolted mapping"
267 " at 0x%p\n", addr);
268 return;
269 }
270 vunmap(addr);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000271}
272
Benjamin Herrenschmidt68a64352006-11-13 09:27:39 +1100273void iounmap(volatile void __iomem *token)
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100274{
275 if (ppc_md.iounmap)
276 ppc_md.iounmap(token);
277 else
278 __iounmap(token);
279}
280
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000281EXPORT_SYMBOL(ioremap);
Anton Blanchardbe135f42011-05-08 21:41:59 +0000282EXPORT_SYMBOL(ioremap_wc);
Anton Blanchard40f1ce72011-05-08 21:43:47 +0000283EXPORT_SYMBOL(ioremap_prot);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000284EXPORT_SYMBOL(__ioremap);
Olof Johanssona302cb92007-08-31 13:58:51 +1000285EXPORT_SYMBOL(__ioremap_at);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000286EXPORT_SYMBOL(iounmap);
Benjamin Herrenschmidt4cb3cee2006-11-11 17:25:10 +1100287EXPORT_SYMBOL(__iounmap);
Olof Johanssona302cb92007-08-31 13:58:51 +1000288EXPORT_SYMBOL(__iounmap_at);
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000289
Aneesh Kumar K.V06743522014-11-05 21:57:39 +0530290#ifndef __PAGETABLE_PUD_FOLDED
291/* 4 level page table */
292struct page *pgd_page(pgd_t pgd)
293{
294 if (pgd_huge(pgd))
295 return pte_page(pgd_pte(pgd));
296 return virt_to_page(pgd_page_vaddr(pgd));
297}
298#endif
299
300struct page *pud_page(pud_t pud)
301{
302 if (pud_huge(pud))
303 return pte_page(pud_pte(pud));
304 return virt_to_page(pud_page_vaddr(pud));
305}
306
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530307/*
308 * For hugepage we have pfn in the pmd, we use PTE_RPN_SHIFT bits for flags
309 * For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address.
310 */
311struct page *pmd_page(pmd_t pmd)
312{
Aneesh Kumar K.V06743522014-11-05 21:57:39 +0530313 if (pmd_trans_huge(pmd) || pmd_huge(pmd))
Aneesh Kumar K.Ve34aa032015-12-01 09:06:53 +0530314 return pte_page(pmd_pte(pmd));
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530315 return virt_to_page(pmd_page_vaddr(pmd));
316}
317
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000318#ifdef CONFIG_PPC_64K_PAGES
319static pte_t *get_from_cache(struct mm_struct *mm)
320{
321 void *pte_frag, *ret;
322
323 spin_lock(&mm->page_table_lock);
324 ret = mm->context.pte_frag;
325 if (ret) {
326 pte_frag = ret + PTE_FRAG_SIZE;
327 /*
328 * If we have taken up all the fragments mark PTE page NULL
329 */
330 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
331 pte_frag = NULL;
332 mm->context.pte_frag = pte_frag;
333 }
334 spin_unlock(&mm->page_table_lock);
335 return (pte_t *)ret;
336}
337
338static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
339{
340 void *ret = NULL;
341 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
342 __GFP_REPEAT | __GFP_ZERO);
343 if (!page)
344 return NULL;
Kirill A. Shutemov4f8049432013-11-14 14:31:38 -0800345 if (!kernel && !pgtable_page_ctor(page)) {
346 __free_page(page);
347 return NULL;
348 }
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000349
350 ret = page_address(page);
351 spin_lock(&mm->page_table_lock);
352 /*
353 * If we find pgtable_page set, we return
354 * the allocated page with single fragement
355 * count.
356 */
357 if (likely(!mm->context.pte_frag)) {
Joonsoo Kimfe896d12016-03-17 14:19:26 -0700358 set_page_count(page, PTE_FRAG_NR);
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000359 mm->context.pte_frag = ret + PTE_FRAG_SIZE;
360 }
361 spin_unlock(&mm->page_table_lock);
362
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000363 return (pte_t *)ret;
364}
365
Aneesh Kumar K.V74701d52016-04-29 23:26:17 +1000366pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000367{
368 pte_t *pte;
369
370 pte = get_from_cache(mm);
371 if (pte)
372 return pte;
373
374 return __alloc_for_cache(mm, kernel);
375}
Aneesh Kumar K.V934828e2016-04-29 23:26:18 +1000376#endif /* CONFIG_PPC_64K_PAGES */
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000377
Aneesh Kumar K.V74701d52016-04-29 23:26:17 +1000378void pte_fragment_free(unsigned long *table, int kernel)
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000379{
380 struct page *page = virt_to_page(table);
381 if (put_page_testzero(page)) {
382 if (!kernel)
383 pgtable_page_dtor(page);
384 free_hot_cold_page(page, 0);
385 }
386}
387
388#ifdef CONFIG_SMP
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000389void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
390{
391 unsigned long pgf = (unsigned long)table;
392
393 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
394 pgf |= shift;
395 tlb_remove_table(tlb, (void *)pgf);
396}
397
398void __tlb_remove_table(void *_table)
399{
400 void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
401 unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
402
403 if (!shift)
404 /* PTE page needs special handling */
Aneesh Kumar K.V74701d52016-04-29 23:26:17 +1000405 pte_fragment_free(table, 0);
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000406 else {
407 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
408 kmem_cache_free(PGT_CACHE(shift), table);
409 }
410}
411#else
412void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
413{
414 if (!shift) {
415 /* PTE page needs special handling */
Aneesh Kumar K.V74701d52016-04-29 23:26:17 +1000416 pte_fragment_free(table, 0);
Aneesh Kumar K.V5c1f6ee2013-04-28 09:37:33 +0000417 } else {
418 BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
419 kmem_cache_free(PGT_CACHE(shift), table);
420 }
421}
422#endif
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530423
424#ifdef CONFIG_TRANSPARENT_HUGEPAGE
425
426/*
427 * This is called when relaxing access to a hugepage. It's also called in the page
428 * fault path when we don't hit any of the major fault cases, ie, a minor
429 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
430 * handled those two for us, we additionally deal with missing execute
431 * permission here on some processors
432 */
433int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
434 pmd_t *pmdp, pmd_t entry, int dirty)
435{
436 int changed;
437#ifdef CONFIG_DEBUG_VM
438 WARN_ON(!pmd_trans_huge(*pmdp));
439 assert_spin_locked(&vma->vm_mm->page_table_lock);
440#endif
441 changed = !pmd_same(*(pmdp), entry);
442 if (changed) {
443 __ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
444 /*
445 * Since we are not supporting SW TLB systems, we don't
446 * have any thing similar to flush_tlb_page_nohash()
447 */
448 }
449 return changed;
450}
451
452unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
Aneesh Kumar K.V88247e82014-02-12 09:13:36 +0530453 pmd_t *pmdp, unsigned long clr,
454 unsigned long set)
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530455{
456
Aneesh Kumar K.V5dc1ef82016-04-29 23:25:28 +1000457 __be64 old_be, tmp;
458 unsigned long old;
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530459
460#ifdef CONFIG_DEBUG_VM
461 WARN_ON(!pmd_trans_huge(*pmdp));
462 assert_spin_locked(&mm->page_table_lock);
463#endif
464
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530465 __asm__ __volatile__(
466 "1: ldarx %0,0,%3\n\
Aneesh Kumar K.V5dc1ef82016-04-29 23:25:28 +1000467 and. %1,%0,%6\n\
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530468 bne- 1b \n\
469 andc %1,%0,%4 \n\
Aneesh Kumar K.V88247e82014-02-12 09:13:36 +0530470 or %1,%1,%7\n\
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530471 stdcx. %1,0,%3 \n\
472 bne- 1b"
Aneesh Kumar K.V5dc1ef82016-04-29 23:25:28 +1000473 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
474 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
Aneesh Kumar K.V945537d2016-04-29 23:25:45 +1000475 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530476 : "cc" );
Aneesh Kumar K.V4bece392016-04-29 23:25:26 +1000477
Aneesh Kumar K.V5dc1ef82016-04-29 23:25:28 +1000478 old = be64_to_cpu(old_be);
479
Aneesh Kumar K.V9e813302014-08-13 12:32:04 +0530480 trace_hugepage_update(addr, old, clr, set);
Aneesh Kumar K.V945537d2016-04-29 23:25:45 +1000481 if (old & H_PAGE_HASHPTE)
Aneesh Kumar K.Vfc047952014-08-13 12:32:00 +0530482 hpte_do_hugepage_flush(mm, addr, pmdp, old);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530483 return old;
484}
485
Aneesh Kumar K.V15a25b22015-06-24 16:57:39 -0700486pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
487 pmd_t *pmdp)
488{
489 pmd_t pmd;
490
491 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
492 VM_BUG_ON(pmd_trans_huge(*pmdp));
493
494 pmd = *pmdp;
495 pmd_clear(pmdp);
496 /*
497 * Wait for all pending hash_page to finish. This is needed
498 * in case of subpage collapse. When we collapse normal pages
499 * to hugepage, we first clear the pmd, then invalidate all
500 * the PTE entries. The assumption here is that any low level
501 * page fault will see a none pmd and take the slow path that
502 * will wait on mmap_sem. But we could very well be in a
503 * hash_page with local ptep pointer value. Such a hash page
504 * can result in adding new HPTE entries for normal subpages.
505 * That means we could be modifying the page content as we
506 * copy them to a huge page. So wait for parallel hash_page
507 * to finish before invalidating HPTE entries. We can do this
508 * by sending an IPI to all the cpus and executing a dummy
509 * function there.
510 */
511 kick_all_cpus_sync();
512 /*
513 * Now invalidate the hpte entries in the range
514 * covered by pmd. This make sure we take a
515 * fault and will find the pmd as none, which will
516 * result in a major fault which takes mmap_sem and
517 * hence wait for collapse to complete. Without this
518 * the __collapse_huge_page_copy can result in copying
519 * the old content.
520 */
521 flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530522 return pmd;
523}
524
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530525/*
526 * We currently remove entries from the hashtable regardless of whether
Aneesh Kumar K.Vff844b72016-04-29 23:25:39 +1000527 * the entry was young or dirty.
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530528 *
529 * We should be more intelligent about this but for the moment we override
530 * these functions and force a tlb flush unconditionally
531 */
Aneesh Kumar K.Vff844b72016-04-29 23:25:39 +1000532int pmdp_test_and_clear_young(struct vm_area_struct *vma,
533 unsigned long address, pmd_t *pmdp)
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530534{
535 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp);
536}
537
538/*
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530539 * We want to put the pgtable in pmd and use pgtable for tracking
540 * the base page size hptes
541 */
542void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
543 pgtable_t pgtable)
544{
545 pgtable_t *pgtable_slot;
546 assert_spin_locked(&mm->page_table_lock);
547 /*
548 * we store the pgtable in the second half of PMD
549 */
550 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
551 *pgtable_slot = pgtable;
552 /*
553 * expose the deposited pgtable to other cpus.
554 * before we set the hugepage PTE at pmd level
555 * hash fault code looks at the deposted pgtable
556 * to store hash index values.
557 */
558 smp_wmb();
559}
560
561pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
562{
563 pgtable_t pgtable;
564 pgtable_t *pgtable_slot;
565
566 assert_spin_locked(&mm->page_table_lock);
567 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
568 pgtable = *pgtable_slot;
569 /*
570 * Once we withdraw, mark the entry NULL.
571 */
572 *pgtable_slot = NULL;
573 /*
574 * We store HPTE information in the deposited PTE fragment.
575 * zero out the content on withdraw.
576 */
577 memset(pgtable, 0, PTE_FRAG_SIZE);
578 return pgtable;
579}
580
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +0530581void pmdp_huge_split_prepare(struct vm_area_struct *vma,
582 unsigned long address, pmd_t *pmdp)
583{
584 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
585 VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
586
587 /*
588 * We can't mark the pmd none here, because that will cause a race
589 * against exit_mmap. We need to continue mark pmd TRANS HUGE, while
590 * we spilt, but at the same time we wan't rest of the ppc64 code
591 * not to insert hash pte on this, because we will be modifying
592 * the deposited pgtable in the caller of this function. Hence
593 * clear the _PAGE_USER so that we move the fault handling to
594 * higher level function and that will serialize against ptl.
595 * We need to flush existing hash pte entries here even though,
596 * the translation is still valid, because we will withdraw
597 * pgtable_t after this.
598 */
Aneesh Kumar K.Vac29c642016-04-29 23:25:34 +1000599 pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +0530600}
601
602
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530603/*
604 * set a new huge pmd. We should not be called for updating
605 * an existing pmd entry. That should go via pmd_hugepage_update.
606 */
607void set_pmd_at(struct mm_struct *mm, unsigned long addr,
608 pmd_t *pmdp, pmd_t pmd)
609{
610#ifdef CONFIG_DEBUG_VM
Aneesh Kumar K.Vc7d54842016-04-29 23:25:30 +1000611 WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530612 assert_spin_locked(&mm->page_table_lock);
613 WARN_ON(!pmd_trans_huge(pmd));
614#endif
Michael Ellerman4f9c53c2015-03-25 20:11:57 +1100615 trace_hugepage_set_pmd(addr, pmd_val(pmd));
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530616 return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
617}
618
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +0530619/*
620 * We use this to invalidate a pmdp entry before switching from a
621 * hugepte to regular pmd entry.
622 */
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530623void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
624 pmd_t *pmdp)
625{
Aneesh Kumar K.V88247e82014-02-12 09:13:36 +0530626 pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +0530627
628 /*
629 * This ensures that generic code that rely on IRQ disabling
630 * to prevent a parallel THP split work as expected.
631 */
632 kick_all_cpus_sync();
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530633}
634
635/*
636 * A linux hugepage PMD was changed and the corresponding hash table entries
637 * neesd to be flushed.
638 */
639void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
Aneesh Kumar K.Vfc047952014-08-13 12:32:00 +0530640 pmd_t *pmdp, unsigned long old_pmd)
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530641{
Aneesh Kumar K.Vaefa5682014-12-04 11:00:14 +0530642 int ssize;
Aneesh Kumar K.Vf1581bf2014-11-02 21:15:27 +0530643 unsigned int psize;
644 unsigned long vsid;
Aneesh Kumar K.Vaefa5682014-12-04 11:00:14 +0530645 unsigned long flags = 0;
Aneesh Kumar K.Vd557b092014-11-02 21:15:28 +0530646 const struct cpumask *tmp;
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530647
Aneesh Kumar K.Vfa1f8ae2014-08-13 12:31:58 +0530648 /* get the base page size,vsid and segment size */
Aneesh Kumar K.Vfc047952014-08-13 12:32:00 +0530649#ifdef CONFIG_DEBUG_VM
Aneesh Kumar K.Vf1581bf2014-11-02 21:15:27 +0530650 psize = get_slice_psize(mm, addr);
Aneesh Kumar K.Vfc047952014-08-13 12:32:00 +0530651 BUG_ON(psize == MMU_PAGE_16M);
652#endif
Aneesh Kumar K.V945537d2016-04-29 23:25:45 +1000653 if (old_pmd & H_PAGE_COMBO)
Aneesh Kumar K.Vfc047952014-08-13 12:32:00 +0530654 psize = MMU_PAGE_4K;
655 else
656 psize = MMU_PAGE_64K;
657
Aneesh Kumar K.Vf1581bf2014-11-02 21:15:27 +0530658 if (!is_kernel_addr(addr)) {
659 ssize = user_segment_size(addr);
660 vsid = get_vsid(mm->context.id, addr, ssize);
Aneesh Kumar K.Vfa1f8ae2014-08-13 12:31:58 +0530661 WARN_ON(vsid == 0);
662 } else {
Aneesh Kumar K.Vf1581bf2014-11-02 21:15:27 +0530663 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
Aneesh Kumar K.Vfa1f8ae2014-08-13 12:31:58 +0530664 ssize = mmu_kernel_ssize;
665 }
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530666
Aneesh Kumar K.Vd557b092014-11-02 21:15:28 +0530667 tmp = cpumask_of(smp_processor_id());
668 if (cpumask_equal(mm_cpumask(mm), tmp))
Aneesh Kumar K.Vaefa5682014-12-04 11:00:14 +0530669 flags |= HPTE_LOCAL_UPDATE;
Aneesh Kumar K.Vd557b092014-11-02 21:15:28 +0530670
Aneesh Kumar K.Vaefa5682014-12-04 11:00:14 +0530671 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530672}
673
674static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
675{
Aneesh Kumar K.Vf281b5d2015-12-01 09:06:35 +0530676 return __pmd(pmd_val(pmd) | pgprot_val(pgprot));
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530677}
678
679pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
680{
Aneesh Kumar K.Vf281b5d2015-12-01 09:06:35 +0530681 unsigned long pmdv;
Aneesh Kumar K.V6a119ea2015-12-01 09:06:54 +0530682
Aneesh Kumar K.V96270b12016-04-29 23:25:35 +1000683 pmdv = (pfn << PAGE_SHIFT) & PTE_RPN_MASK;
Aneesh Kumar K.Vf281b5d2015-12-01 09:06:35 +0530684 return pmd_set_protbits(__pmd(pmdv), pgprot);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530685}
686
687pmd_t mk_pmd(struct page *page, pgprot_t pgprot)
688{
689 return pfn_pmd(page_to_pfn(page), pgprot);
690}
691
692pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
693{
Aneesh Kumar K.Vf281b5d2015-12-01 09:06:35 +0530694 unsigned long pmdv;
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530695
Aneesh Kumar K.Vf281b5d2015-12-01 09:06:35 +0530696 pmdv = pmd_val(pmd);
697 pmdv &= _HPAGE_CHG_MASK;
698 return pmd_set_protbits(__pmd(pmdv), newprot);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530699}
700
701/*
702 * This is called at the end of handling a user page fault, when the
703 * fault has been handled by updating a HUGE PMD entry in the linux page tables.
704 * We use it to preload an HPTE into the hash table corresponding to
705 * the updated linux HUGE PMD entry.
706 */
707void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
708 pmd_t *pmd)
709{
710 return;
711}
712
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -0700713pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
714 unsigned long addr, pmd_t *pmdp)
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530715{
716 pmd_t old_pmd;
717 pgtable_t pgtable;
718 unsigned long old;
719 pgtable_t *pgtable_slot;
720
Aneesh Kumar K.V88247e82014-02-12 09:13:36 +0530721 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530722 old_pmd = __pmd(old);
723 /*
724 * We have pmd == none and we are holding page_table_lock.
725 * So we can safely go and clear the pgtable hash
726 * index info.
727 */
728 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
729 pgtable = *pgtable_slot;
730 /*
731 * Let's zero out old valid and hash index details
732 * hash fault look at them.
733 */
734 memset(pgtable, 0, PTE_FRAG_SIZE);
Aneesh Kumar K.V13bd8172015-05-11 11:56:01 +0530735 /*
736 * Serialize against find_linux_pte_or_hugepte which does lock-less
737 * lookup in page tables with local interrupts disabled. For huge pages
738 * it casts pmd_t to pte_t. Since format of pte_t is different from
739 * pmd_t we want to prevent transit from pmd pointing to page table
740 * to pmd pointing to huge page (and back) while interrupts are disabled.
741 * We clear pmd to possibly replace it with page table pointer in
742 * different code paths. So make sure we wait for the parallel
743 * find_linux_pte_or_hugepage to finish.
744 */
745 kick_all_cpus_sync();
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530746 return old_pmd;
747}
Aneesh Kumar K.V437d4962013-06-20 14:30:26 +0530748
749int has_transparent_hugepage(void)
750{
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530751
Aneesh Kumar K.V437d4962013-06-20 14:30:26 +0530752 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
753 return 0;
754 /*
755 * We support THP only if PMD_SIZE is 16MB.
756 */
757 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
758 return 0;
759 /*
760 * We need to make sure that we support 16MB hugepage in a segement
761 * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
762 * of 64K.
763 */
764 /*
765 * If we have 64K HPTE, we will be using that by default
766 */
767 if (mmu_psize_defs[MMU_PAGE_64K].shift &&
768 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
769 return 0;
770 /*
771 * Ok we only have 4K HPTE
772 */
773 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
774 return 0;
775
776 return 1;
777}
Aneesh Kumar K.V074c2ea2013-06-20 14:30:15 +0530778#endif /* CONFIG_TRANSPARENT_HUGEPAGE */