Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef _ASM_PGALLOC_H |
| 2 | #define _ASM_PGALLOC_H |
| 3 | |
| 4 | #include <linux/gfp.h> |
| 5 | #include <linux/mm.h> |
| 6 | #include <linux/threads.h> |
| 7 | #include <asm/processor.h> |
| 8 | #include <asm/fixmap.h> |
| 9 | |
| 10 | #include <asm/cache.h> |
| 11 | |
| 12 | /* Allocate the top level pgd (page directory) |
| 13 | * |
| 14 | * Here (for 64 bit kernels) we implement a Hybrid L2/L3 scheme: we |
| 15 | * allocate the first pmd adjacent to the pgd. This means that we can |
| 16 | * subtract a constant offset to get to it. The pmd and pgd sizes are |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 17 | * arranged so that a single pmd covers 4GB (giving a full 64-bit |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | * process access to 8TB) so our lookups are effectively L2 for the |
| 19 | * first 4GB of the kernel (i.e. for all ILP32 processes and all the |
| 20 | * kernel for machines with under 4GB of memory) */ |
| 21 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
| 22 | { |
| 23 | pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, |
| 24 | PGD_ALLOC_ORDER); |
| 25 | pgd_t *actual_pgd = pgd; |
| 26 | |
| 27 | if (likely(pgd != NULL)) { |
| 28 | memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 29 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | actual_pgd += PTRS_PER_PGD; |
| 31 | /* Populate first pmd with allocated memory. We mark it |
| 32 | * with PxD_FLAG_ATTACHED as a signal to the system that this |
| 33 | * pmd entry may not be cleared. */ |
| 34 | __pgd_val_set(*actual_pgd, (PxD_FLAG_PRESENT | |
| 35 | PxD_FLAG_VALID | |
| 36 | PxD_FLAG_ATTACHED) |
| 37 | + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); |
| 38 | /* The first pmd entry also is marked with _PAGE_GATEWAY as |
| 39 | * a signal that this pmd may not be freed */ |
| 40 | __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); |
| 41 | #endif |
| 42 | } |
| 43 | return actual_pgd; |
| 44 | } |
| 45 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 46 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | { |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 48 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | pgd -= PTRS_PER_PGD; |
| 50 | #endif |
| 51 | free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); |
| 52 | } |
| 53 | |
| 54 | #if PT_NLEVELS == 3 |
| 55 | |
| 56 | /* Three Level Page Table Support for pmd's */ |
| 57 | |
| 58 | static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) |
| 59 | { |
| 60 | __pgd_val_set(*pgd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) + |
| 61 | (__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT)); |
| 62 | } |
| 63 | |
| 64 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) |
| 65 | { |
| 66 | pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, |
| 67 | PMD_ORDER); |
| 68 | if (pmd) |
| 69 | memset(pmd, 0, PAGE_SIZE<<PMD_ORDER); |
| 70 | return pmd; |
| 71 | } |
| 72 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 73 | static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | { |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 75 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 76 | if(pmd_flag(*pmd) & PxD_FLAG_ATTACHED) |
Mikulas Patocka | 0e0da48 | 2015-03-18 13:42:38 -0400 | [diff] [blame^] | 77 | /* |
| 78 | * This is the permanent pmd attached to the pgd; |
| 79 | * cannot free it. |
| 80 | * Increment the counter to compensate for the decrement |
| 81 | * done by generic mm code. |
| 82 | */ |
| 83 | mm_inc_nr_pmds(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | return; |
| 85 | #endif |
| 86 | free_pages((unsigned long)pmd, PMD_ORDER); |
| 87 | } |
| 88 | |
| 89 | #else |
| 90 | |
| 91 | /* Two Level Page Table Support for pmd's */ |
| 92 | |
| 93 | /* |
| 94 | * allocating and freeing a pmd is trivial: the 1-entry pmd is |
| 95 | * inside the pgd, so has no extra memory associated with it. |
| 96 | */ |
| 97 | |
| 98 | #define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 99 | #define pmd_free(mm, x) do { } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | #define pgd_populate(mm, pmd, pte) BUG() |
| 101 | |
| 102 | #endif |
| 103 | |
| 104 | static inline void |
| 105 | pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) |
| 106 | { |
Helge Deller | 513e7ec | 2007-01-28 15:09:20 +0100 | [diff] [blame] | 107 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | /* preserve the gateway marker if this is the beginning of |
| 109 | * the permanent pmd */ |
| 110 | if(pmd_flag(*pmd) & PxD_FLAG_ATTACHED) |
| 111 | __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | |
| 112 | PxD_FLAG_VALID | |
| 113 | PxD_FLAG_ATTACHED) |
| 114 | + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); |
| 115 | else |
| 116 | #endif |
| 117 | __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) |
| 118 | + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); |
| 119 | } |
| 120 | |
| 121 | #define pmd_populate(mm, pmd, pte_page) \ |
| 122 | pmd_populate_kernel(mm, pmd, page_address(pte_page)) |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 123 | #define pmd_pgtable(pmd) pmd_page(pmd) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 125 | static inline pgtable_t |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | pte_alloc_one(struct mm_struct *mm, unsigned long address) |
| 127 | { |
| 128 | struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); |
Kirill A. Shutemov | bc16640 | 2013-11-14 14:31:37 -0800 | [diff] [blame] | 129 | if (!page) |
| 130 | return NULL; |
| 131 | if (!pgtable_page_ctor(page)) { |
| 132 | __free_page(page); |
| 133 | return NULL; |
| 134 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 135 | return page; |
| 136 | } |
| 137 | |
| 138 | static inline pte_t * |
| 139 | pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) |
| 140 | { |
| 141 | pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); |
| 142 | return pte; |
| 143 | } |
| 144 | |
Benjamin Herrenschmidt | 5e54197 | 2008-02-04 22:29:14 -0800 | [diff] [blame] | 145 | static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | { |
| 147 | free_page((unsigned long)pte); |
| 148 | } |
| 149 | |
Kyle McMartin | 9aa150b | 2008-02-18 14:16:26 -0800 | [diff] [blame] | 150 | static inline void pte_free(struct mm_struct *mm, struct page *pte) |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 151 | { |
| 152 | pgtable_page_dtor(pte); |
Kyle McMartin | 9aa150b | 2008-02-18 14:16:26 -0800 | [diff] [blame] | 153 | pte_free_kernel(mm, page_address(pte)); |
Martin Schwidefsky | 2f569af | 2008-02-08 04:22:04 -0800 | [diff] [blame] | 154 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | #define check_pgt_cache() do { } while (0) |
| 157 | |
| 158 | #endif |