Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * S390 version |
Heiko Carstens | a53c8fa | 2012-07-20 11:15:04 +0200 | [diff] [blame] | 3 | * Copyright IBM Corp. 1999, 2000 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * Author(s): Hartmut Penner (hp@de.ibm.com) |
| 5 | * Martin Schwidefsky (schwidefsky@de.ibm.com) |
| 6 | * |
| 7 | * Derived from "include/asm-i386/pgalloc.h" |
| 8 | * Copyright (C) 1994 Linus Torvalds |
| 9 | */ |
| 10 | |
| 11 | #ifndef _S390_PGALLOC_H |
| 12 | #define _S390_PGALLOC_H |
| 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/threads.h> |
| 15 | #include <linux/gfp.h> |
| 16 | #include <linux/mm.h> |
| 17 | |
Martin Schwidefsky | 043d070 | 2011-05-23 10:24:23 +0200 | [diff] [blame] | 18 | unsigned long *crst_table_alloc(struct mm_struct *); |
Martin Schwidefsky | 146e4b3 | 2008-02-09 18:24:35 +0100 | [diff] [blame] | 19 | void crst_table_free(struct mm_struct *, unsigned long *); |
Gerald Schaefer | 9282ed9 | 2006-09-20 15:59:37 +0200 | [diff] [blame] | 20 | |
Martin Schwidefsky | 527e30b | 2014-04-30 16:04:25 +0200 | [diff] [blame] | 21 | unsigned long *page_table_alloc(struct mm_struct *); |
Martin Schwidefsky | 146e4b3 | 2008-02-09 18:24:35 +0100 | [diff] [blame] | 22 | void page_table_free(struct mm_struct *, unsigned long *); |
Martin Schwidefsky | 527e30b | 2014-04-30 16:04:25 +0200 | [diff] [blame] | 23 | void page_table_free_rcu(struct mmu_gather *, unsigned long *, unsigned long); |
Martin Schwidefsky | 0b46e0a | 2015-04-15 13:23:26 +0200 | [diff] [blame] | 24 | extern int page_table_allocate_pgste; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 26 | static inline void clear_table(unsigned long *s, unsigned long val, size_t n) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | { |
Martin Schwidefsky | 4f7e90d | 2008-12-25 13:39:26 +0100 | [diff] [blame] | 28 | typedef struct { char _[n]; } addrtype; |
| 29 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 30 | *s = val; |
| 31 | n = (n / 256) - 1; |
| 32 | asm volatile( |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 33 | " mvc 8(248,%0),0(%0)\n" |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 34 | "0: mvc 256(256,%0),0(%0)\n" |
| 35 | " la %0,256(%0)\n" |
| 36 | " brct %1,0b\n" |
Martin Schwidefsky | 4f7e90d | 2008-12-25 13:39:26 +0100 | [diff] [blame] | 37 | : "+a" (s), "+d" (n), "=m" (*(addrtype *) s) |
| 38 | : "m" (*(addrtype *) s)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | } |
| 40 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 41 | static inline void crst_table_init(unsigned long *crst, unsigned long entry) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | { |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 43 | clear_table(crst, entry, sizeof(unsigned long)*2048); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | } |
| 45 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 46 | static inline unsigned long pgd_entry_type(struct mm_struct *mm) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | { |
Martin Schwidefsky | 6252d70 | 2008-02-09 18:24:37 +0100 | [diff] [blame] | 48 | if (mm->context.asce_limit <= (1UL << 31)) |
| 49 | return _SEGMENT_ENTRY_EMPTY; |
| 50 | if (mm->context.asce_limit <= (1UL << 42)) |
| 51 | return _REGION3_ENTRY_EMPTY; |
Martin Schwidefsky | 5a216a2 | 2008-02-09 18:24:36 +0100 | [diff] [blame] | 52 | return _REGION2_ENTRY_EMPTY; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 53 | } |
| 54 | |
Gerald Schaefer | 723cacb | 2016-04-15 16:38:40 +0200 | [diff] [blame] | 55 | int crst_table_upgrade(struct mm_struct *); |
| 56 | void crst_table_downgrade(struct mm_struct *); |
Martin Schwidefsky | 6252d70 | 2008-02-09 18:24:37 +0100 | [diff] [blame] | 57 | |
Martin Schwidefsky | 5a216a2 | 2008-02-09 18:24:36 +0100 | [diff] [blame] | 58 | static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) |
| 59 | { |
Martin Schwidefsky | 043d070 | 2011-05-23 10:24:23 +0200 | [diff] [blame] | 60 | unsigned long *table = crst_table_alloc(mm); |
Martin Schwidefsky | 5a216a2 | 2008-02-09 18:24:36 +0100 | [diff] [blame] | 61 | if (table) |
| 62 | crst_table_init(table, _REGION3_ENTRY_EMPTY); |
| 63 | return (pud_t *) table; |
| 64 | } |
| 65 | #define pud_free(mm, pud) crst_table_free(mm, (unsigned long *) pud) |
Martin Schwidefsky | 190a1d7 | 2007-10-22 12:52:48 +0200 | [diff] [blame] | 66 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 67 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 68 | { |
Martin Schwidefsky | 043d070 | 2011-05-23 10:24:23 +0200 | [diff] [blame] | 69 | unsigned long *table = crst_table_alloc(mm); |
Martin Schwidefsky | ec66ad6 | 2014-02-12 14:16:18 +0100 | [diff] [blame] | 70 | |
| 71 | if (!table) |
| 72 | return NULL; |
| 73 | crst_table_init(table, _SEGMENT_ENTRY_EMPTY); |
| 74 | if (!pgtable_pmd_page_ctor(virt_to_page(table))) { |
| 75 | crst_table_free(mm, table); |
| 76 | return NULL; |
| 77 | } |
Martin Schwidefsky | 146e4b3 | 2008-02-09 18:24:35 +0100 | [diff] [blame] | 78 | return (pmd_t *) table; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | } |
Martin Schwidefsky | ec66ad6 | 2014-02-12 14:16:18 +0100 | [diff] [blame] | 80 | |
| 81 | static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) |
| 82 | { |
| 83 | pgtable_pmd_page_dtor(virt_to_page(pmd)); |
| 84 | crst_table_free(mm, (unsigned long *) pmd); |
| 85 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 86 | |
Martin Schwidefsky | b2fa47e | 2011-05-23 10:24:40 +0200 | [diff] [blame] | 87 | static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) |
Martin Schwidefsky | 5a216a2 | 2008-02-09 18:24:36 +0100 | [diff] [blame] | 88 | { |
| 89 | pgd_val(*pgd) = _REGION2_ENTRY | __pa(pud); |
| 90 | } |
| 91 | |
Martin Schwidefsky | 190a1d7 | 2007-10-22 12:52:48 +0200 | [diff] [blame] | 92 | static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) |
Gerald Schaefer | c1821c2 | 2007-02-05 21:18:17 +0100 | [diff] [blame] | 93 | { |
Martin Schwidefsky | b2fa47e | 2011-05-23 10:24:40 +0200 | [diff] [blame] | 94 | pud_val(*pud) = _REGION3_ENTRY | __pa(pmd); |
Gerald Schaefer | c1821c2 | 2007-02-05 21:18:17 +0100 | [diff] [blame] | 95 | } |
| 96 | |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 97 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
| 98 | { |
Martin Schwidefsky | 3446c13 | 2016-02-15 14:46:49 +0100 | [diff] [blame] | 99 | unsigned long *table = crst_table_alloc(mm); |
| 100 | |
| 101 | if (!table) |
| 102 | return NULL; |
| 103 | if (mm->context.asce_limit == (1UL << 31)) { |
| 104 | /* Forking a compat process with 2 page table levels */ |
| 105 | if (!pgtable_pmd_page_ctor(virt_to_page(table))) { |
| 106 | crst_table_free(mm, table); |
| 107 | return NULL; |
| 108 | } |
| 109 | } |
| 110 | return (pgd_t *) table; |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 111 | } |
Martin Schwidefsky | 3446c13 | 2016-02-15 14:46:49 +0100 | [diff] [blame] | 112 | |
| 113 | static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
| 114 | { |
| 115 | if (mm->context.asce_limit == (1UL << 31)) |
| 116 | pgtable_pmd_page_dtor(virt_to_page(pgd)); |
| 117 | crst_table_free(mm, (unsigned long *) pgd); |
| 118 | } |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 119 | |
Martin Schwidefsky | b2fa47e | 2011-05-23 10:24:40 +0200 | [diff] [blame] | 120 | static inline void pmd_populate(struct mm_struct *mm, |
| 121 | pmd_t *pmd, pgtable_t pte) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | { |
Martin Schwidefsky | 3610cce | 2007-10-22 12:52:47 +0200 | [diff] [blame] | 123 | pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | } |
| 125 | |
Martin Schwidefsky | b2fa47e | 2011-05-23 10:24:40 +0200 | [diff] [blame] | 126 | #define pmd_populate_kernel(mm, pmd, pte) pmd_populate(mm, pmd, pte) |
Martin Schwidefsky | 146e4b3 | 2008-02-09 18:24:35 +0100 | [diff] [blame] | 127 | |
| 128 | #define pmd_pgtable(pmd) \ |
| 129 | (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | |
| 131 | /* |
| 132 | * page table entry allocation/free routines. |
| 133 | */ |
Martin Schwidefsky | 527e30b | 2014-04-30 16:04:25 +0200 | [diff] [blame] | 134 | #define pte_alloc_one_kernel(mm, vmaddr) ((pte_t *) page_table_alloc(mm)) |
| 135 | #define pte_alloc_one(mm, vmaddr) ((pte_t *) page_table_alloc(mm)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | |
Martin Schwidefsky | 146e4b3 | 2008-02-09 18:24:35 +0100 | [diff] [blame] | 137 | #define pte_free_kernel(mm, pte) page_table_free(mm, (unsigned long *) pte) |
| 138 | #define pte_free(mm, pte) page_table_free(mm, (unsigned long *) pte) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | |
Martin Schwidefsky | 8021714 | 2010-10-25 16:10:11 +0200 | [diff] [blame] | 140 | extern void rcu_table_freelist_finish(void); |
| 141 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | #endif /* _S390_PGALLOC_H */ |