blob: 4a64c0e5428fdc4956d495d3702365469e1cbd0e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * S390 version
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02003 * Copyright IBM Corp. 1999, 2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Author(s): Hartmut Penner (hp@de.ibm.com)
5 * Ulrich Weigand (weigand@de.ibm.com)
6 * Martin Schwidefsky (schwidefsky@de.ibm.com)
7 *
8 * Derived from "include/asm-i386/pgtable.h"
9 */
10
11#ifndef _ASM_S390_PGTABLE_H
12#define _ASM_S390_PGTABLE_H
13
Linus Torvalds1da177e2005-04-16 15:20:36 -070014/*
15 * The Linux memory management assumes a three-level page table setup. For
16 * s390 31 bit we "fold" the mid level into the top-level page table, so
17 * that we physically have the same two-level page table as the s390 mmu
18 * expects in 31 bit mode. For s390 64 bit we use three of the five levels
19 * the hardware provides (region first and region second tables are not
20 * used).
21 *
22 * The "pgd_xxx()" functions are trivial for a folded two-level
23 * setup: the pgd is never bad, and a pmd always exists (as it's folded
24 * into the pgd entry)
25 *
26 * This file contains the functions and defines necessary to modify and use
27 * the S390 page table tree.
28 */
29#ifndef __ASSEMBLY__
Heiko Carstens9789db02008-07-14 09:59:11 +020030#include <linux/sched.h>
Heiko Carstens2dcea572006-09-29 01:58:41 -070031#include <linux/mm_types.h>
Martin Schwidefskyabf09be2012-11-07 13:17:37 +010032#include <linux/page-flags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <asm/bug.h>
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +020034#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
Linus Torvalds1da177e2005-04-16 15:20:36 -070036extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
37extern void paging_init(void);
Heiko Carstens2b67fc42007-02-05 21:16:47 +010038extern void vmem_map_init(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
40/*
41 * The S390 doesn't have any external MMU info: the kernel page
42 * tables contain all the necessary information.
43 */
Russell King4b3073e2009-12-18 16:40:18 +000044#define update_mmu_cache(vma, address, ptep) do { } while (0)
David Millerb113da62012-10-08 16:34:25 -070045#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
47/*
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020048 * ZERO_PAGE is a global shared page that is always zero; used
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 * for zero-mapped memory areas etc..
50 */
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020051
52extern unsigned long empty_zero_page;
53extern unsigned long zero_page_mask;
54
55#define ZERO_PAGE(vaddr) \
56 (virt_to_page((void *)(empty_zero_page + \
57 (((unsigned long)(vaddr)) &zero_page_mask))))
Kirill A. Shutemov816422a2012-12-12 13:52:36 -080058#define __HAVE_COLOR_ZERO_PAGE
Martin Schwidefsky238ec4e2010-10-25 16:10:07 +020059
Linus Torvalds4f2e2902013-04-17 08:46:19 -070060/* TODO: s390 cannot support io_remap_pfn_range... */
61#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \
62 remap_pfn_range(vma, vaddr, pfn, size, prot)
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064#endif /* !__ASSEMBLY__ */
65
66/*
67 * PMD_SHIFT determines the size of the area a second-level page
68 * table can map
69 * PGDIR_SHIFT determines what a third-level page table entry can map
70 */
Heiko Carstensf4815ac2012-05-23 16:24:51 +020071#ifndef CONFIG_64BIT
Martin Schwidefsky146e4b32008-02-09 18:24:35 +010072# define PMD_SHIFT 20
73# define PUD_SHIFT 20
74# define PGDIR_SHIFT 20
Heiko Carstensf4815ac2012-05-23 16:24:51 +020075#else /* CONFIG_64BIT */
Martin Schwidefsky146e4b32008-02-09 18:24:35 +010076# define PMD_SHIFT 20
Martin Schwidefsky190a1d72007-10-22 12:52:48 +020077# define PUD_SHIFT 31
Martin Schwidefsky5a216a22008-02-09 18:24:36 +010078# define PGDIR_SHIFT 42
Heiko Carstensf4815ac2012-05-23 16:24:51 +020079#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
81#define PMD_SIZE (1UL << PMD_SHIFT)
82#define PMD_MASK (~(PMD_SIZE-1))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +020083#define PUD_SIZE (1UL << PUD_SHIFT)
84#define PUD_MASK (~(PUD_SIZE-1))
Martin Schwidefsky5a216a22008-02-09 18:24:36 +010085#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
86#define PGDIR_MASK (~(PGDIR_SIZE-1))
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
88/*
89 * entries per page directory level: the S390 is two-level, so
90 * we don't really have any PMD directory physically.
91 * for S390 segment-table entries are combined to one PGD
92 * that leads to 1024 pte per pgd
93 */
Martin Schwidefsky146e4b32008-02-09 18:24:35 +010094#define PTRS_PER_PTE 256
Heiko Carstensf4815ac2012-05-23 16:24:51 +020095#ifndef CONFIG_64BIT
Martin Schwidefsky146e4b32008-02-09 18:24:35 +010096#define PTRS_PER_PMD 1
Martin Schwidefsky5a216a22008-02-09 18:24:36 +010097#define PTRS_PER_PUD 1
Heiko Carstensf4815ac2012-05-23 16:24:51 +020098#else /* CONFIG_64BIT */
Martin Schwidefsky146e4b32008-02-09 18:24:35 +010099#define PTRS_PER_PMD 2048
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100100#define PTRS_PER_PUD 2048
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200101#endif /* CONFIG_64BIT */
Martin Schwidefsky146e4b32008-02-09 18:24:35 +0100102#define PTRS_PER_PGD 2048
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Hugh Dickinsd455a362005-04-19 13:29:23 -0700104#define FIRST_USER_ADDRESS 0
105
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#define pte_ERROR(e) \
107 printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
108#define pmd_ERROR(e) \
109 printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200110#define pud_ERROR(e) \
111 printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#define pgd_ERROR(e) \
113 printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
114
115#ifndef __ASSEMBLY__
116/*
Heiko Carstensc972cc62012-10-05 16:52:18 +0200117 * The vmalloc and module area will always be on the topmost area of the kernel
118 * mapping. We reserve 96MB (31bit) / 128GB (64bit) for vmalloc and modules.
119 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where
120 * modules will reside. That makes sure that inter module branches always
121 * happen without trampolines and in addition the placement within a 2GB frame
122 * is branch prediction unit friendly.
Heiko Carstens8b62bc92006-12-04 15:40:56 +0100123 */
Heiko Carstens239a64252009-06-12 10:26:33 +0200124extern unsigned long VMALLOC_START;
Martin Schwidefsky14045eb2011-12-27 11:27:07 +0100125extern unsigned long VMALLOC_END;
126extern struct page *vmemmap;
Heiko Carstens239a64252009-06-12 10:26:33 +0200127
Martin Schwidefsky14045eb2011-12-27 11:27:07 +0100128#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
Christian Borntraeger5fd9c6e2008-01-26 14:11:00 +0100129
Heiko Carstensc972cc62012-10-05 16:52:18 +0200130#ifdef CONFIG_64BIT
131extern unsigned long MODULES_VADDR;
132extern unsigned long MODULES_END;
133#define MODULES_VADDR MODULES_VADDR
134#define MODULES_END MODULES_END
135#define MODULES_LEN (1UL << 31)
136#endif
137
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138/*
139 * A 31 bit pagetable entry of S390 has following format:
140 * | PFRA | | OS |
141 * 0 0IP0
142 * 00000000001111111111222222222233
143 * 01234567890123456789012345678901
144 *
145 * I Page-Invalid Bit: Page is not available for address-translation
146 * P Page-Protection Bit: Store access not possible for page
147 *
148 * A 31 bit segmenttable entry of S390 has following format:
149 * | P-table origin | |PTL
150 * 0 IC
151 * 00000000001111111111222222222233
152 * 01234567890123456789012345678901
153 *
154 * I Segment-Invalid Bit: Segment is not available for address-translation
155 * C Common-Segment Bit: Segment is not private (PoP 3-30)
156 * PTL Page-Table-Length: Page-table length (PTL+1*16 entries -> up to 256)
157 *
158 * The 31 bit segmenttable origin of S390 has following format:
159 *
160 * |S-table origin | | STL |
161 * X **GPS
162 * 00000000001111111111222222222233
163 * 01234567890123456789012345678901
164 *
165 * X Space-Switch event:
166 * G Segment-Invalid Bit: *
167 * P Private-Space Bit: Segment is not private (PoP 3-30)
168 * S Storage-Alteration:
169 * STL Segment-Table-Length: Segment-table length (STL+1*16 entries -> up to 2048)
170 *
171 * A 64 bit pagetable entry of S390 has following format:
Christian Borntraeger6a985c62009-12-07 12:52:11 +0100172 * | PFRA |0IPC| OS |
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 * 0000000000111111111122222222223333333333444444444455555555556666
174 * 0123456789012345678901234567890123456789012345678901234567890123
175 *
176 * I Page-Invalid Bit: Page is not available for address-translation
177 * P Page-Protection Bit: Store access not possible for page
Christian Borntraeger6a985c62009-12-07 12:52:11 +0100178 * C Change-bit override: HW is not required to set change bit
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 *
180 * A 64 bit segmenttable entry of S390 has following format:
181 * | P-table origin | TT
182 * 0000000000111111111122222222223333333333444444444455555555556666
183 * 0123456789012345678901234567890123456789012345678901234567890123
184 *
185 * I Segment-Invalid Bit: Segment is not available for address-translation
186 * C Common-Segment Bit: Segment is not private (PoP 3-30)
187 * P Page-Protection Bit: Store access not possible for page
188 * TT Type 00
189 *
190 * A 64 bit region table entry of S390 has following format:
191 * | S-table origin | TF TTTL
192 * 0000000000111111111122222222223333333333444444444455555555556666
193 * 0123456789012345678901234567890123456789012345678901234567890123
194 *
195 * I Segment-Invalid Bit: Segment is not available for address-translation
196 * TT Type 01
197 * TF
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200198 * TL Table length
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 *
200 * The 64 bit regiontable origin of S390 has following format:
201 * | region table origon | DTTL
202 * 0000000000111111111122222222223333333333444444444455555555556666
203 * 0123456789012345678901234567890123456789012345678901234567890123
204 *
205 * X Space-Switch event:
206 * G Segment-Invalid Bit:
207 * P Private-Space Bit:
208 * S Storage-Alteration:
209 * R Real space
210 * TL Table-Length:
211 *
212 * A storage key has the following format:
213 * | ACC |F|R|C|0|
214 * 0 3 4 5 6 7
215 * ACC: access key
216 * F : fetch protection bit
217 * R : referenced bit
218 * C : changed bit
219 */
220
221/* Hardware bits in the page table entry */
Christian Borntraeger6a985c62009-12-07 12:52:11 +0100222#define _PAGE_CO 0x100 /* HW Change-bit override */
Martin Schwidefsky83377482006-10-18 18:30:51 +0200223#define _PAGE_RO 0x200 /* HW read-only bit */
224#define _PAGE_INVALID 0x400 /* HW invalid bit */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200225
226/* Software bits in the page table entry */
Martin Schwidefsky83377482006-10-18 18:30:51 +0200227#define _PAGE_SWT 0x001 /* SW pte type bit t */
228#define _PAGE_SWX 0x002 /* SW pte type bit x */
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100229#define _PAGE_SWC 0x004 /* SW pte changed bit */
230#define _PAGE_SWR 0x008 /* SW pte referenced bit */
231#define _PAGE_SWW 0x010 /* SW pte write bit */
232#define _PAGE_SPECIAL 0x020 /* SW associated with special page */
Nick Piggina08cb622008-04-28 02:13:03 -0700233#define __HAVE_ARCH_PTE_SPECIAL
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234
Nick Piggin138c9022008-07-08 11:31:06 +0200235/* Set of bits not changed in pte_modify */
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100236#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \
237 _PAGE_SWC | _PAGE_SWR)
Nick Piggin138c9022008-07-08 11:31:06 +0200238
Martin Schwidefsky83377482006-10-18 18:30:51 +0200239/* Six different types of pages. */
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200240#define _PAGE_TYPE_EMPTY 0x400
241#define _PAGE_TYPE_NONE 0x401
Martin Schwidefsky83377482006-10-18 18:30:51 +0200242#define _PAGE_TYPE_SWAP 0x403
243#define _PAGE_TYPE_FILE 0x601 /* bit 0x002 is used for offset !! */
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200244#define _PAGE_TYPE_RO 0x200
245#define _PAGE_TYPE_RW 0x000
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246
Martin Schwidefsky83377482006-10-18 18:30:51 +0200247/*
Gerald Schaefer53492b12008-04-30 13:38:46 +0200248 * Only four types for huge pages, using the invalid bit and protection bit
249 * of a segment table entry.
250 */
251#define _HPAGE_TYPE_EMPTY 0x020 /* _SEGMENT_ENTRY_INV */
252#define _HPAGE_TYPE_NONE 0x220
253#define _HPAGE_TYPE_RO 0x200 /* _SEGMENT_ENTRY_RO */
254#define _HPAGE_TYPE_RW 0x000
255
256/*
Martin Schwidefsky83377482006-10-18 18:30:51 +0200257 * PTE type bits are rather complicated. handle_pte_fault uses pte_present,
258 * pte_none and pte_file to find out the pte type WITHOUT holding the page
259 * table lock. ptep_clear_flush on the other hand uses ptep_clear_flush to
260 * invalidate a given pte. ipte sets the hw invalid bit and clears all tlbs
261 * for the page. The page table entry is set to _PAGE_TYPE_EMPTY afterwards.
262 * This change is done while holding the lock, but the intermediate step
263 * of a previously valid pte with the hw invalid bit set can be observed by
264 * handle_pte_fault. That makes it necessary that all valid pte types with
265 * the hw invalid bit set must be distinguishable from the four pte types
266 * empty, none, swap and file.
267 *
268 * irxt ipte irxt
269 * _PAGE_TYPE_EMPTY 1000 -> 1000
270 * _PAGE_TYPE_NONE 1001 -> 1001
271 * _PAGE_TYPE_SWAP 1011 -> 1011
272 * _PAGE_TYPE_FILE 11?1 -> 11?1
273 * _PAGE_TYPE_RO 0100 -> 1100
274 * _PAGE_TYPE_RW 0000 -> 1000
275 *
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100276 * pte_none is true for bits combinations 1000, 1010, 1100, 1110
Martin Schwidefsky83377482006-10-18 18:30:51 +0200277 * pte_present is true for bits combinations 0000, 0010, 0100, 0110, 1001
278 * pte_file is true for bits combinations 1101, 1111
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100279 * swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid.
Martin Schwidefsky83377482006-10-18 18:30:51 +0200280 */
281
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200282#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200284/* Bits in the segment table address-space-control-element */
285#define _ASCE_SPACE_SWITCH 0x80000000UL /* space switch event */
286#define _ASCE_ORIGIN_MASK 0x7ffff000UL /* segment table origin */
287#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
288#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
289#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */
290
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291/* Bits in the segment table entry */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200292#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */
Martin Schwidefsky80217142010-10-25 16:10:11 +0200293#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200294#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */
295#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */
296#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */
297
298#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL)
299#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
300
Martin Schwidefsky6c61cfe2011-06-06 14:14:42 +0200301/* Page status table bits for virtualization */
302#define RCP_ACC_BITS 0xf0000000UL
303#define RCP_FP_BIT 0x08000000UL
304#define RCP_PCL_BIT 0x00800000UL
305#define RCP_HR_BIT 0x00400000UL
306#define RCP_HC_BIT 0x00200000UL
307#define RCP_GR_BIT 0x00040000UL
308#define RCP_GC_BIT 0x00020000UL
309
310/* User dirty / referenced bit for KVM's migration feature */
311#define KVM_UR_BIT 0x00008000UL
312#define KVM_UC_BIT 0x00004000UL
313
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200314#else /* CONFIG_64BIT */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200315
316/* Bits in the segment/region table address-space-control-element */
317#define _ASCE_ORIGIN ~0xfffUL/* segment table origin */
318#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
319#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
320#define _ASCE_SPACE_SWITCH 0x40 /* space switch event */
321#define _ASCE_REAL_SPACE 0x20 /* real space control */
322#define _ASCE_TYPE_MASK 0x0c /* asce table type mask */
323#define _ASCE_TYPE_REGION1 0x0c /* region first table type */
324#define _ASCE_TYPE_REGION2 0x08 /* region second table type */
325#define _ASCE_TYPE_REGION3 0x04 /* region third table type */
326#define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */
327#define _ASCE_TABLE_LENGTH 0x03 /* region table length */
328
329/* Bits in the region table entry */
330#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100331#define _REGION_ENTRY_RO 0x200 /* region protection bit */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200332#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */
333#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */
334#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
335#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */
336#define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */
337#define _REGION_ENTRY_LENGTH 0x03 /* region third length */
338
339#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
340#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV)
341#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
342#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV)
343#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
344#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV)
345
Heiko Carstens18da2362012-10-08 09:18:26 +0200346#define _REGION3_ENTRY_LARGE 0x400 /* RTTE-format control, large page */
Heiko Carstens1819ed12013-02-16 11:47:27 +0100347#define _REGION3_ENTRY_RO 0x200 /* page protection bit */
348#define _REGION3_ENTRY_CO 0x100 /* change-recording override */
Heiko Carstens18da2362012-10-08 09:18:26 +0200349
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200350/* Bits in the segment table entry */
Heiko Carstensea815312013-03-21 12:50:39 +0100351#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200352#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */
353#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */
354#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */
355
356#define _SEGMENT_ENTRY (0)
357#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
358
Gerald Schaefer53492b12008-04-30 13:38:46 +0200359#define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */
360#define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */
Gerald Schaefer75077af2012-10-08 16:30:15 -0700361#define _SEGMENT_ENTRY_SPLIT_BIT 0 /* THP splitting bit number */
362#define _SEGMENT_ENTRY_SPLIT (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
Gerald Schaefer53492b12008-04-30 13:38:46 +0200363
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -0700364/* Set of bits not changed in pmd_modify */
365#define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
366 | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
367
Martin Schwidefsky6c61cfe2011-06-06 14:14:42 +0200368/* Page status table bits for virtualization */
369#define RCP_ACC_BITS 0xf000000000000000UL
370#define RCP_FP_BIT 0x0800000000000000UL
371#define RCP_PCL_BIT 0x0080000000000000UL
372#define RCP_HR_BIT 0x0040000000000000UL
373#define RCP_HC_BIT 0x0020000000000000UL
374#define RCP_GR_BIT 0x0004000000000000UL
375#define RCP_GC_BIT 0x0002000000000000UL
376
377/* User dirty / referenced bit for KVM's migration feature */
378#define KVM_UR_BIT 0x0000800000000000UL
379#define KVM_UC_BIT 0x0000400000000000UL
380
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200381#endif /* CONFIG_64BIT */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200382
383/*
384 * A user page table pointer has the space-switch-event bit, the
385 * private-space-control bit and the storage-alteration-event-control
386 * bit set. A kernel page table pointer doesn't need them.
387 */
388#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
389 _ASCE_ALT_EVENT)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391/*
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200392 * Page protection definitions.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 */
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200394#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE)
395#define PAGE_RO __pgprot(_PAGE_TYPE_RO)
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100396#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW)
397#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC)
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200398
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100399#define PAGE_KERNEL PAGE_RWC
Heiko Carstensbddb7ae2013-01-30 16:38:55 +0100400#define PAGE_SHARED PAGE_KERNEL
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200401#define PAGE_COPY PAGE_RO
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
403/*
Martin Schwidefsky043d0702011-05-23 10:24:23 +0200404 * On s390 the page table entry has an invalid bit and a read-only bit.
405 * Read permission implies execute permission and write permission
406 * implies read permission.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 */
408 /*xwr*/
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200409#define __P000 PAGE_NONE
410#define __P001 PAGE_RO
411#define __P010 PAGE_RO
412#define __P011 PAGE_RO
Martin Schwidefsky043d0702011-05-23 10:24:23 +0200413#define __P100 PAGE_RO
414#define __P101 PAGE_RO
415#define __P110 PAGE_RO
416#define __P111 PAGE_RO
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200418#define __S000 PAGE_NONE
419#define __S001 PAGE_RO
420#define __S010 PAGE_RW
421#define __S011 PAGE_RW
Martin Schwidefsky043d0702011-05-23 10:24:23 +0200422#define __S100 PAGE_RO
423#define __S101 PAGE_RO
424#define __S110 PAGE_RW
425#define __S111 PAGE_RW
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200427static inline int mm_exclusive(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200429 return likely(mm == current->active_mm &&
430 atomic_read(&mm->context.attach_count) <= 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200433static inline int mm_has_pgste(struct mm_struct *mm)
434{
435#ifdef CONFIG_PGSTE
436 if (unlikely(mm->context.has_pgste))
437 return 1;
438#endif
439 return 0;
440}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441/*
442 * pgd/pmd/pte query functions
443 */
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200444#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800446static inline int pgd_present(pgd_t pgd) { return 1; }
447static inline int pgd_none(pgd_t pgd) { return 0; }
448static inline int pgd_bad(pgd_t pgd) { return 0; }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200450static inline int pud_present(pud_t pud) { return 1; }
451static inline int pud_none(pud_t pud) { return 0; }
Heiko Carstens18da2362012-10-08 09:18:26 +0200452static inline int pud_large(pud_t pud) { return 0; }
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200453static inline int pud_bad(pud_t pud) { return 0; }
454
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200455#else /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100457static inline int pgd_present(pgd_t pgd)
458{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100459 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
460 return 1;
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100461 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
462}
463
464static inline int pgd_none(pgd_t pgd)
465{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100466 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
467 return 0;
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100468 return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL;
469}
470
471static inline int pgd_bad(pgd_t pgd)
472{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100473 /*
474 * With dynamic page table levels the pgd can be a region table
475 * entry or a segment table entry. Check for the bit that are
476 * invalid for either table entry.
477 */
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100478 unsigned long mask =
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100479 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100480 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
481 return (pgd_val(pgd) & mask) != 0;
482}
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200483
484static inline int pud_present(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100486 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
487 return 1;
Martin Schwidefsky0d017922007-12-17 16:25:48 +0100488 return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489}
490
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200491static inline int pud_none(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100493 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
494 return 0;
Martin Schwidefsky0d017922007-12-17 16:25:48 +0100495 return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496}
497
Heiko Carstens18da2362012-10-08 09:18:26 +0200498static inline int pud_large(pud_t pud)
499{
500 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
501 return 0;
502 return !!(pud_val(pud) & _REGION3_ENTRY_LARGE);
503}
504
Martin Schwidefsky190a1d72007-10-22 12:52:48 +0200505static inline int pud_bad(pud_t pud)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506{
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100507 /*
508 * With dynamic page table levels the pud can be a region table
509 * entry or a segment table entry. Check for the bit that are
510 * invalid for either table entry.
511 */
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100512 unsigned long mask =
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100513 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100514 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
515 return (pud_val(pud) & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516}
517
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200518#endif /* CONFIG_64BIT */
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200519
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800520static inline int pmd_present(pmd_t pmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521{
Gerald Schaeferd8e7a332012-10-25 17:42:50 +0200522 unsigned long mask = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO;
523 return (pmd_val(pmd) & mask) == _HPAGE_TYPE_NONE ||
524 !(pmd_val(pmd) & _SEGMENT_ENTRY_INV);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525}
526
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800527static inline int pmd_none(pmd_t pmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
Gerald Schaeferd8e7a332012-10-25 17:42:50 +0200529 return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) &&
530 !(pmd_val(pmd) & _SEGMENT_ENTRY_RO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531}
532
Heiko Carstens378b1e72012-10-01 12:58:34 +0200533static inline int pmd_large(pmd_t pmd)
534{
535#ifdef CONFIG_64BIT
536 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
537#else
538 return 0;
539#endif
540}
541
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800542static inline int pmd_bad(pmd_t pmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543{
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200544 unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV;
545 return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546}
547
Gerald Schaefer75077af2012-10-08 16:30:15 -0700548#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
549extern void pmdp_splitting_flush(struct vm_area_struct *vma,
550 unsigned long addr, pmd_t *pmdp);
551
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -0700552#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
553extern int pmdp_set_access_flags(struct vm_area_struct *vma,
554 unsigned long address, pmd_t *pmdp,
555 pmd_t entry, int dirty);
556
557#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
558extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
559 unsigned long address, pmd_t *pmdp);
560
561#define __HAVE_ARCH_PMD_WRITE
562static inline int pmd_write(pmd_t pmd)
563{
564 return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
565}
566
567static inline int pmd_young(pmd_t pmd)
568{
569 return 0;
570}
571
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800572static inline int pte_none(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573{
Martin Schwidefsky83377482006-10-18 18:30:51 +0200574 return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575}
576
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800577static inline int pte_present(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578{
Martin Schwidefsky83377482006-10-18 18:30:51 +0200579 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX;
580 return (pte_val(pte) & mask) == _PAGE_TYPE_NONE ||
581 (!(pte_val(pte) & _PAGE_INVALID) &&
582 !(pte_val(pte) & _PAGE_SWT));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583}
584
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800585static inline int pte_file(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586{
Martin Schwidefsky83377482006-10-18 18:30:51 +0200587 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT;
588 return (pte_val(pte) & mask) == _PAGE_TYPE_FILE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589}
590
Nick Piggin7e675132008-04-28 02:13:00 -0700591static inline int pte_special(pte_t pte)
592{
Nick Piggina08cb622008-04-28 02:13:03 -0700593 return (pte_val(pte) & _PAGE_SPECIAL);
Nick Piggin7e675132008-04-28 02:13:00 -0700594}
595
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200596#define __HAVE_ARCH_PTE_SAME
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200597static inline int pte_same(pte_t a, pte_t b)
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100598{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200599 return pte_val(a) == pte_val(b);
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100600}
601
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200602static inline pgste_t pgste_get_lock(pte_t *ptep)
603{
604 unsigned long new = 0;
605#ifdef CONFIG_PGSTE
606 unsigned long old;
607
608 preempt_disable();
609 asm(
610 " lg %0,%2\n"
611 "0: lgr %1,%0\n"
612 " nihh %0,0xff7f\n" /* clear RCP_PCL_BIT in old */
613 " oihh %1,0x0080\n" /* set RCP_PCL_BIT in new */
614 " csg %0,%1,%2\n"
615 " jl 0b\n"
616 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
617 : "Q" (ptep[PTRS_PER_PTE]) : "cc");
618#endif
619 return __pgste(new);
620}
621
622static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100623{
624#ifdef CONFIG_PGSTE
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200625 asm(
626 " nihh %1,0xff7f\n" /* clear RCP_PCL_BIT */
627 " stg %1,%0\n"
628 : "=Q" (ptep[PTRS_PER_PTE])
629 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) : "cc");
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100630 preempt_enable();
631#endif
632}
633
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200634static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100635{
636#ifdef CONFIG_PGSTE
Heiko Carstensa43a9d92011-05-29 12:40:50 +0200637 unsigned long address, bits;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200638 unsigned char skey;
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100639
Martin Schwidefsky09b53882011-11-14 11:19:00 +0100640 if (!pte_present(*ptep))
641 return pgste;
Heiko Carstensa43a9d92011-05-29 12:40:50 +0200642 address = pte_val(*ptep) & PAGE_MASK;
643 skey = page_get_storage_key(address);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200644 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
645 /* Clear page changed & referenced bit in the storage key */
Carsten Otte7c818782011-12-01 13:32:16 +0100646 if (bits & _PAGE_CHANGED)
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100647 page_set_storage_key(address, skey ^ bits, 0);
Carsten Otte7c818782011-12-01 13:32:16 +0100648 else if (bits)
649 page_reset_referenced(address);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200650 /* Transfer page changed & referenced bit to guest bits in pgste */
651 pgste_val(pgste) |= bits << 48; /* RCP_GR_BIT & RCP_GC_BIT */
652 /* Get host changed & referenced bits from pgste */
653 bits |= (pgste_val(pgste) & (RCP_HR_BIT | RCP_HC_BIT)) >> 52;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100654 /* Transfer page changed & referenced bit to kvm user bits */
655 pgste_val(pgste) |= bits << 45; /* KVM_UR_BIT & KVM_UC_BIT */
656 /* Clear relevant host bits in pgste. */
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200657 pgste_val(pgste) &= ~(RCP_HR_BIT | RCP_HC_BIT);
658 pgste_val(pgste) &= ~(RCP_ACC_BITS | RCP_FP_BIT);
659 /* Copy page access key and fetch protection bit to pgste */
660 pgste_val(pgste) |=
661 (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100662 /* Transfer referenced bit to pte */
663 pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1;
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100664#endif
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200665 return pgste;
666
667}
668
669static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
670{
671#ifdef CONFIG_PGSTE
672 int young;
673
Martin Schwidefsky09b53882011-11-14 11:19:00 +0100674 if (!pte_present(*ptep))
675 return pgste;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100676 /* Get referenced bit from storage key */
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200677 young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100678 if (young)
679 pgste_val(pgste) |= RCP_GR_BIT;
680 /* Get host referenced bit from pgste */
681 if (pgste_val(pgste) & RCP_HR_BIT) {
682 pgste_val(pgste) &= ~RCP_HR_BIT;
683 young = 1;
684 }
685 /* Transfer referenced bit to kvm user bits and pte */
686 if (young) {
687 pgste_val(pgste) |= KVM_UR_BIT;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200688 pte_val(*ptep) |= _PAGE_SWR;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100689 }
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200690#endif
691 return pgste;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200692}
693
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100694static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200695{
696#ifdef CONFIG_PGSTE
Heiko Carstensa43a9d92011-05-29 12:40:50 +0200697 unsigned long address;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200698 unsigned long okey, nkey;
699
Martin Schwidefsky09b53882011-11-14 11:19:00 +0100700 if (!pte_present(entry))
701 return;
702 address = pte_val(entry) & PAGE_MASK;
Heiko Carstensa43a9d92011-05-29 12:40:50 +0200703 okey = nkey = page_get_storage_key(address);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200704 nkey &= ~(_PAGE_ACC_BITS | _PAGE_FP_BIT);
705 /* Set page access key and fetch protection bit from pgste */
706 nkey |= (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
707 if (okey != nkey)
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100708 page_set_storage_key(address, nkey, 0);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200709#endif
710}
711
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100712static inline void pgste_set_pte(pte_t *ptep, pte_t entry)
713{
714 if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) {
715 /*
716 * Without enhanced suppression-on-protection force
717 * the dirty bit on for all writable ptes.
718 */
719 pte_val(entry) |= _PAGE_SWC;
720 pte_val(entry) &= ~_PAGE_RO;
721 }
722 *ptep = entry;
723}
724
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200725/**
726 * struct gmap_struct - guest address space
727 * @mm: pointer to the parent mm_struct
728 * @table: pointer to the page directory
Christian Borntraeger480e5922011-09-20 17:07:28 +0200729 * @asce: address space control element for gmap page table
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200730 * @crst_list: list of all crst tables used in the guest address space
731 */
732struct gmap {
733 struct list_head list;
734 struct mm_struct *mm;
735 unsigned long *table;
Christian Borntraeger480e5922011-09-20 17:07:28 +0200736 unsigned long asce;
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200737 struct list_head crst_list;
738};
739
740/**
741 * struct gmap_rmap - reverse mapping for segment table entries
742 * @next: pointer to the next gmap_rmap structure in the list
743 * @entry: pointer to a segment table entry
744 */
745struct gmap_rmap {
746 struct list_head list;
747 unsigned long *entry;
748};
749
750/**
751 * struct gmap_pgtable - gmap information attached to a page table
752 * @vmaddr: address of the 1MB segment in the process virtual memory
753 * @mapper: list of segment table entries maping a page table
754 */
755struct gmap_pgtable {
756 unsigned long vmaddr;
757 struct list_head mapper;
758};
759
760struct gmap *gmap_alloc(struct mm_struct *mm);
761void gmap_free(struct gmap *gmap);
762void gmap_enable(struct gmap *gmap);
763void gmap_disable(struct gmap *gmap);
764int gmap_map_segment(struct gmap *gmap, unsigned long from,
765 unsigned long to, unsigned long length);
766int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len);
Heiko Carstensc5034942012-09-10 16:14:33 +0200767unsigned long __gmap_translate(unsigned long address, struct gmap *);
768unsigned long gmap_translate(unsigned long address, struct gmap *);
Carsten Otte499069e2011-10-30 15:17:02 +0100769unsigned long __gmap_fault(unsigned long address, struct gmap *);
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200770unsigned long gmap_fault(unsigned long address, struct gmap *);
Christian Borntraeger388186b2011-10-30 15:17:03 +0100771void gmap_discard(unsigned long from, unsigned long to, struct gmap *);
Martin Schwidefskye5992f22011-07-24 10:48:20 +0200772
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200773/*
774 * Certain architectures need to do special things when PTEs
775 * within a page table are directly modified. Thus, the following
776 * hook is made available.
777 */
778static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
779 pte_t *ptep, pte_t entry)
780{
781 pgste_t pgste;
782
783 if (mm_has_pgste(mm)) {
784 pgste = pgste_get_lock(ptep);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100785 pgste_set_key(ptep, pgste, entry);
786 pgste_set_pte(ptep, entry);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200787 pgste_set_unlock(ptep, pgste);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100788 } else {
789 if (!(pte_val(entry) & _PAGE_INVALID) && MACHINE_HAS_EDAT1)
790 pte_val(entry) |= _PAGE_CO;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200791 *ptep = entry;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100792 }
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100793}
794
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795/*
796 * query functions pte_write/pte_dirty/pte_young only work if
797 * pte_present() is true. Undefined behaviour if not..
798 */
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800799static inline int pte_write(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100801 return (pte_val(pte) & _PAGE_SWW) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802}
803
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800804static inline int pte_dirty(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100806 return (pte_val(pte) & _PAGE_SWC) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807}
808
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800809static inline int pte_young(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200811#ifdef CONFIG_PGSTE
812 if (pte_val(pte) & _PAGE_SWR)
813 return 1;
814#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 return 0;
816}
817
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818/*
819 * pgd/pmd/pte modification functions
820 */
821
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200822static inline void pgd_clear(pgd_t *pgd)
Martin Schwidefsky5a216a22008-02-09 18:24:36 +0100823{
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200824#ifdef CONFIG_64BIT
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100825 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
826 pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200827#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828}
829
Martin Schwidefsky6252d702008-02-09 18:24:37 +0100830static inline void pud_clear(pud_t *pud)
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100831{
Heiko Carstensf4815ac2012-05-23 16:24:51 +0200832#ifdef CONFIG_64BIT
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200833 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
834 pud_val(*pud) = _REGION3_ENTRY_EMPTY;
835#endif
Gerald Schaeferc1821c22007-02-05 21:18:17 +0100836}
Martin Schwidefsky146e4b32008-02-09 18:24:35 +0100837
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200838static inline void pmd_clear(pmd_t *pmdp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839{
Martin Schwidefsky3610cce2007-10-22 12:52:47 +0200840 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841}
842
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800843static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844{
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200845 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846}
847
848/*
849 * The following pte modification functions only work if
850 * pte_present() is true. Undefined behaviour if not..
851 */
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800852static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853{
Nick Piggin138c9022008-07-08 11:31:06 +0200854 pte_val(pte) &= _PAGE_CHG_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 pte_val(pte) |= pgprot_val(newprot);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100856 if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW))
857 pte_val(pte) &= ~_PAGE_RO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 return pte;
859}
860
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800861static inline pte_t pte_wrprotect(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100863 pte_val(pte) &= ~_PAGE_SWW;
Gerald Schaefer9282ed92006-09-20 15:59:37 +0200864 /* Do not clobber _PAGE_TYPE_NONE pages! */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 if (!(pte_val(pte) & _PAGE_INVALID))
866 pte_val(pte) |= _PAGE_RO;
867 return pte;
868}
869
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800870static inline pte_t pte_mkwrite(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100872 pte_val(pte) |= _PAGE_SWW;
873 if (pte_val(pte) & _PAGE_SWC)
874 pte_val(pte) &= ~_PAGE_RO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 return pte;
876}
877
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800878static inline pte_t pte_mkclean(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200880 pte_val(pte) &= ~_PAGE_SWC;
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100881 /* Do not clobber _PAGE_TYPE_NONE pages! */
882 if (!(pte_val(pte) & _PAGE_INVALID))
883 pte_val(pte) |= _PAGE_RO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 return pte;
885}
886
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800887static inline pte_t pte_mkdirty(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100889 pte_val(pte) |= _PAGE_SWC;
890 if (pte_val(pte) & _PAGE_SWW)
891 pte_val(pte) &= ~_PAGE_RO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 return pte;
893}
894
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800895static inline pte_t pte_mkold(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200897#ifdef CONFIG_PGSTE
898 pte_val(pte) &= ~_PAGE_SWR;
899#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 return pte;
901}
902
Adrian Bunk4448aaf2005-11-08 21:34:42 -0800903static inline pte_t pte_mkyoung(pte_t pte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 return pte;
906}
907
Nick Piggin7e675132008-04-28 02:13:00 -0700908static inline pte_t pte_mkspecial(pte_t pte)
909{
Nick Piggina08cb622008-04-28 02:13:03 -0700910 pte_val(pte) |= _PAGE_SPECIAL;
Nick Piggin7e675132008-04-28 02:13:00 -0700911 return pte;
912}
913
Heiko Carstens84afdce2010-10-25 16:10:36 +0200914#ifdef CONFIG_HUGETLB_PAGE
915static inline pte_t pte_mkhuge(pte_t pte)
916{
917 /*
918 * PROT_NONE needs to be remapped from the pte type to the ste type.
919 * The HW invalid bit is also different for pte and ste. The pte
920 * invalid bit happens to be the same as the ste _SEGMENT_ENTRY_LARGE
921 * bit, so we don't have to clear it.
922 */
923 if (pte_val(pte) & _PAGE_INVALID) {
924 if (pte_val(pte) & _PAGE_SWT)
925 pte_val(pte) |= _HPAGE_TYPE_NONE;
926 pte_val(pte) |= _SEGMENT_ENTRY_INV;
927 }
928 /*
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100929 * Clear SW pte bits, there are no SW bits in a segment table entry.
Heiko Carstens84afdce2010-10-25 16:10:36 +0200930 */
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100931 pte_val(pte) &= ~(_PAGE_SWT | _PAGE_SWX | _PAGE_SWC |
932 _PAGE_SWR | _PAGE_SWW);
Heiko Carstens84afdce2010-10-25 16:10:36 +0200933 /*
934 * Also set the change-override bit because we don't need dirty bit
935 * tracking for hugetlbfs pages.
936 */
937 pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO);
938 return pte;
939}
940#endif
941
Florian Funke15e86b02008-10-10 21:33:26 +0200942/*
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200943 * Get (and clear) the user dirty bit for a pte.
Florian Funke15e86b02008-10-10 21:33:26 +0200944 */
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200945static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
946 pte_t *ptep)
Florian Funke15e86b02008-10-10 21:33:26 +0200947{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200948 pgste_t pgste;
949 int dirty = 0;
Florian Funke15e86b02008-10-10 21:33:26 +0200950
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200951 if (mm_has_pgste(mm)) {
952 pgste = pgste_get_lock(ptep);
953 pgste = pgste_update_all(ptep, pgste);
954 dirty = !!(pgste_val(pgste) & KVM_UC_BIT);
955 pgste_val(pgste) &= ~KVM_UC_BIT;
956 pgste_set_unlock(ptep, pgste);
957 return dirty;
Florian Funke15e86b02008-10-10 21:33:26 +0200958 }
Florian Funke15e86b02008-10-10 21:33:26 +0200959 return dirty;
960}
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200961
962/*
963 * Get (and clear) the user referenced bit for a pte.
964 */
965static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
966 pte_t *ptep)
967{
968 pgste_t pgste;
969 int young = 0;
970
971 if (mm_has_pgste(mm)) {
972 pgste = pgste_get_lock(ptep);
973 pgste = pgste_update_young(ptep, pgste);
974 young = !!(pgste_val(pgste) & KVM_UR_BIT);
975 pgste_val(pgste) &= ~KVM_UR_BIT;
976 pgste_set_unlock(ptep, pgste);
977 }
978 return young;
979}
Florian Funke15e86b02008-10-10 21:33:26 +0200980
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200981#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
982static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
983 unsigned long addr, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200985 pgste_t pgste;
986 pte_t pte;
Christian Borntraeger5b7baf02008-03-25 18:47:12 +0100987
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +0200988 if (mm_has_pgste(vma->vm_mm)) {
989 pgste = pgste_get_lock(ptep);
990 pgste = pgste_update_young(ptep, pgste);
991 pte = *ptep;
992 *ptep = pte_mkold(pte);
993 pgste_set_unlock(ptep, pgste);
994 return pte_young(pte);
995 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 return 0;
997}
998
Martin Schwidefskyba8a9222007-10-22 12:52:44 +0200999#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1000static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1001 unsigned long address, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002{
Christian Borntraeger5b7baf02008-03-25 18:47:12 +01001003 /* No need to flush TLB
1004 * On s390 reference bits are in storage key and never in TLB
1005 * With virtualization we handle the reference bit, without we
1006 * we can simply return */
Christian Borntraeger5b7baf02008-03-25 18:47:12 +01001007 return ptep_test_and_clear_young(vma, address, ptep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008}
1009
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001010static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
1011{
1012 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001013#ifndef CONFIG_64BIT
Martin Schwidefsky146e4b32008-02-09 18:24:35 +01001014 /* pto must point to the start of the segment table */
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001015 pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
1016#else
1017 /* ipte in zarch mode can do the math */
1018 pte_t *pto = ptep;
1019#endif
Martin Schwidefsky94c12cc2006-09-28 16:56:43 +02001020 asm volatile(
1021 " ipte %2,%3"
1022 : "=m" (*ptep) : "m" (*ptep),
1023 "a" (pto), "a" (address));
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001024 }
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001025}
1026
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001027/*
1028 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1029 * both clear the TLB for the unmapped pte. The reason is that
1030 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1031 * to modify an active pte. The sequence is
1032 * 1) ptep_get_and_clear
1033 * 2) set_pte_at
1034 * 3) flush_tlb_range
1035 * On s390 the tlb needs to get flushed with the modification of the pte
1036 * if the pte is active. The only way how this can be implemented is to
1037 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1038 * is a nop.
1039 */
1040#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001041static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1042 unsigned long address, pte_t *ptep)
1043{
1044 pgste_t pgste;
1045 pte_t pte;
1046
1047 mm->context.flush_mm = 1;
1048 if (mm_has_pgste(mm))
1049 pgste = pgste_get_lock(ptep);
1050
1051 pte = *ptep;
1052 if (!mm_exclusive(mm))
1053 __ptep_ipte(address, ptep);
1054 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1055
1056 if (mm_has_pgste(mm)) {
1057 pgste = pgste_update_all(&pte, pgste);
1058 pgste_set_unlock(ptep, pgste);
1059 }
1060 return pte;
1061}
1062
1063#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1064static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
1065 unsigned long address,
1066 pte_t *ptep)
1067{
1068 pte_t pte;
1069
1070 mm->context.flush_mm = 1;
1071 if (mm_has_pgste(mm))
1072 pgste_get_lock(ptep);
1073
1074 pte = *ptep;
1075 if (!mm_exclusive(mm))
1076 __ptep_ipte(address, ptep);
1077 return pte;
1078}
1079
1080static inline void ptep_modify_prot_commit(struct mm_struct *mm,
1081 unsigned long address,
1082 pte_t *ptep, pte_t pte)
1083{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001084 if (mm_has_pgste(mm)) {
1085 pgste_set_pte(ptep, pte);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001086 pgste_set_unlock(ptep, *(pgste_t *)(ptep + PTRS_PER_PTE));
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001087 } else
1088 *ptep = pte;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001089}
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001090
1091#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
Martin Schwidefskyf0e47c22007-07-17 04:03:03 -07001092static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1093 unsigned long address, pte_t *ptep)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001095 pgste_t pgste;
1096 pte_t pte;
1097
1098 if (mm_has_pgste(vma->vm_mm))
1099 pgste = pgste_get_lock(ptep);
1100
1101 pte = *ptep;
1102 __ptep_ipte(address, ptep);
1103 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1104
1105 if (mm_has_pgste(vma->vm_mm)) {
1106 pgste = pgste_update_all(&pte, pgste);
1107 pgste_set_unlock(ptep, pgste);
1108 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 return pte;
1110}
1111
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001112/*
1113 * The batched pte unmap code uses ptep_get_and_clear_full to clear the
1114 * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
1115 * tlbs of an mm if it can guarantee that the ptes of the mm_struct
1116 * cannot be accessed while the batched unmap is running. In this case
1117 * full==1 and a simple pte_clear is enough. See tlb.h.
1118 */
1119#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1120static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001121 unsigned long address,
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001122 pte_t *ptep, int full)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123{
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001124 pgste_t pgste;
1125 pte_t pte;
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001126
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001127 if (mm_has_pgste(mm))
1128 pgste = pgste_get_lock(ptep);
1129
1130 pte = *ptep;
1131 if (!full)
1132 __ptep_ipte(address, ptep);
1133 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1134
1135 if (mm_has_pgste(mm)) {
1136 pgste = pgste_update_all(&pte, pgste);
1137 pgste_set_unlock(ptep, pgste);
1138 }
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001139 return pte;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140}
1141
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001142#define __HAVE_ARCH_PTEP_SET_WRPROTECT
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001143static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
1144 unsigned long address, pte_t *ptep)
1145{
1146 pgste_t pgste;
1147 pte_t pte = *ptep;
1148
1149 if (pte_write(pte)) {
1150 mm->context.flush_mm = 1;
1151 if (mm_has_pgste(mm))
1152 pgste = pgste_get_lock(ptep);
1153
1154 if (!mm_exclusive(mm))
1155 __ptep_ipte(address, ptep);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001156 pte = pte_wrprotect(pte);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001157
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001158 if (mm_has_pgste(mm)) {
1159 pgste_set_pte(ptep, pte);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001160 pgste_set_unlock(ptep, pgste);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001161 } else
1162 *ptep = pte;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001163 }
1164 return pte;
1165}
Martin Schwidefskyba8a9222007-10-22 12:52:44 +02001166
1167#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001168static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1169 unsigned long address, pte_t *ptep,
1170 pte_t entry, int dirty)
1171{
1172 pgste_t pgste;
1173
1174 if (pte_same(*ptep, entry))
1175 return 0;
1176 if (mm_has_pgste(vma->vm_mm))
1177 pgste = pgste_get_lock(ptep);
1178
1179 __ptep_ipte(address, ptep);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001180
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001181 if (mm_has_pgste(vma->vm_mm)) {
1182 pgste_set_pte(ptep, entry);
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001183 pgste_set_unlock(ptep, pgste);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001184 } else
1185 *ptep = entry;
Martin Schwidefskyb2fa47e2011-05-23 10:24:40 +02001186 return 1;
1187}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188
1189/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190 * Conversion functions: convert a page and protection to a page entry,
1191 * and a page entry and page directory to the page they refer to.
1192 */
1193static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1194{
1195 pte_t __pte;
1196 pte_val(__pte) = physpage + pgprot_val(pgprot);
1197 return __pte;
1198}
1199
Heiko Carstens2dcea572006-09-29 01:58:41 -07001200static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1201{
Heiko Carstens0b2b6e12006-10-04 20:02:23 +02001202 unsigned long physpage = page_to_phys(page);
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001203 pte_t __pte = mk_pte_phys(physpage, pgprot);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001205 if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) {
1206 pte_val(__pte) |= _PAGE_SWC;
1207 pte_val(__pte) &= ~_PAGE_RO;
1208 }
1209 return __pte;
Heiko Carstens2dcea572006-09-29 01:58:41 -07001210}
1211
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001213#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1214#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1215#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001217#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218#define pgd_offset_k(address) pgd_offset(&init_mm, address)
1219
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001220#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001222#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1223#define pud_deref(pmd) ({ BUG(); 0UL; })
1224#define pgd_deref(pmd) ({ BUG(); 0UL; })
1225
1226#define pud_offset(pgd, address) ((pud_t *) pgd)
1227#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001229#else /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001231#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
1232#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001233#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001234
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001235static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
1236{
Martin Schwidefsky6252d702008-02-09 18:24:37 +01001237 pud_t *pud = (pud_t *) pgd;
1238 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1239 pud = (pud_t *) pgd_deref(*pgd);
Martin Schwidefsky5a216a22008-02-09 18:24:36 +01001240 return pud + pud_index(address);
1241}
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001242
1243static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
1244{
Martin Schwidefsky6252d702008-02-09 18:24:37 +01001245 pmd_t *pmd = (pmd_t *) pud;
1246 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1247 pmd = (pmd_t *) pud_deref(*pud);
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001248 return pmd + pmd_index(address);
1249}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001251#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252
Martin Schwidefsky190a1d72007-10-22 12:52:48 +02001253#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
1254#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1255#define pte_page(x) pfn_to_page(pte_pfn(x))
1256
1257#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
1258
1259/* Find an entry in the lowest level page table.. */
1260#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
1261#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263#define pte_unmap(pte) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001265static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
1266{
1267 unsigned long sto = (unsigned long) pmdp -
1268 pmd_index(address) * sizeof(pmd_t);
1269
1270 if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
1271 asm volatile(
1272 " .insn rrf,0xb98e0000,%2,%3,0,0"
1273 : "=m" (*pmdp)
1274 : "m" (*pmdp), "a" (sto),
1275 "a" ((address & HPAGE_MASK))
1276 : "cc"
1277 );
1278 }
1279}
1280
Gerald Schaefer75077af2012-10-08 16:30:15 -07001281#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Gerald Schaeferd8e7a332012-10-25 17:42:50 +02001282
1283#define SEGMENT_NONE __pgprot(_HPAGE_TYPE_NONE)
1284#define SEGMENT_RO __pgprot(_HPAGE_TYPE_RO)
1285#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW)
1286
Gerald Schaefer9501d092012-10-08 16:30:18 -07001287#define __HAVE_ARCH_PGTABLE_DEPOSIT
1288extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable);
1289
1290#define __HAVE_ARCH_PGTABLE_WITHDRAW
1291extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm);
1292
Gerald Schaefer75077af2012-10-08 16:30:15 -07001293static inline int pmd_trans_splitting(pmd_t pmd)
1294{
1295 return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
1296}
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001297
1298static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1299 pmd_t *pmdp, pmd_t entry)
1300{
Martin Schwidefskyabf09be2012-11-07 13:17:37 +01001301 if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1)
1302 pmd_val(entry) |= _SEGMENT_ENTRY_CO;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001303 *pmdp = entry;
1304}
1305
1306static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1307{
Gerald Schaeferd8e7a332012-10-25 17:42:50 +02001308 /*
1309 * pgprot is PAGE_NONE, PAGE_RO, or PAGE_RW (see __Pxxx / __Sxxx)
1310 * Convert to segment table entry format.
1311 */
1312 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1313 return pgprot_val(SEGMENT_NONE);
1314 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1315 return pgprot_val(SEGMENT_RO);
1316 return pgprot_val(SEGMENT_RW);
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001317}
1318
1319static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1320{
1321 pmd_val(pmd) &= _SEGMENT_CHG_MASK;
1322 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1323 return pmd;
1324}
1325
1326static inline pmd_t pmd_mkhuge(pmd_t pmd)
1327{
1328 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
1329 return pmd;
1330}
1331
1332static inline pmd_t pmd_mkwrite(pmd_t pmd)
1333{
Gerald Schaeferd8e7a332012-10-25 17:42:50 +02001334 /* Do not clobber _HPAGE_TYPE_NONE pages! */
1335 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_INV))
1336 pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001337 return pmd;
1338}
1339
1340static inline pmd_t pmd_wrprotect(pmd_t pmd)
1341{
1342 pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
1343 return pmd;
1344}
1345
1346static inline pmd_t pmd_mkdirty(pmd_t pmd)
1347{
1348 /* No dirty bit in the segment table entry. */
1349 return pmd;
1350}
1351
1352static inline pmd_t pmd_mkold(pmd_t pmd)
1353{
1354 /* No referenced bit in the segment table entry. */
1355 return pmd;
1356}
1357
1358static inline pmd_t pmd_mkyoung(pmd_t pmd)
1359{
1360 /* No referenced bit in the segment table entry. */
1361 return pmd;
1362}
1363
1364#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1365static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1366 unsigned long address, pmd_t *pmdp)
1367{
1368 unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
1369 long tmp, rc;
1370 int counter;
1371
1372 rc = 0;
1373 if (MACHINE_HAS_RRBM) {
1374 counter = PTRS_PER_PTE >> 6;
1375 asm volatile(
1376 "0: .insn rre,0xb9ae0000,%0,%3\n" /* rrbm */
1377 " ogr %1,%0\n"
1378 " la %3,0(%4,%3)\n"
1379 " brct %2,0b\n"
1380 : "=&d" (tmp), "+&d" (rc), "+d" (counter),
1381 "+a" (pmd_addr)
1382 : "a" (64 * 4096UL) : "cc");
1383 rc = !!rc;
1384 } else {
1385 counter = PTRS_PER_PTE;
1386 asm volatile(
1387 "0: rrbe 0,%2\n"
1388 " la %2,0(%3,%2)\n"
1389 " brc 12,1f\n"
1390 " lhi %0,1\n"
1391 "1: brct %1,0b\n"
1392 : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
1393 : "a" (4096UL) : "cc");
1394 }
1395 return rc;
1396}
1397
1398#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
1399static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
1400 unsigned long address, pmd_t *pmdp)
1401{
1402 pmd_t pmd = *pmdp;
1403
1404 __pmd_idte(address, pmdp);
1405 pmd_clear(pmdp);
1406 return pmd;
1407}
1408
1409#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
1410static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
1411 unsigned long address, pmd_t *pmdp)
1412{
1413 return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
1414}
1415
1416#define __HAVE_ARCH_PMDP_INVALIDATE
1417static inline void pmdp_invalidate(struct vm_area_struct *vma,
1418 unsigned long address, pmd_t *pmdp)
1419{
1420 __pmd_idte(address, pmdp);
1421}
1422
Gerald Schaeferbe328652013-01-21 16:48:07 +01001423#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1424static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1425 unsigned long address, pmd_t *pmdp)
1426{
1427 pmd_t pmd = *pmdp;
1428
1429 if (pmd_write(pmd)) {
1430 __pmd_idte(address, pmdp);
1431 set_pmd_at(mm, address, pmdp, pmd_wrprotect(pmd));
1432 }
1433}
1434
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001435static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1436{
1437 pmd_t __pmd;
1438 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
1439 return __pmd;
1440}
1441
1442#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
1443#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1444
1445static inline int pmd_trans_huge(pmd_t pmd)
1446{
1447 return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
1448}
1449
1450static inline int has_transparent_hugepage(void)
1451{
1452 return MACHINE_HAS_HPAGE ? 1 : 0;
1453}
1454
1455static inline unsigned long pmd_pfn(pmd_t pmd)
1456{
Gerald Schaefer171c4002013-01-09 18:49:51 +01001457 return pmd_val(pmd) >> PAGE_SHIFT;
Gerald Schaefer1ae1c1d2012-10-08 16:30:24 -07001458}
Gerald Schaefer75077af2012-10-08 16:30:15 -07001459#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1460
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461/*
1462 * 31 bit swap entry format:
1463 * A page-table entry has some bits we have to treat in a special way.
1464 * Bits 0, 20 and bit 23 have to be zero, otherwise an specification
1465 * exception will occur instead of a page translation exception. The
1466 * specifiation exception has the bad habit not to store necessary
1467 * information in the lowcore.
1468 * Bit 21 and bit 22 are the page invalid bit and the page protection
1469 * bit. We set both to indicate a swapped page.
1470 * Bit 30 and 31 are used to distinguish the different page types. For
1471 * a swapped page these bits need to be zero.
1472 * This leaves the bits 1-19 and bits 24-29 to store type and offset.
1473 * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19
1474 * plus 24 for the offset.
1475 * 0| offset |0110|o|type |00|
1476 * 0 0000000001111111111 2222 2 22222 33
1477 * 0 1234567890123456789 0123 4 56789 01
1478 *
1479 * 64 bit swap entry format:
1480 * A page-table entry has some bits we have to treat in a special way.
1481 * Bits 52 and bit 55 have to be zero, otherwise an specification
1482 * exception will occur instead of a page translation exception. The
1483 * specifiation exception has the bad habit not to store necessary
1484 * information in the lowcore.
1485 * Bit 53 and bit 54 are the page invalid bit and the page protection
1486 * bit. We set both to indicate a swapped page.
1487 * Bit 62 and 63 are used to distinguish the different page types. For
1488 * a swapped page these bits need to be zero.
1489 * This leaves the bits 0-51 and bits 56-61 to store type and offset.
1490 * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51
1491 * plus 56 for the offset.
1492 * | offset |0110|o|type |00|
1493 * 0000000000111111111122222222223333333333444444444455 5555 5 55566 66
1494 * 0123456789012345678901234567890123456789012345678901 2345 6 78901 23
1495 */
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001496#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497#define __SWP_OFFSET_MASK (~0UL >> 12)
1498#else
1499#define __SWP_OFFSET_MASK (~0UL >> 11)
1500#endif
Adrian Bunk4448aaf2005-11-08 21:34:42 -08001501static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502{
1503 pte_t pte;
1504 offset &= __SWP_OFFSET_MASK;
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001505 pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11);
1507 return pte;
1508}
1509
1510#define __swp_type(entry) (((entry).val >> 2) & 0x1f)
1511#define __swp_offset(entry) (((entry).val >> 11) | (((entry).val >> 7) & 1))
1512#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
1513
1514#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1515#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1516
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001517#ifndef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518# define PTE_FILE_MAX_BITS 26
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001519#else /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520# define PTE_FILE_MAX_BITS 59
Heiko Carstensf4815ac2012-05-23 16:24:51 +02001521#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
1523#define pte_to_pgoff(__pte) \
1524 ((((__pte).pte >> 12) << 7) + (((__pte).pte >> 1) & 0x7f))
1525
1526#define pgoff_to_pte(__off) \
1527 ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \
Gerald Schaefer9282ed92006-09-20 15:59:37 +02001528 | _PAGE_TYPE_FILE })
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
1530#endif /* !__ASSEMBLY__ */
1531
1532#define kern_addr_valid(addr) (1)
1533
Heiko Carstens17f34582008-04-30 13:38:47 +02001534extern int vmem_add_mapping(unsigned long start, unsigned long size);
1535extern int vmem_remove_mapping(unsigned long start, unsigned long size);
Carsten Otte402b0862008-03-25 18:47:10 +01001536extern int s390_enable_sie(void);
Heiko Carstensf4eb07c2006-12-08 15:56:07 +01001537
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538/*
1539 * No page table caches to initialise
1540 */
Heiko Carstens765a0ca2013-03-23 10:29:01 +01001541static inline void pgtable_cache_init(void) { }
1542static inline void check_pgt_cache(void) { }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544#include <asm-generic/pgtable.h>
1545
1546#endif /* _S390_PAGE_H */