blob: e0d02c4a2615f8787abc1b13737f7b78e67bc5a5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * ppc64 MMU hashtable management routines
3 *
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +11004 * (c) Copyright IBM Corp. 2003, 2005
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
12
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +110013#include <linux/config.h>
Paul Mackerrasab1f9da2005-10-10 21:58:35 +100014#include <asm/reg.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <asm/pgtable.h>
16#include <asm/mmu.h>
17#include <asm/page.h>
18#include <asm/types.h>
19#include <asm/ppc_asm.h>
Sam Ravnborg0013a852005-09-09 20:57:26 +020020#include <asm/asm-offsets.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/cputable.h>
22
23 .text
24
25/*
26 * Stackframe:
27 *
28 * +-> Back chain (SP + 256)
29 * | General register save area (SP + 112)
30 * | Parameter save area (SP + 48)
31 * | TOC save area (SP + 40)
32 * | link editor doubleword (SP + 32)
33 * | compiler doubleword (SP + 24)
34 * | LR save area (SP + 16)
35 * | CR save area (SP + 8)
36 * SP ---> +-- Back chain (SP + 0)
37 */
38#define STACKFRAMESIZE 256
39
40/* Save parameters offsets */
41#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
42
43/* Save non-volatile offsets */
44#define STK_REG(i) (112 + ((i)-14)*8)
45
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +110046
47#ifndef CONFIG_PPC_64K_PAGES
48
49/*****************************************************************************
50 * *
51 * 4K SW & 4K HW pages implementation *
52 * *
53 *****************************************************************************/
54
55
Linus Torvalds1da177e2005-04-16 15:20:36 -070056/*
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +110057 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
58 * pte_t *ptep, unsigned long trap, int local)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059 *
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +110060 * Adds a 4K page to the hash table in a segment of 4K pages only
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 */
62
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +110063_GLOBAL(__hash_page_4K)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064 mflr r0
65 std r0,16(r1)
66 stdu r1,-STACKFRAMESIZE(r1)
67 /* Save all params that we need after a function call */
68 std r6,STK_PARM(r6)(r1)
69 std r8,STK_PARM(r8)(r1)
70
71 /* Add _PAGE_PRESENT to access */
72 ori r4,r4,_PAGE_PRESENT
73
74 /* Save non-volatile registers.
75 * r31 will hold "old PTE"
76 * r30 is "new PTE"
77 * r29 is "va"
78 * r28 is a hash value
79 * r27 is hashtab mask (maybe dynamic patched instead ?)
80 */
81 std r27,STK_REG(r27)(r1)
82 std r28,STK_REG(r28)(r1)
83 std r29,STK_REG(r29)(r1)
84 std r30,STK_REG(r30)(r1)
85 std r31,STK_REG(r31)(r1)
86
87 /* Step 1:
88 *
89 * Check permissions, atomically mark the linux PTE busy
90 * and hashed.
91 */
921:
93 ldarx r31,0,r6
94 /* Check access rights (access & ~(pte_val(*ptep))) */
95 andc. r0,r4,r31
96 bne- htab_wrong_access
97 /* Check if PTE is busy */
98 andi. r0,r31,_PAGE_BUSY
Olof Johanssond03853d2005-05-01 08:58:45 -070099 /* If so, just bail out and refault if needed. Someone else
100 * is changing this PTE anyway and might hash it.
101 */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100102 bne- htab_bail_ok
103
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 /* Prepare new PTE value (turn access RW into DIRTY, then
105 * add BUSY,HASHPTE and ACCESSED)
106 */
107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108 or r30,r30,r31
109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110 /* Write the linux PTE atomically (setting busy) */
111 stdcx. r30,0,r6
112 bne- 1b
113 isync
114
115 /* Step 2:
116 *
117 * Insert/Update the HPTE in the hash table. At this point,
118 * r4 (access) is re-useable, we use it for the new HPTE flags
119 */
120
121 /* Calc va and put it in r29 */
122 rldicr r29,r5,28,63-28
123 rldicl r3,r3,0,36
124 or r29,r3,r29
125
126 /* Calculate hash value for primary slot and store it in r28 */
127 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
128 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
129 xor r28,r5,r0
130
131 /* Convert linux PTE bits into HW equivalents */
132 andi. r3,r30,0x1fe /* Get basic set of flags */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100133 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
135 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100136 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 andc r0,r30,r0 /* r0 = pte & ~r0 */
138 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
139
140 /* We eventually do the icache sync here (maybe inline that
141 * code rather than call a C function...)
142 */
143BEGIN_FTR_SECTION
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 mr r4,r30
145 mr r5,r7
146 bl .hash_page_do_lazy_icache
David Gibson8913ca12005-07-27 15:47:23 +1000147END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
149 /* At this point, r3 contains new PP bits, save them in
150 * place of "access" in the param area (sic)
151 */
152 std r3,STK_PARM(r4)(r1)
153
154 /* Get htab_hash_mask */
155 ld r4,htab_hash_mask@got(2)
156 ld r27,0(r4) /* htab_hash_mask -> r27 */
157
158 /* Check if we may already be in the hashtable, in this case, we
159 * go to out-of-line code to try to modify the HPTE
160 */
161 andi. r0,r31,_PAGE_HASHPTE
162 bne htab_modify_pte
163
164htab_insert_pte:
165 /* Clear hpte bits in new pte (we also clear BUSY btw) and
166 * add _PAGE_HASHPTE
167 */
168 lis r0,_PAGE_HPTEFLAGS@h
169 ori r0,r0,_PAGE_HPTEFLAGS@l
170 andc r30,r30,r0
171 ori r30,r30,_PAGE_HASHPTE
172
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100173 /* physical address r5 */
174 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
175 sldi r5,r5,PAGE_SHIFT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176
177 /* Calculate primary group hash */
178 and r0,r28,r27
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100179 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
181 /* Call ppc_md.hpte_insert */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100182 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 mr r4,r29 /* Retreive va */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100184 li r7,0 /* !bolted, !secondary */
185 li r8,MMU_PAGE_4K /* page size */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186_GLOBAL(htab_call_hpte_insert1)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100187 bl . /* Patched by htab_finish_init() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 cmpdi 0,r3,0
189 bge htab_pte_insert_ok /* Insertion successful */
190 cmpdi 0,r3,-2 /* Critical failure */
191 beq- htab_pte_insert_failure
192
193 /* Now try secondary slot */
194
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100195 /* physical address r5 */
196 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
197 sldi r5,r5,PAGE_SHIFT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
199 /* Calculate secondary group hash */
200 andc r0,r27,r28
201 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
202
203 /* Call ppc_md.hpte_insert */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100204 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 mr r4,r29 /* Retreive va */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100206 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
207 li r8,MMU_PAGE_4K /* page size */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208_GLOBAL(htab_call_hpte_insert2)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100209 bl . /* Patched by htab_finish_init() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 cmpdi 0,r3,0
211 bge+ htab_pte_insert_ok /* Insertion successful */
212 cmpdi 0,r3,-2 /* Critical failure */
213 beq- htab_pte_insert_failure
214
215 /* Both are full, we need to evict something */
216 mftb r0
217 /* Pick a random group based on TB */
218 andi. r0,r0,1
219 mr r5,r28
220 bne 2f
221 not r5,r5
2222: and r0,r5,r27
223 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
224 /* Call ppc_md.hpte_remove */
225_GLOBAL(htab_call_hpte_remove)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100226 bl . /* Patched by htab_finish_init() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
228 /* Try all again */
229 b htab_insert_pte
230
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100231htab_bail_ok:
Olof Johanssond03853d2005-05-01 08:58:45 -0700232 li r3,0
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100233 b htab_bail
Olof Johanssond03853d2005-05-01 08:58:45 -0700234
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235htab_pte_insert_ok:
236 /* Insert slot number & secondary bit in PTE */
237 rldimi r30,r3,12,63-15
238
239 /* Write out the PTE with a normal write
240 * (maybe add eieio may be good still ?)
241 */
242htab_write_out_pte:
243 ld r6,STK_PARM(r6)(r1)
244 std r30,0(r6)
245 li r3, 0
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100246htab_bail:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 ld r27,STK_REG(r27)(r1)
248 ld r28,STK_REG(r28)(r1)
249 ld r29,STK_REG(r29)(r1)
250 ld r30,STK_REG(r30)(r1)
251 ld r31,STK_REG(r31)(r1)
252 addi r1,r1,STACKFRAMESIZE
253 ld r0,16(r1)
254 mtlr r0
255 blr
256
257htab_modify_pte:
258 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
259 mr r4,r3
260 rlwinm r3,r31,32-12,29,31
261
262 /* Secondary group ? if yes, get a inverted hash value */
263 mr r5,r28
264 andi. r0,r31,_PAGE_SECONDARY
265 beq 1f
266 not r5,r5
2671:
268 /* Calculate proper slot value for ppc_md.hpte_updatepp */
269 and r0,r5,r27
270 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
271 add r3,r0,r3 /* add slot idx */
272
273 /* Call ppc_md.hpte_updatepp */
274 mr r5,r29 /* va */
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100275 li r6,MMU_PAGE_4K /* page size */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 ld r7,STK_PARM(r8)(r1) /* get "local" param */
277_GLOBAL(htab_call_hpte_updatepp)
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100278 bl . /* Patched by htab_finish_init() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
280 /* if we failed because typically the HPTE wasn't really here
281 * we try an insertion.
282 */
283 cmpdi 0,r3,-1
284 beq- htab_insert_pte
285
286 /* Clear the BUSY bit and Write out the PTE */
287 li r0,_PAGE_BUSY
288 andc r30,r30,r0
289 b htab_write_out_pte
290
291htab_wrong_access:
292 /* Bail out clearing reservation */
293 stdcx. r31,0,r6
294 li r3,1
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100295 b htab_bail
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296
297htab_pte_insert_failure:
298 /* Bail out restoring old PTE */
299 ld r6,STK_PARM(r6)(r1)
300 std r31,0(r6)
301 li r3,-1
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100302 b htab_bail
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
304
Benjamin Herrenschmidt3c726f82005-11-07 11:06:55 +1100305#else /* CONFIG_PPC_64K_PAGES */
306
307
308/*****************************************************************************
309 * *
310 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
311 * *
312 *****************************************************************************/
313
314/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
315 * pte_t *ptep, unsigned long trap, int local)
316 */
317
318/*
319 * For now, we do NOT implement Admixed pages
320 */
321_GLOBAL(__hash_page_4K)
322 mflr r0
323 std r0,16(r1)
324 stdu r1,-STACKFRAMESIZE(r1)
325 /* Save all params that we need after a function call */
326 std r6,STK_PARM(r6)(r1)
327 std r8,STK_PARM(r8)(r1)
328
329 /* Add _PAGE_PRESENT to access */
330 ori r4,r4,_PAGE_PRESENT
331
332 /* Save non-volatile registers.
333 * r31 will hold "old PTE"
334 * r30 is "new PTE"
335 * r29 is "va"
336 * r28 is a hash value
337 * r27 is hashtab mask (maybe dynamic patched instead ?)
338 * r26 is the hidx mask
339 * r25 is the index in combo page
340 */
341 std r25,STK_REG(r25)(r1)
342 std r26,STK_REG(r26)(r1)
343 std r27,STK_REG(r27)(r1)
344 std r28,STK_REG(r28)(r1)
345 std r29,STK_REG(r29)(r1)
346 std r30,STK_REG(r30)(r1)
347 std r31,STK_REG(r31)(r1)
348
349 /* Step 1:
350 *
351 * Check permissions, atomically mark the linux PTE busy
352 * and hashed.
353 */
3541:
355 ldarx r31,0,r6
356 /* Check access rights (access & ~(pte_val(*ptep))) */
357 andc. r0,r4,r31
358 bne- htab_wrong_access
359 /* Check if PTE is busy */
360 andi. r0,r31,_PAGE_BUSY
361 /* If so, just bail out and refault if needed. Someone else
362 * is changing this PTE anyway and might hash it.
363 */
364 bne- htab_bail_ok
365 /* Prepare new PTE value (turn access RW into DIRTY, then
366 * add BUSY and ACCESSED)
367 */
368 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
369 or r30,r30,r31
370 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
371 /* Write the linux PTE atomically (setting busy) */
372 stdcx. r30,0,r6
373 bne- 1b
374 isync
375
376 /* Step 2:
377 *
378 * Insert/Update the HPTE in the hash table. At this point,
379 * r4 (access) is re-useable, we use it for the new HPTE flags
380 */
381
382 /* Load the hidx index */
383 rldicl r25,r3,64-12,60
384
385 /* Calc va and put it in r29 */
386 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
387 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
388 or r29,r3,r29 /* r29 = va
389
390 /* Calculate hash value for primary slot and store it in r28 */
391 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
392 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
393 xor r28,r5,r0
394
395 /* Convert linux PTE bits into HW equivalents */
396 andi. r3,r30,0x1fe /* Get basic set of flags */
397 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
398 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
399 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
400 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
401 andc r0,r30,r0 /* r0 = pte & ~r0 */
402 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
403
404 /* We eventually do the icache sync here (maybe inline that
405 * code rather than call a C function...)
406 */
407BEGIN_FTR_SECTION
408 mr r4,r30
409 mr r5,r7
410 bl .hash_page_do_lazy_icache
411END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
412
413 /* At this point, r3 contains new PP bits, save them in
414 * place of "access" in the param area (sic)
415 */
416 std r3,STK_PARM(r4)(r1)
417
418 /* Get htab_hash_mask */
419 ld r4,htab_hash_mask@got(2)
420 ld r27,0(r4) /* htab_hash_mask -> r27 */
421
422 /* Check if we may already be in the hashtable, in this case, we
423 * go to out-of-line code to try to modify the HPTE. We look for
424 * the bit at (1 >> (index + 32))
425 */
426 andi. r0,r31,_PAGE_HASHPTE
427 li r26,0 /* Default hidx */
428 beq htab_insert_pte
429 ld r6,STK_PARM(r6)(r1)
430 ori r26,r6,0x8000 /* Load the hidx mask */
431 ld r26,0(r26)
432 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
433 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
434 bne htab_modify_pte
435
436htab_insert_pte:
437 /* real page number in r5, PTE RPN value + index */
438 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
439 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
440 add r5,r5,r25
441 sldi r5,r5,HW_PAGE_SHIFT
442
443 /* Calculate primary group hash */
444 and r0,r28,r27
445 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
446
447 /* Call ppc_md.hpte_insert */
448 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
449 mr r4,r29 /* Retreive va */
450 li r7,0 /* !bolted, !secondary */
451 li r8,MMU_PAGE_4K /* page size */
452_GLOBAL(htab_call_hpte_insert1)
453 bl . /* patched by htab_finish_init() */
454 cmpdi 0,r3,0
455 bge htab_pte_insert_ok /* Insertion successful */
456 cmpdi 0,r3,-2 /* Critical failure */
457 beq- htab_pte_insert_failure
458
459 /* Now try secondary slot */
460
461 /* real page number in r5, PTE RPN value + index */
462 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
463 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
464 add r5,r5,r25
465 sldi r5,r5,HW_PAGE_SHIFT
466
467 /* Calculate secondary group hash */
468 andc r0,r27,r28
469 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
470
471 /* Call ppc_md.hpte_insert */
472 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
473 mr r4,r29 /* Retreive va */
474 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
475 li r8,MMU_PAGE_4K /* page size */
476_GLOBAL(htab_call_hpte_insert2)
477 bl . /* patched by htab_finish_init() */
478 cmpdi 0,r3,0
479 bge+ htab_pte_insert_ok /* Insertion successful */
480 cmpdi 0,r3,-2 /* Critical failure */
481 beq- htab_pte_insert_failure
482
483 /* Both are full, we need to evict something */
484 mftb r0
485 /* Pick a random group based on TB */
486 andi. r0,r0,1
487 mr r5,r28
488 bne 2f
489 not r5,r5
4902: and r0,r5,r27
491 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
492 /* Call ppc_md.hpte_remove */
493_GLOBAL(htab_call_hpte_remove)
494 bl . /* patched by htab_finish_init() */
495
496 /* Try all again */
497 b htab_insert_pte
498
499htab_bail_ok:
500 li r3,0
501 b htab_bail
502
503htab_pte_insert_ok:
504 /* Insert slot number & secondary bit in PTE second half,
505 * clear _PAGE_BUSY and set approriate HPTE slot bit
506 */
507 ld r6,STK_PARM(r6)(r1)
508 li r0,_PAGE_BUSY
509 andc r30,r30,r0
510 /* HPTE SUB bit */
511 li r0,1
512 subfic r5,r25,27 /* Must match bit position in */
513 sld r0,r0,r5 /* pgtable.h */
514 or r30,r30,r0
515 /* hindx */
516 sldi r5,r25,2
517 sld r3,r3,r5
518 li r4,0xf
519 sld r4,r4,r5
520 andc r26,r26,r4
521 or r26,r26,r3
522 ori r5,r6,0x8000
523 std r26,0(r5)
524 lwsync
525 std r30,0(r6)
526 li r3, 0
527htab_bail:
528 ld r25,STK_REG(r25)(r1)
529 ld r26,STK_REG(r26)(r1)
530 ld r27,STK_REG(r27)(r1)
531 ld r28,STK_REG(r28)(r1)
532 ld r29,STK_REG(r29)(r1)
533 ld r30,STK_REG(r30)(r1)
534 ld r31,STK_REG(r31)(r1)
535 addi r1,r1,STACKFRAMESIZE
536 ld r0,16(r1)
537 mtlr r0
538 blr
539
540htab_modify_pte:
541 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
542 mr r4,r3
543 sldi r5,r25,2
544 srd r3,r26,r5
545
546 /* Secondary group ? if yes, get a inverted hash value */
547 mr r5,r28
548 andi. r0,r3,0x8 /* page secondary ? */
549 beq 1f
550 not r5,r5
5511: andi. r3,r3,0x7 /* extract idx alone */
552
553 /* Calculate proper slot value for ppc_md.hpte_updatepp */
554 and r0,r5,r27
555 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
556 add r3,r0,r3 /* add slot idx */
557
558 /* Call ppc_md.hpte_updatepp */
559 mr r5,r29 /* va */
560 li r6,MMU_PAGE_4K /* page size */
561 ld r7,STK_PARM(r8)(r1) /* get "local" param */
562_GLOBAL(htab_call_hpte_updatepp)
563 bl . /* patched by htab_finish_init() */
564
565 /* if we failed because typically the HPTE wasn't really here
566 * we try an insertion.
567 */
568 cmpdi 0,r3,-1
569 beq- htab_insert_pte
570
571 /* Clear the BUSY bit and Write out the PTE */
572 li r0,_PAGE_BUSY
573 andc r30,r30,r0
574 ld r6,STK_PARM(r6)(r1)
575 std r30,0(r6)
576 li r3,0
577 b htab_bail
578
579htab_wrong_access:
580 /* Bail out clearing reservation */
581 stdcx. r31,0,r6
582 li r3,1
583 b htab_bail
584
585htab_pte_insert_failure:
586 /* Bail out restoring old PTE */
587 ld r6,STK_PARM(r6)(r1)
588 std r31,0(r6)
589 li r3,-1
590 b htab_bail
591
592
593/*****************************************************************************
594 * *
595 * 64K SW & 64K HW in a 64K segment pages implementation *
596 * *
597 *****************************************************************************/
598
599_GLOBAL(__hash_page_64K)
600 mflr r0
601 std r0,16(r1)
602 stdu r1,-STACKFRAMESIZE(r1)
603 /* Save all params that we need after a function call */
604 std r6,STK_PARM(r6)(r1)
605 std r8,STK_PARM(r8)(r1)
606
607 /* Add _PAGE_PRESENT to access */
608 ori r4,r4,_PAGE_PRESENT
609
610 /* Save non-volatile registers.
611 * r31 will hold "old PTE"
612 * r30 is "new PTE"
613 * r29 is "va"
614 * r28 is a hash value
615 * r27 is hashtab mask (maybe dynamic patched instead ?)
616 */
617 std r27,STK_REG(r27)(r1)
618 std r28,STK_REG(r28)(r1)
619 std r29,STK_REG(r29)(r1)
620 std r30,STK_REG(r30)(r1)
621 std r31,STK_REG(r31)(r1)
622
623 /* Step 1:
624 *
625 * Check permissions, atomically mark the linux PTE busy
626 * and hashed.
627 */
6281:
629 ldarx r31,0,r6
630 /* Check access rights (access & ~(pte_val(*ptep))) */
631 andc. r0,r4,r31
632 bne- ht64_wrong_access
633 /* Check if PTE is busy */
634 andi. r0,r31,_PAGE_BUSY
635 /* If so, just bail out and refault if needed. Someone else
636 * is changing this PTE anyway and might hash it.
637 */
638 bne- ht64_bail_ok
639 /* Prepare new PTE value (turn access RW into DIRTY, then
640 * add BUSY,HASHPTE and ACCESSED)
641 */
642 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
643 or r30,r30,r31
644 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
645 /* Write the linux PTE atomically (setting busy) */
646 stdcx. r30,0,r6
647 bne- 1b
648 isync
649
650 /* Step 2:
651 *
652 * Insert/Update the HPTE in the hash table. At this point,
653 * r4 (access) is re-useable, we use it for the new HPTE flags
654 */
655
656 /* Calc va and put it in r29 */
657 rldicr r29,r5,28,63-28
658 rldicl r3,r3,0,36
659 or r29,r3,r29
660
661 /* Calculate hash value for primary slot and store it in r28 */
662 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
663 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
664 xor r28,r5,r0
665
666 /* Convert linux PTE bits into HW equivalents */
667 andi. r3,r30,0x1fe /* Get basic set of flags */
668 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
669 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
670 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
671 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
672 andc r0,r30,r0 /* r0 = pte & ~r0 */
673 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
674
675 /* We eventually do the icache sync here (maybe inline that
676 * code rather than call a C function...)
677 */
678BEGIN_FTR_SECTION
679 mr r4,r30
680 mr r5,r7
681 bl .hash_page_do_lazy_icache
682END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
683
684 /* At this point, r3 contains new PP bits, save them in
685 * place of "access" in the param area (sic)
686 */
687 std r3,STK_PARM(r4)(r1)
688
689 /* Get htab_hash_mask */
690 ld r4,htab_hash_mask@got(2)
691 ld r27,0(r4) /* htab_hash_mask -> r27 */
692
693 /* Check if we may already be in the hashtable, in this case, we
694 * go to out-of-line code to try to modify the HPTE
695 */
696 andi. r0,r31,_PAGE_HASHPTE
697 bne ht64_modify_pte
698
699ht64_insert_pte:
700 /* Clear hpte bits in new pte (we also clear BUSY btw) and
701 * add _PAGE_HASHPTE
702 */
703 lis r0,_PAGE_HPTEFLAGS@h
704 ori r0,r0,_PAGE_HPTEFLAGS@l
705 andc r30,r30,r0
706 ori r30,r30,_PAGE_HASHPTE
707
708 /* Phyical address in r5 */
709 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
710 sldi r5,r5,PAGE_SHIFT
711
712 /* Calculate primary group hash */
713 and r0,r28,r27
714 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
715
716 /* Call ppc_md.hpte_insert */
717 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
718 mr r4,r29 /* Retreive va */
719 li r7,0 /* !bolted, !secondary */
720 li r8,MMU_PAGE_64K
721_GLOBAL(ht64_call_hpte_insert1)
722 bl . /* patched by htab_finish_init() */
723 cmpdi 0,r3,0
724 bge ht64_pte_insert_ok /* Insertion successful */
725 cmpdi 0,r3,-2 /* Critical failure */
726 beq- ht64_pte_insert_failure
727
728 /* Now try secondary slot */
729
730 /* Phyical address in r5 */
731 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
732 sldi r5,r5,PAGE_SHIFT
733
734 /* Calculate secondary group hash */
735 andc r0,r27,r28
736 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
737
738 /* Call ppc_md.hpte_insert */
739 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
740 mr r4,r29 /* Retreive va */
741 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
742 li r8,MMU_PAGE_64K
743_GLOBAL(ht64_call_hpte_insert2)
744 bl . /* patched by htab_finish_init() */
745 cmpdi 0,r3,0
746 bge+ ht64_pte_insert_ok /* Insertion successful */
747 cmpdi 0,r3,-2 /* Critical failure */
748 beq- ht64_pte_insert_failure
749
750 /* Both are full, we need to evict something */
751 mftb r0
752 /* Pick a random group based on TB */
753 andi. r0,r0,1
754 mr r5,r28
755 bne 2f
756 not r5,r5
7572: and r0,r5,r27
758 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
759 /* Call ppc_md.hpte_remove */
760_GLOBAL(ht64_call_hpte_remove)
761 bl . /* patched by htab_finish_init() */
762
763 /* Try all again */
764 b ht64_insert_pte
765
766ht64_bail_ok:
767 li r3,0
768 b ht64_bail
769
770ht64_pte_insert_ok:
771 /* Insert slot number & secondary bit in PTE */
772 rldimi r30,r3,12,63-15
773
774 /* Write out the PTE with a normal write
775 * (maybe add eieio may be good still ?)
776 */
777ht64_write_out_pte:
778 ld r6,STK_PARM(r6)(r1)
779 std r30,0(r6)
780 li r3, 0
781ht64_bail:
782 ld r27,STK_REG(r27)(r1)
783 ld r28,STK_REG(r28)(r1)
784 ld r29,STK_REG(r29)(r1)
785 ld r30,STK_REG(r30)(r1)
786 ld r31,STK_REG(r31)(r1)
787 addi r1,r1,STACKFRAMESIZE
788 ld r0,16(r1)
789 mtlr r0
790 blr
791
792ht64_modify_pte:
793 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
794 mr r4,r3
795 rlwinm r3,r31,32-12,29,31
796
797 /* Secondary group ? if yes, get a inverted hash value */
798 mr r5,r28
799 andi. r0,r31,_PAGE_F_SECOND
800 beq 1f
801 not r5,r5
8021:
803 /* Calculate proper slot value for ppc_md.hpte_updatepp */
804 and r0,r5,r27
805 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
806 add r3,r0,r3 /* add slot idx */
807
808 /* Call ppc_md.hpte_updatepp */
809 mr r5,r29 /* va */
810 li r6,MMU_PAGE_64K
811 ld r7,STK_PARM(r8)(r1) /* get "local" param */
812_GLOBAL(ht64_call_hpte_updatepp)
813 bl . /* patched by htab_finish_init() */
814
815 /* if we failed because typically the HPTE wasn't really here
816 * we try an insertion.
817 */
818 cmpdi 0,r3,-1
819 beq- ht64_insert_pte
820
821 /* Clear the BUSY bit and Write out the PTE */
822 li r0,_PAGE_BUSY
823 andc r30,r30,r0
824 b ht64_write_out_pte
825
826ht64_wrong_access:
827 /* Bail out clearing reservation */
828 stdcx. r31,0,r6
829 li r3,1
830 b ht64_bail
831
832ht64_pte_insert_failure:
833 /* Bail out restoring old PTE */
834 ld r6,STK_PARM(r6)(r1)
835 std r31,0(r6)
836 li r3,-1
837 b ht64_bail
838
839
840#endif /* CONFIG_PPC_64K_PAGES */
841
842
843/*****************************************************************************
844 * *
845 * Huge pages implementation is in hugetlbpage.c *
846 * *
847 *****************************************************************************/