Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * iSeries hashtable management. |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 3 | * Derived from pSeries_htab.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * |
| 5 | * SMP scalability work: |
| 6 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 7 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | * This program is free software; you can redistribute it and/or |
| 9 | * modify it under the terms of the GNU General Public License |
| 10 | * as published by the Free Software Foundation; either version |
| 11 | * 2 of the License, or (at your option) any later version. |
| 12 | */ |
| 13 | #include <asm/machdep.h> |
| 14 | #include <asm/pgtable.h> |
| 15 | #include <asm/mmu.h> |
| 16 | #include <asm/mmu_context.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | #include <asm/abs_addr.h> |
| 18 | #include <linux/spinlock.h> |
| 19 | |
Stephen Rothwell | 0e29bb1 | 2005-10-14 17:09:16 +1000 | [diff] [blame] | 20 | #include "call_hpt.h" |
| 21 | |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 22 | static spinlock_t iSeries_hlocks[64] __cacheline_aligned_in_smp = |
| 23 | { [0 ... 63] = SPIN_LOCK_UNLOCKED}; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | |
| 25 | /* |
| 26 | * Very primitive algorithm for picking up a lock |
| 27 | */ |
| 28 | static inline void iSeries_hlock(unsigned long slot) |
| 29 | { |
| 30 | if (slot & 0x8) |
| 31 | slot = ~slot; |
| 32 | spin_lock(&iSeries_hlocks[(slot >> 4) & 0x3f]); |
| 33 | } |
| 34 | |
| 35 | static inline void iSeries_hunlock(unsigned long slot) |
| 36 | { |
| 37 | if (slot & 0x8) |
| 38 | slot = ~slot; |
| 39 | spin_unlock(&iSeries_hlocks[(slot >> 4) & 0x3f]); |
| 40 | } |
| 41 | |
| 42 | static long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va, |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 43 | unsigned long prpn, unsigned long vflags, |
| 44 | unsigned long rflags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | { |
Michael Ellerman | aefd16b | 2005-08-03 20:21:24 +1000 | [diff] [blame] | 46 | unsigned long arpn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | long slot; |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 48 | hpte_t lhpte; |
| 49 | int secondary = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 50 | |
| 51 | /* |
| 52 | * The hypervisor tries both primary and secondary. |
| 53 | * If we are being called to insert in the secondary, |
| 54 | * it means we have already tried both primary and secondary, |
| 55 | * so we return failure immediately. |
| 56 | */ |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 57 | if (vflags & HPTE_V_SECONDARY) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | return -1; |
| 59 | |
| 60 | iSeries_hlock(hpte_group); |
| 61 | |
| 62 | slot = HvCallHpt_findValid(&lhpte, va >> PAGE_SHIFT); |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 63 | BUG_ON(lhpte.v & HPTE_V_VALID); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | |
| 65 | if (slot == -1) { /* No available entry found in either group */ |
| 66 | iSeries_hunlock(hpte_group); |
| 67 | return -1; |
| 68 | } |
| 69 | |
| 70 | if (slot < 0) { /* MSB set means secondary group */ |
David Gibson | 3078fcc | 2005-10-21 13:41:19 +1000 | [diff] [blame] | 71 | vflags |= HPTE_V_SECONDARY; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | secondary = 1; |
| 73 | slot &= 0x7fffffffffffffff; |
| 74 | } |
| 75 | |
Michael Ellerman | aefd16b | 2005-08-03 20:21:24 +1000 | [diff] [blame] | 76 | arpn = phys_to_abs(prpn << PAGE_SHIFT) >> PAGE_SHIFT; |
| 77 | |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 78 | lhpte.v = (va >> 23) << HPTE_V_AVPN_SHIFT | vflags | HPTE_V_VALID; |
Michael Ellerman | aefd16b | 2005-08-03 20:21:24 +1000 | [diff] [blame] | 79 | lhpte.r = (arpn << HPTE_R_RPN_SHIFT) | rflags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | |
| 81 | /* Now fill in the actual HPTE */ |
| 82 | HvCallHpt_addValidate(slot, secondary, &lhpte); |
| 83 | |
| 84 | iSeries_hunlock(hpte_group); |
| 85 | |
| 86 | return (secondary << 3) | (slot & 7); |
| 87 | } |
| 88 | |
Michael Ellerman | 4c55130 | 2005-09-23 14:47:58 +1000 | [diff] [blame] | 89 | long iSeries_hpte_bolt_or_insert(unsigned long hpte_group, |
| 90 | unsigned long va, unsigned long prpn, unsigned long vflags, |
| 91 | unsigned long rflags) |
| 92 | { |
| 93 | long slot; |
| 94 | hpte_t lhpte; |
| 95 | |
| 96 | slot = HvCallHpt_findValid(&lhpte, va >> PAGE_SHIFT); |
| 97 | |
| 98 | if (lhpte.v & HPTE_V_VALID) { |
| 99 | /* Bolt the existing HPTE */ |
| 100 | HvCallHpt_setSwBits(slot, 0x10, 0); |
| 101 | HvCallHpt_setPp(slot, PP_RWXX); |
| 102 | return 0; |
| 103 | } |
| 104 | |
| 105 | return iSeries_hpte_insert(hpte_group, va, prpn, vflags, rflags); |
| 106 | } |
| 107 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | static unsigned long iSeries_hpte_getword0(unsigned long slot) |
| 109 | { |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 110 | hpte_t hpte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | |
| 112 | HvCallHpt_get(&hpte, slot); |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 113 | return hpte.v; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | } |
| 115 | |
| 116 | static long iSeries_hpte_remove(unsigned long hpte_group) |
| 117 | { |
| 118 | unsigned long slot_offset; |
| 119 | int i; |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 120 | unsigned long hpte_v; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | |
| 122 | /* Pick a random slot to start at */ |
| 123 | slot_offset = mftb() & 0x7; |
| 124 | |
| 125 | iSeries_hlock(hpte_group); |
| 126 | |
| 127 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 128 | hpte_v = iSeries_hpte_getword0(hpte_group + slot_offset); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 129 | |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 130 | if (! (hpte_v & HPTE_V_BOLTED)) { |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 131 | HvCallHpt_invalidateSetSwBitsGet(hpte_group + |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | slot_offset, 0, 0); |
| 133 | iSeries_hunlock(hpte_group); |
| 134 | return i; |
| 135 | } |
| 136 | |
| 137 | slot_offset++; |
| 138 | slot_offset &= 0x7; |
| 139 | } |
| 140 | |
| 141 | iSeries_hunlock(hpte_group); |
| 142 | |
| 143 | return -1; |
| 144 | } |
| 145 | |
| 146 | /* |
| 147 | * The HyperVisor expects the "flags" argument in this form: |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 148 | * bits 0..59 : reserved |
| 149 | * bit 60 : N |
| 150 | * bits 61..63 : PP2,PP1,PP0 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | */ |
| 152 | static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp, |
| 153 | unsigned long va, int large, int local) |
| 154 | { |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 155 | hpte_t hpte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | unsigned long avpn = va >> 23; |
| 157 | |
| 158 | iSeries_hlock(slot); |
| 159 | |
| 160 | HvCallHpt_get(&hpte, slot); |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 161 | if ((HPTE_V_AVPN_VAL(hpte.v) == avpn) && (hpte.v & HPTE_V_VALID)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 162 | /* |
| 163 | * Hypervisor expects bits as NPPP, which is |
| 164 | * different from how they are mapped in our PP. |
| 165 | */ |
| 166 | HvCallHpt_setPp(slot, (newpp & 0x3) | ((newpp & 0x4) << 1)); |
| 167 | iSeries_hunlock(slot); |
| 168 | return 0; |
| 169 | } |
| 170 | iSeries_hunlock(slot); |
| 171 | |
| 172 | return -1; |
| 173 | } |
| 174 | |
| 175 | /* |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 176 | * Functions used to find the PTE for a particular virtual address. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | * Only used during boot when bolting pages. |
| 178 | * |
| 179 | * Input : vpn : virtual page number |
| 180 | * Output: PTE index within the page table of the entry |
| 181 | * -1 on failure |
| 182 | */ |
| 183 | static long iSeries_hpte_find(unsigned long vpn) |
| 184 | { |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 185 | hpte_t hpte; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | long slot; |
| 187 | |
| 188 | /* |
| 189 | * The HvCallHpt_findValid interface is as follows: |
| 190 | * 0xffffffffffffffff : No entry found. |
| 191 | * 0x00000000xxxxxxxx : Entry found in primary group, slot x |
| 192 | * 0x80000000xxxxxxxx : Entry found in secondary group, slot x |
| 193 | */ |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 194 | slot = HvCallHpt_findValid(&hpte, vpn); |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 195 | if (hpte.v & HPTE_V_VALID) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | if (slot < 0) { |
| 197 | slot &= 0x7fffffffffffffff; |
| 198 | slot = -slot; |
| 199 | } |
| 200 | } else |
| 201 | slot = -1; |
| 202 | return slot; |
| 203 | } |
| 204 | |
| 205 | /* |
| 206 | * Update the page protection bits. Intended to be used to create |
| 207 | * guard pages for kernel data structures on pages which are bolted |
| 208 | * in the HPT. Assumes pages being operated on will not be stolen. |
| 209 | * Does not work on large pages. |
| 210 | * |
| 211 | * No need to lock here because we should be the only user. |
| 212 | */ |
| 213 | static void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea) |
| 214 | { |
| 215 | unsigned long vsid,va,vpn; |
| 216 | long slot; |
| 217 | |
| 218 | vsid = get_kernel_vsid(ea); |
| 219 | va = (vsid << 28) | (ea & 0x0fffffff); |
| 220 | vpn = va >> PAGE_SHIFT; |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 221 | slot = iSeries_hpte_find(vpn); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | if (slot == -1) |
| 223 | panic("updateboltedpp: Could not find page to bolt\n"); |
| 224 | HvCallHpt_setPp(slot, newpp); |
| 225 | } |
| 226 | |
| 227 | static void iSeries_hpte_invalidate(unsigned long slot, unsigned long va, |
| 228 | int large, int local) |
| 229 | { |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 230 | unsigned long hpte_v; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 231 | unsigned long avpn = va >> 23; |
| 232 | unsigned long flags; |
| 233 | |
| 234 | local_irq_save(flags); |
| 235 | |
| 236 | iSeries_hlock(slot); |
| 237 | |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 238 | hpte_v = iSeries_hpte_getword0(slot); |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 239 | |
David Gibson | 96e2844 | 2005-07-13 01:11:42 -0700 | [diff] [blame] | 240 | if ((HPTE_V_AVPN_VAL(hpte_v) == avpn) && (hpte_v & HPTE_V_VALID)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 241 | HvCallHpt_invalidateSetSwBitsGet(slot, 0, 0); |
| 242 | |
| 243 | iSeries_hunlock(slot); |
| 244 | |
| 245 | local_irq_restore(flags); |
| 246 | } |
| 247 | |
| 248 | void hpte_init_iSeries(void) |
| 249 | { |
| 250 | ppc_md.hpte_invalidate = iSeries_hpte_invalidate; |
| 251 | ppc_md.hpte_updatepp = iSeries_hpte_updatepp; |
| 252 | ppc_md.hpte_updateboltedpp = iSeries_hpte_updateboltedpp; |
| 253 | ppc_md.hpte_insert = iSeries_hpte_insert; |
Stephen Rothwell | e508f43 | 2005-09-28 02:28:45 +1000 | [diff] [blame] | 254 | ppc_md.hpte_remove = iSeries_hpte_remove; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | |
| 256 | htab_finish_init(); |
| 257 | } |