Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1 | /* |
| 2 | * This program is free software; you can redistribute it and/or modify |
| 3 | * it under the terms of the GNU General Public License, version 2, as |
| 4 | * published by the Free Software Foundation. |
| 5 | * |
| 6 | * This program is distributed in the hope that it will be useful, |
| 7 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 8 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 9 | * GNU General Public License for more details. |
| 10 | * |
| 11 | * You should have received a copy of the GNU General Public License |
| 12 | * along with this program; if not, write to the Free Software |
| 13 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 14 | * |
| 15 | * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
| 16 | */ |
| 17 | |
| 18 | #include <linux/types.h> |
| 19 | #include <linux/string.h> |
| 20 | #include <linux/kvm.h> |
| 21 | #include <linux/kvm_host.h> |
| 22 | #include <linux/highmem.h> |
| 23 | #include <linux/gfp.h> |
| 24 | #include <linux/slab.h> |
| 25 | #include <linux/hugetlb.h> |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 26 | #include <linux/vmalloc.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 27 | |
| 28 | #include <asm/tlbflush.h> |
| 29 | #include <asm/kvm_ppc.h> |
| 30 | #include <asm/kvm_book3s.h> |
| 31 | #include <asm/mmu-hash64.h> |
| 32 | #include <asm/hvcall.h> |
| 33 | #include <asm/synch.h> |
| 34 | #include <asm/ppc-opcode.h> |
| 35 | #include <asm/cputable.h> |
| 36 | |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 37 | /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */ |
| 38 | #define MAX_LPID_970 63 |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 39 | #define NR_LPIDS (LPID_RSVD + 1) |
| 40 | unsigned long lpid_inuse[BITS_TO_LONGS(NR_LPIDS)]; |
| 41 | |
| 42 | long kvmppc_alloc_hpt(struct kvm *kvm) |
| 43 | { |
| 44 | unsigned long hpt; |
| 45 | unsigned long lpid; |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 46 | struct revmap_entry *rev; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 47 | |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 48 | /* Allocate guest's hashed page table */ |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 49 | hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_REPEAT|__GFP_NOWARN, |
| 50 | HPT_ORDER - PAGE_SHIFT); |
| 51 | if (!hpt) { |
| 52 | pr_err("kvm_alloc_hpt: Couldn't alloc HPT\n"); |
| 53 | return -ENOMEM; |
| 54 | } |
| 55 | kvm->arch.hpt_virt = hpt; |
| 56 | |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 57 | /* Allocate reverse map array */ |
| 58 | rev = vmalloc(sizeof(struct revmap_entry) * HPT_NPTE); |
| 59 | if (!rev) { |
| 60 | pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n"); |
| 61 | goto out_freehpt; |
| 62 | } |
| 63 | kvm->arch.revmap = rev; |
| 64 | |
| 65 | /* Allocate the guest's logical partition ID */ |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 66 | do { |
| 67 | lpid = find_first_zero_bit(lpid_inuse, NR_LPIDS); |
| 68 | if (lpid >= NR_LPIDS) { |
| 69 | pr_err("kvm_alloc_hpt: No LPIDs free\n"); |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 70 | goto out_freeboth; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 71 | } |
| 72 | } while (test_and_set_bit(lpid, lpid_inuse)); |
| 73 | |
| 74 | kvm->arch.sdr1 = __pa(hpt) | (HPT_ORDER - 18); |
| 75 | kvm->arch.lpid = lpid; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 76 | |
| 77 | pr_info("KVM guest htab at %lx, LPID %lx\n", hpt, lpid); |
| 78 | return 0; |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 79 | |
| 80 | out_freeboth: |
| 81 | vfree(rev); |
| 82 | out_freehpt: |
| 83 | free_pages(hpt, HPT_ORDER - PAGE_SHIFT); |
| 84 | return -ENOMEM; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 85 | } |
| 86 | |
| 87 | void kvmppc_free_hpt(struct kvm *kvm) |
| 88 | { |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 89 | clear_bit(kvm->arch.lpid, lpid_inuse); |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 90 | vfree(kvm->arch.revmap); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 91 | free_pages(kvm->arch.hpt_virt, HPT_ORDER - PAGE_SHIFT); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 92 | } |
| 93 | |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 94 | /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */ |
| 95 | static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 96 | { |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 97 | return (pgsize > 0x1000) ? HPTE_V_LARGE : 0; |
| 98 | } |
| 99 | |
| 100 | /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */ |
| 101 | static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize) |
| 102 | { |
| 103 | return (pgsize == 0x10000) ? 0x1000 : 0; |
| 104 | } |
| 105 | |
| 106 | void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, |
| 107 | unsigned long porder) |
| 108 | { |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 109 | unsigned long i; |
Paul Mackerras | b2b2f16 | 2011-12-12 12:28:21 +0000 | [diff] [blame] | 110 | unsigned long npages; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 111 | unsigned long hp_v, hp_r; |
| 112 | unsigned long addr, hash; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 113 | unsigned long psize; |
| 114 | unsigned long hp0, hp1; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 115 | long ret; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 116 | |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 117 | psize = 1ul << porder; |
| 118 | npages = memslot->npages >> (porder - PAGE_SHIFT); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 119 | |
| 120 | /* VRMA can't be > 1TB */ |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 121 | if (npages > 1ul << (40 - porder)) |
| 122 | npages = 1ul << (40 - porder); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 123 | /* Can't use more than 1 HPTE per HPTEG */ |
| 124 | if (npages > HPT_NPTEG) |
| 125 | npages = HPT_NPTEG; |
| 126 | |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 127 | hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) | |
| 128 | HPTE_V_BOLTED | hpte0_pgsize_encoding(psize); |
| 129 | hp1 = hpte1_pgsize_encoding(psize) | |
| 130 | HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX; |
| 131 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 132 | for (i = 0; i < npages; ++i) { |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 133 | addr = i << porder; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 134 | /* can't use hpt_hash since va > 64 bits */ |
| 135 | hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & HPT_HASH_MASK; |
| 136 | /* |
| 137 | * We assume that the hash table is empty and no |
| 138 | * vcpus are using it at this stage. Since we create |
| 139 | * at most one HPTE per HPTEG, we just assume entry 7 |
| 140 | * is available and use it. |
| 141 | */ |
Paul Mackerras | 8936dda | 2011-12-12 12:27:39 +0000 | [diff] [blame] | 142 | hash = (hash << 3) + 7; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 143 | hp_v = hp0 | ((addr >> 16) & ~0x7fUL); |
| 144 | hp_r = hp1 | addr; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 145 | ret = kvmppc_virtmode_h_enter(vcpu, H_EXACT, hash, hp_v, hp_r); |
| 146 | if (ret != H_SUCCESS) { |
| 147 | pr_err("KVM: map_vrma at %lx failed, ret=%ld\n", |
| 148 | addr, ret); |
| 149 | break; |
| 150 | } |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 151 | } |
| 152 | } |
| 153 | |
| 154 | int kvmppc_mmu_hv_init(void) |
| 155 | { |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 156 | unsigned long host_lpid, rsvd_lpid; |
| 157 | |
| 158 | if (!cpu_has_feature(CPU_FTR_HVMODE)) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 159 | return -EINVAL; |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 160 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 161 | memset(lpid_inuse, 0, sizeof(lpid_inuse)); |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 162 | |
| 163 | if (cpu_has_feature(CPU_FTR_ARCH_206)) { |
| 164 | host_lpid = mfspr(SPRN_LPID); /* POWER7 */ |
| 165 | rsvd_lpid = LPID_RSVD; |
| 166 | } else { |
| 167 | host_lpid = 0; /* PPC970 */ |
| 168 | rsvd_lpid = MAX_LPID_970; |
| 169 | } |
| 170 | |
| 171 | set_bit(host_lpid, lpid_inuse); |
| 172 | /* rsvd_lpid is reserved for use in partition switching */ |
| 173 | set_bit(rsvd_lpid, lpid_inuse); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 174 | |
| 175 | return 0; |
| 176 | } |
| 177 | |
| 178 | void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) |
| 179 | { |
| 180 | } |
| 181 | |
| 182 | static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu) |
| 183 | { |
| 184 | kvmppc_set_msr(vcpu, MSR_SF | MSR_ME); |
| 185 | } |
| 186 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 187 | /* |
| 188 | * This is called to get a reference to a guest page if there isn't |
| 189 | * one already in the kvm->arch.slot_phys[][] arrays. |
| 190 | */ |
| 191 | static long kvmppc_get_guest_page(struct kvm *kvm, unsigned long gfn, |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 192 | struct kvm_memory_slot *memslot, |
| 193 | unsigned long psize) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 194 | { |
| 195 | unsigned long start; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 196 | long np, err; |
| 197 | struct page *page, *hpage, *pages[1]; |
| 198 | unsigned long s, pgsize; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 199 | unsigned long *physp; |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 200 | unsigned int is_io, got, pgorder; |
| 201 | struct vm_area_struct *vma; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 202 | unsigned long pfn, i, npages; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 203 | |
| 204 | physp = kvm->arch.slot_phys[memslot->id]; |
| 205 | if (!physp) |
| 206 | return -EINVAL; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 207 | if (physp[gfn - memslot->base_gfn]) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 208 | return 0; |
| 209 | |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 210 | is_io = 0; |
| 211 | got = 0; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 212 | page = NULL; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 213 | pgsize = psize; |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 214 | err = -EINVAL; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 215 | start = gfn_to_hva_memslot(memslot, gfn); |
| 216 | |
| 217 | /* Instantiate and get the page we want access to */ |
| 218 | np = get_user_pages_fast(start, 1, 1, pages); |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 219 | if (np != 1) { |
| 220 | /* Look up the vma for the page */ |
| 221 | down_read(¤t->mm->mmap_sem); |
| 222 | vma = find_vma(current->mm, start); |
| 223 | if (!vma || vma->vm_start > start || |
| 224 | start + psize > vma->vm_end || |
| 225 | !(vma->vm_flags & VM_PFNMAP)) |
| 226 | goto up_err; |
| 227 | is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot)); |
| 228 | pfn = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); |
| 229 | /* check alignment of pfn vs. requested page size */ |
| 230 | if (psize > PAGE_SIZE && (pfn & ((psize >> PAGE_SHIFT) - 1))) |
| 231 | goto up_err; |
| 232 | up_read(¤t->mm->mmap_sem); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 233 | |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 234 | } else { |
| 235 | page = pages[0]; |
| 236 | got = KVMPPC_GOT_PAGE; |
| 237 | |
| 238 | /* See if this is a large page */ |
| 239 | s = PAGE_SIZE; |
| 240 | if (PageHuge(page)) { |
| 241 | hpage = compound_head(page); |
| 242 | s <<= compound_order(hpage); |
| 243 | /* Get the whole large page if slot alignment is ok */ |
| 244 | if (s > psize && slot_is_aligned(memslot, s) && |
| 245 | !(memslot->userspace_addr & (s - 1))) { |
| 246 | start &= ~(s - 1); |
| 247 | pgsize = s; |
| 248 | page = hpage; |
| 249 | } |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 250 | } |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 251 | if (s < psize) |
| 252 | goto out; |
| 253 | pfn = page_to_pfn(page); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 254 | } |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 255 | |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 256 | npages = pgsize >> PAGE_SHIFT; |
| 257 | pgorder = __ilog2(npages); |
| 258 | physp += (gfn - memslot->base_gfn) & ~(npages - 1); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 259 | spin_lock(&kvm->arch.slot_phys_lock); |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 260 | for (i = 0; i < npages; ++i) { |
| 261 | if (!physp[i]) { |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 262 | physp[i] = ((pfn + i) << PAGE_SHIFT) + |
| 263 | got + is_io + pgorder; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 264 | got = 0; |
| 265 | } |
| 266 | } |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 267 | spin_unlock(&kvm->arch.slot_phys_lock); |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 268 | err = 0; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 269 | |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 270 | out: |
| 271 | if (got) { |
| 272 | if (PageHuge(page)) |
| 273 | page = compound_head(page); |
| 274 | put_page(page); |
| 275 | } |
| 276 | return err; |
Paul Mackerras | 9d0ef5ea | 2011-12-12 12:32:27 +0000 | [diff] [blame] | 277 | |
| 278 | up_err: |
| 279 | up_read(¤t->mm->mmap_sem); |
| 280 | return err; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | /* |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 284 | * We come here on a H_ENTER call from the guest when we are not |
| 285 | * using mmu notifiers and we don't have the requested page pinned |
| 286 | * already. |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 287 | */ |
| 288 | long kvmppc_virtmode_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, |
| 289 | long pte_index, unsigned long pteh, unsigned long ptel) |
| 290 | { |
| 291 | struct kvm *kvm = vcpu->kvm; |
| 292 | unsigned long psize, gpa, gfn; |
| 293 | struct kvm_memory_slot *memslot; |
| 294 | long ret; |
| 295 | |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 296 | if (kvm->arch.using_mmu_notifiers) |
| 297 | goto do_insert; |
| 298 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 299 | psize = hpte_page_size(pteh, ptel); |
| 300 | if (!psize) |
| 301 | return H_PARAMETER; |
| 302 | |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 303 | pteh &= ~(HPTE_V_HVLOCK | HPTE_V_ABSENT | HPTE_V_VALID); |
| 304 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 305 | /* Find the memslot (if any) for this address */ |
| 306 | gpa = (ptel & HPTE_R_RPN) & ~(psize - 1); |
| 307 | gfn = gpa >> PAGE_SHIFT; |
| 308 | memslot = gfn_to_memslot(kvm, gfn); |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 309 | if (memslot && !(memslot->flags & KVM_MEMSLOT_INVALID)) { |
| 310 | if (!slot_is_aligned(memslot, psize)) |
| 311 | return H_PARAMETER; |
| 312 | if (kvmppc_get_guest_page(kvm, gfn, memslot, psize) < 0) |
| 313 | return H_PARAMETER; |
| 314 | } |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 315 | |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 316 | do_insert: |
| 317 | /* Protect linux PTE lookup from page table destruction */ |
| 318 | rcu_read_lock_sched(); /* this disables preemption too */ |
| 319 | vcpu->arch.pgdir = current->mm->pgd; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 320 | ret = kvmppc_h_enter(vcpu, flags, pte_index, pteh, ptel); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 321 | rcu_read_unlock_sched(); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 322 | if (ret == H_TOO_HARD) { |
| 323 | /* this can't happen */ |
| 324 | pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n"); |
| 325 | ret = H_RESOURCE; /* or something */ |
| 326 | } |
| 327 | return ret; |
| 328 | |
| 329 | } |
| 330 | |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 331 | static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu, |
| 332 | gva_t eaddr) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 333 | { |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 334 | u64 mask; |
| 335 | int i; |
| 336 | |
| 337 | for (i = 0; i < vcpu->arch.slb_nr; i++) { |
| 338 | if (!(vcpu->arch.slb[i].orige & SLB_ESID_V)) |
| 339 | continue; |
| 340 | |
| 341 | if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T) |
| 342 | mask = ESID_MASK_1T; |
| 343 | else |
| 344 | mask = ESID_MASK; |
| 345 | |
| 346 | if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0) |
| 347 | return &vcpu->arch.slb[i]; |
| 348 | } |
| 349 | return NULL; |
| 350 | } |
| 351 | |
| 352 | static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r, |
| 353 | unsigned long ea) |
| 354 | { |
| 355 | unsigned long ra_mask; |
| 356 | |
| 357 | ra_mask = hpte_page_size(v, r) - 1; |
| 358 | return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask); |
| 359 | } |
| 360 | |
| 361 | static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, |
| 362 | struct kvmppc_pte *gpte, bool data) |
| 363 | { |
| 364 | struct kvm *kvm = vcpu->kvm; |
| 365 | struct kvmppc_slb *slbe; |
| 366 | unsigned long slb_v; |
| 367 | unsigned long pp, key; |
| 368 | unsigned long v, gr; |
| 369 | unsigned long *hptep; |
| 370 | int index; |
| 371 | int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR); |
| 372 | |
| 373 | /* Get SLB entry */ |
| 374 | if (virtmode) { |
| 375 | slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr); |
| 376 | if (!slbe) |
| 377 | return -EINVAL; |
| 378 | slb_v = slbe->origv; |
| 379 | } else { |
| 380 | /* real mode access */ |
| 381 | slb_v = vcpu->kvm->arch.vrma_slb_v; |
| 382 | } |
| 383 | |
| 384 | /* Find the HPTE in the hash table */ |
| 385 | index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v, |
| 386 | HPTE_V_VALID | HPTE_V_ABSENT); |
| 387 | if (index < 0) |
| 388 | return -ENOENT; |
| 389 | hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4)); |
| 390 | v = hptep[0] & ~HPTE_V_HVLOCK; |
| 391 | gr = kvm->arch.revmap[index].guest_rpte; |
| 392 | |
| 393 | /* Unlock the HPTE */ |
| 394 | asm volatile("lwsync" : : : "memory"); |
| 395 | hptep[0] = v; |
| 396 | |
| 397 | gpte->eaddr = eaddr; |
| 398 | gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff); |
| 399 | |
| 400 | /* Get PP bits and key for permission check */ |
| 401 | pp = gr & (HPTE_R_PP0 | HPTE_R_PP); |
| 402 | key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS; |
| 403 | key &= slb_v; |
| 404 | |
| 405 | /* Calculate permissions */ |
| 406 | gpte->may_read = hpte_read_permission(pp, key); |
| 407 | gpte->may_write = hpte_write_permission(pp, key); |
| 408 | gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G)); |
| 409 | |
| 410 | /* Storage key permission check for POWER7 */ |
| 411 | if (data && virtmode && cpu_has_feature(CPU_FTR_ARCH_206)) { |
| 412 | int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr); |
| 413 | if (amrfield & 1) |
| 414 | gpte->may_read = 0; |
| 415 | if (amrfield & 2) |
| 416 | gpte->may_write = 0; |
| 417 | } |
| 418 | |
| 419 | /* Get the guest physical address */ |
| 420 | gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr); |
| 421 | return 0; |
| 422 | } |
| 423 | |
| 424 | /* |
| 425 | * Quick test for whether an instruction is a load or a store. |
| 426 | * If the instruction is a load or a store, then this will indicate |
| 427 | * which it is, at least on server processors. (Embedded processors |
| 428 | * have some external PID instructions that don't follow the rule |
| 429 | * embodied here.) If the instruction isn't a load or store, then |
| 430 | * this doesn't return anything useful. |
| 431 | */ |
| 432 | static int instruction_is_store(unsigned int instr) |
| 433 | { |
| 434 | unsigned int mask; |
| 435 | |
| 436 | mask = 0x10000000; |
| 437 | if ((instr & 0xfc000000) == 0x7c000000) |
| 438 | mask = 0x100; /* major opcode 31 */ |
| 439 | return (instr & mask) != 0; |
| 440 | } |
| 441 | |
| 442 | static int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu, |
| 443 | unsigned long gpa, int is_store) |
| 444 | { |
| 445 | int ret; |
| 446 | u32 last_inst; |
| 447 | unsigned long srr0 = kvmppc_get_pc(vcpu); |
| 448 | |
| 449 | /* We try to load the last instruction. We don't let |
| 450 | * emulate_instruction do it as it doesn't check what |
| 451 | * kvmppc_ld returns. |
| 452 | * If we fail, we just return to the guest and try executing it again. |
| 453 | */ |
| 454 | if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED) { |
| 455 | ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false); |
| 456 | if (ret != EMULATE_DONE || last_inst == KVM_INST_FETCH_FAILED) |
| 457 | return RESUME_GUEST; |
| 458 | vcpu->arch.last_inst = last_inst; |
| 459 | } |
| 460 | |
| 461 | /* |
| 462 | * WARNING: We do not know for sure whether the instruction we just |
| 463 | * read from memory is the same that caused the fault in the first |
| 464 | * place. If the instruction we read is neither an load or a store, |
| 465 | * then it can't access memory, so we don't need to worry about |
| 466 | * enforcing access permissions. So, assuming it is a load or |
| 467 | * store, we just check that its direction (load or store) is |
| 468 | * consistent with the original fault, since that's what we |
| 469 | * checked the access permissions against. If there is a mismatch |
| 470 | * we just return and retry the instruction. |
| 471 | */ |
| 472 | |
| 473 | if (instruction_is_store(vcpu->arch.last_inst) != !!is_store) |
| 474 | return RESUME_GUEST; |
| 475 | |
| 476 | /* |
| 477 | * Emulated accesses are emulated by looking at the hash for |
| 478 | * translation once, then performing the access later. The |
| 479 | * translation could be invalidated in the meantime in which |
| 480 | * point performing the subsequent memory access on the old |
| 481 | * physical address could possibly be a security hole for the |
| 482 | * guest (but not the host). |
| 483 | * |
| 484 | * This is less of an issue for MMIO stores since they aren't |
| 485 | * globally visible. It could be an issue for MMIO loads to |
| 486 | * a certain extent but we'll ignore it for now. |
| 487 | */ |
| 488 | |
| 489 | vcpu->arch.paddr_accessed = gpa; |
| 490 | return kvmppc_emulate_mmio(run, vcpu); |
| 491 | } |
| 492 | |
| 493 | int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, |
| 494 | unsigned long ea, unsigned long dsisr) |
| 495 | { |
| 496 | struct kvm *kvm = vcpu->kvm; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 497 | unsigned long *hptep, hpte[3], r; |
| 498 | unsigned long mmu_seq, psize, pte_size; |
| 499 | unsigned long gfn, hva, pfn; |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 500 | struct kvm_memory_slot *memslot; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 501 | unsigned long *rmap; |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 502 | struct revmap_entry *rev; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 503 | struct page *page, *pages[1]; |
| 504 | long index, ret, npages; |
| 505 | unsigned long is_io; |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 506 | unsigned int writing, write_ok; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 507 | struct vm_area_struct *vma; |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 508 | unsigned long rcbits; |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 509 | |
| 510 | /* |
| 511 | * Real-mode code has already searched the HPT and found the |
| 512 | * entry we're interested in. Lock the entry and check that |
| 513 | * it hasn't changed. If it has, just return and re-execute the |
| 514 | * instruction. |
| 515 | */ |
| 516 | if (ea != vcpu->arch.pgfault_addr) |
| 517 | return RESUME_GUEST; |
| 518 | index = vcpu->arch.pgfault_index; |
| 519 | hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4)); |
| 520 | rev = &kvm->arch.revmap[index]; |
| 521 | preempt_disable(); |
| 522 | while (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) |
| 523 | cpu_relax(); |
| 524 | hpte[0] = hptep[0] & ~HPTE_V_HVLOCK; |
| 525 | hpte[1] = hptep[1]; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 526 | hpte[2] = r = rev->guest_rpte; |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 527 | asm volatile("lwsync" : : : "memory"); |
| 528 | hptep[0] = hpte[0]; |
| 529 | preempt_enable(); |
| 530 | |
| 531 | if (hpte[0] != vcpu->arch.pgfault_hpte[0] || |
| 532 | hpte[1] != vcpu->arch.pgfault_hpte[1]) |
| 533 | return RESUME_GUEST; |
| 534 | |
| 535 | /* Translate the logical address and get the page */ |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 536 | psize = hpte_page_size(hpte[0], r); |
| 537 | gfn = hpte_rpn(r, psize); |
Paul Mackerras | 697d389 | 2011-12-12 12:36:37 +0000 | [diff] [blame] | 538 | memslot = gfn_to_memslot(kvm, gfn); |
| 539 | |
| 540 | /* No memslot means it's an emulated MMIO region */ |
| 541 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) { |
| 542 | unsigned long gpa = (gfn << PAGE_SHIFT) | (ea & (psize - 1)); |
| 543 | return kvmppc_hv_emulate_mmio(run, vcpu, gpa, |
| 544 | dsisr & DSISR_ISSTORE); |
| 545 | } |
| 546 | |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 547 | if (!kvm->arch.using_mmu_notifiers) |
| 548 | return -EFAULT; /* should never get here */ |
| 549 | |
| 550 | /* used to check for invalidations in progress */ |
| 551 | mmu_seq = kvm->mmu_notifier_seq; |
| 552 | smp_rmb(); |
| 553 | |
| 554 | is_io = 0; |
| 555 | pfn = 0; |
| 556 | page = NULL; |
| 557 | pte_size = PAGE_SIZE; |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 558 | writing = (dsisr & DSISR_ISSTORE) != 0; |
| 559 | /* If writing != 0, then the HPTE must allow writing, if we get here */ |
| 560 | write_ok = writing; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 561 | hva = gfn_to_hva_memslot(memslot, gfn); |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 562 | npages = get_user_pages_fast(hva, 1, writing, pages); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 563 | if (npages < 1) { |
| 564 | /* Check if it's an I/O mapping */ |
| 565 | down_read(¤t->mm->mmap_sem); |
| 566 | vma = find_vma(current->mm, hva); |
| 567 | if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end && |
| 568 | (vma->vm_flags & VM_PFNMAP)) { |
| 569 | pfn = vma->vm_pgoff + |
| 570 | ((hva - vma->vm_start) >> PAGE_SHIFT); |
| 571 | pte_size = psize; |
| 572 | is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot)); |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 573 | write_ok = vma->vm_flags & VM_WRITE; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 574 | } |
| 575 | up_read(¤t->mm->mmap_sem); |
| 576 | if (!pfn) |
| 577 | return -EFAULT; |
| 578 | } else { |
| 579 | page = pages[0]; |
| 580 | if (PageHuge(page)) { |
| 581 | page = compound_head(page); |
| 582 | pte_size <<= compound_order(page); |
| 583 | } |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 584 | /* if the guest wants write access, see if that is OK */ |
| 585 | if (!writing && hpte_is_writable(r)) { |
| 586 | pte_t *ptep, pte; |
| 587 | |
| 588 | /* |
| 589 | * We need to protect against page table destruction |
| 590 | * while looking up and updating the pte. |
| 591 | */ |
| 592 | rcu_read_lock_sched(); |
| 593 | ptep = find_linux_pte_or_hugepte(current->mm->pgd, |
| 594 | hva, NULL); |
| 595 | if (ptep && pte_present(*ptep)) { |
| 596 | pte = kvmppc_read_update_linux_pte(ptep, 1); |
| 597 | if (pte_write(pte)) |
| 598 | write_ok = 1; |
| 599 | } |
| 600 | rcu_read_unlock_sched(); |
| 601 | } |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 602 | pfn = page_to_pfn(page); |
| 603 | } |
| 604 | |
| 605 | ret = -EFAULT; |
| 606 | if (psize > pte_size) |
| 607 | goto out_put; |
| 608 | |
| 609 | /* Check WIMG vs. the actual page we're accessing */ |
| 610 | if (!hpte_cache_flags_ok(r, is_io)) { |
| 611 | if (is_io) |
| 612 | return -EFAULT; |
| 613 | /* |
| 614 | * Allow guest to map emulated device memory as |
| 615 | * uncacheable, but actually make it cacheable. |
| 616 | */ |
| 617 | r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M; |
| 618 | } |
| 619 | |
| 620 | /* Set the HPTE to point to pfn */ |
| 621 | r = (r & ~(HPTE_R_PP0 - pte_size)) | (pfn << PAGE_SHIFT); |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 622 | if (hpte_is_writable(r) && !write_ok) |
| 623 | r = hpte_make_readonly(r); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 624 | ret = RESUME_GUEST; |
| 625 | preempt_disable(); |
| 626 | while (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) |
| 627 | cpu_relax(); |
| 628 | if ((hptep[0] & ~HPTE_V_HVLOCK) != hpte[0] || hptep[1] != hpte[1] || |
| 629 | rev->guest_rpte != hpte[2]) |
| 630 | /* HPTE has been changed under us; let the guest retry */ |
| 631 | goto out_unlock; |
| 632 | hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID; |
| 633 | |
| 634 | rmap = &memslot->rmap[gfn - memslot->base_gfn]; |
| 635 | lock_rmap(rmap); |
| 636 | |
| 637 | /* Check if we might have been invalidated; let the guest retry if so */ |
| 638 | ret = RESUME_GUEST; |
| 639 | if (mmu_notifier_retry(vcpu, mmu_seq)) { |
| 640 | unlock_rmap(rmap); |
| 641 | goto out_unlock; |
| 642 | } |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 643 | |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 644 | /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */ |
| 645 | rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT; |
| 646 | r &= rcbits | ~(HPTE_R_R | HPTE_R_C); |
| 647 | |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 648 | if (hptep[0] & HPTE_V_VALID) { |
| 649 | /* HPTE was previously valid, so we need to invalidate it */ |
| 650 | unlock_rmap(rmap); |
| 651 | hptep[0] |= HPTE_V_ABSENT; |
| 652 | kvmppc_invalidate_hpte(kvm, hptep, index); |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 653 | /* don't lose previous R and C bits */ |
| 654 | r |= hptep[1] & (HPTE_R_R | HPTE_R_C); |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 655 | } else { |
| 656 | kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0); |
| 657 | } |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 658 | |
| 659 | hptep[1] = r; |
| 660 | eieio(); |
| 661 | hptep[0] = hpte[0]; |
| 662 | asm volatile("ptesync" : : : "memory"); |
| 663 | preempt_enable(); |
Paul Mackerras | 4cf302b | 2011-12-12 12:38:51 +0000 | [diff] [blame] | 664 | if (page && hpte_is_writable(r)) |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 665 | SetPageDirty(page); |
| 666 | |
| 667 | out_put: |
| 668 | if (page) |
| 669 | put_page(page); |
| 670 | return ret; |
| 671 | |
| 672 | out_unlock: |
| 673 | hptep[0] &= ~HPTE_V_HVLOCK; |
| 674 | preempt_enable(); |
| 675 | goto out_put; |
| 676 | } |
| 677 | |
| 678 | static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, |
| 679 | int (*handler)(struct kvm *kvm, unsigned long *rmapp, |
| 680 | unsigned long gfn)) |
| 681 | { |
| 682 | int ret; |
| 683 | int retval = 0; |
| 684 | struct kvm_memslots *slots; |
| 685 | struct kvm_memory_slot *memslot; |
| 686 | |
| 687 | slots = kvm_memslots(kvm); |
| 688 | kvm_for_each_memslot(memslot, slots) { |
| 689 | unsigned long start = memslot->userspace_addr; |
| 690 | unsigned long end; |
| 691 | |
| 692 | end = start + (memslot->npages << PAGE_SHIFT); |
| 693 | if (hva >= start && hva < end) { |
| 694 | gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; |
| 695 | |
| 696 | ret = handler(kvm, &memslot->rmap[gfn_offset], |
| 697 | memslot->base_gfn + gfn_offset); |
| 698 | retval |= ret; |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | return retval; |
| 703 | } |
| 704 | |
| 705 | static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 706 | unsigned long gfn) |
| 707 | { |
| 708 | struct revmap_entry *rev = kvm->arch.revmap; |
| 709 | unsigned long h, i, j; |
| 710 | unsigned long *hptep; |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 711 | unsigned long ptel, psize, rcbits; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 712 | |
| 713 | for (;;) { |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 714 | lock_rmap(rmapp); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 715 | if (!(*rmapp & KVMPPC_RMAP_PRESENT)) { |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 716 | unlock_rmap(rmapp); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 717 | break; |
| 718 | } |
| 719 | |
| 720 | /* |
| 721 | * To avoid an ABBA deadlock with the HPTE lock bit, |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 722 | * we can't spin on the HPTE lock while holding the |
| 723 | * rmap chain lock. |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 724 | */ |
| 725 | i = *rmapp & KVMPPC_RMAP_INDEX; |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 726 | hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); |
| 727 | if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { |
| 728 | /* unlock rmap before spinning on the HPTE lock */ |
| 729 | unlock_rmap(rmapp); |
| 730 | while (hptep[0] & HPTE_V_HVLOCK) |
| 731 | cpu_relax(); |
| 732 | continue; |
| 733 | } |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 734 | j = rev[i].forw; |
| 735 | if (j == i) { |
| 736 | /* chain is now empty */ |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 737 | *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 738 | } else { |
| 739 | /* remove i from chain */ |
| 740 | h = rev[i].back; |
| 741 | rev[h].forw = j; |
| 742 | rev[j].back = h; |
| 743 | rev[i].forw = rev[i].back = i; |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 744 | *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 745 | } |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 746 | |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 747 | /* Now check and modify the HPTE */ |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 748 | ptel = rev[i].guest_rpte; |
| 749 | psize = hpte_page_size(hptep[0], ptel); |
| 750 | if ((hptep[0] & HPTE_V_VALID) && |
| 751 | hpte_rpn(ptel, psize) == gfn) { |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 752 | hptep[0] |= HPTE_V_ABSENT; |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 753 | kvmppc_invalidate_hpte(kvm, hptep, i); |
| 754 | /* Harvest R and C */ |
| 755 | rcbits = hptep[1] & (HPTE_R_R | HPTE_R_C); |
| 756 | *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT; |
| 757 | rev[i].guest_rpte = ptel | rcbits; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 758 | } |
Paul Mackerras | bad3b50 | 2011-12-15 02:02:02 +0000 | [diff] [blame] | 759 | unlock_rmap(rmapp); |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 760 | hptep[0] &= ~HPTE_V_HVLOCK; |
| 761 | } |
| 762 | return 0; |
| 763 | } |
| 764 | |
| 765 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) |
| 766 | { |
| 767 | if (kvm->arch.using_mmu_notifiers) |
| 768 | kvm_handle_hva(kvm, hva, kvm_unmap_rmapp); |
| 769 | return 0; |
| 770 | } |
| 771 | |
| 772 | static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 773 | unsigned long gfn) |
| 774 | { |
Paul Mackerras | 5551489 | 2011-12-15 02:02:47 +0000 | [diff] [blame] | 775 | struct revmap_entry *rev = kvm->arch.revmap; |
| 776 | unsigned long head, i, j; |
| 777 | unsigned long *hptep; |
| 778 | int ret = 0; |
| 779 | |
| 780 | retry: |
| 781 | lock_rmap(rmapp); |
| 782 | if (*rmapp & KVMPPC_RMAP_REFERENCED) { |
| 783 | *rmapp &= ~KVMPPC_RMAP_REFERENCED; |
| 784 | ret = 1; |
| 785 | } |
| 786 | if (!(*rmapp & KVMPPC_RMAP_PRESENT)) { |
| 787 | unlock_rmap(rmapp); |
| 788 | return ret; |
| 789 | } |
| 790 | |
| 791 | i = head = *rmapp & KVMPPC_RMAP_INDEX; |
| 792 | do { |
| 793 | hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); |
| 794 | j = rev[i].forw; |
| 795 | |
| 796 | /* If this HPTE isn't referenced, ignore it */ |
| 797 | if (!(hptep[1] & HPTE_R_R)) |
| 798 | continue; |
| 799 | |
| 800 | if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { |
| 801 | /* unlock rmap before spinning on the HPTE lock */ |
| 802 | unlock_rmap(rmapp); |
| 803 | while (hptep[0] & HPTE_V_HVLOCK) |
| 804 | cpu_relax(); |
| 805 | goto retry; |
| 806 | } |
| 807 | |
| 808 | /* Now check and modify the HPTE */ |
| 809 | if ((hptep[0] & HPTE_V_VALID) && (hptep[1] & HPTE_R_R)) { |
| 810 | kvmppc_clear_ref_hpte(kvm, hptep, i); |
| 811 | rev[i].guest_rpte |= HPTE_R_R; |
| 812 | ret = 1; |
| 813 | } |
| 814 | hptep[0] &= ~HPTE_V_HVLOCK; |
| 815 | } while ((i = j) != head); |
| 816 | |
| 817 | unlock_rmap(rmapp); |
| 818 | return ret; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 819 | } |
| 820 | |
| 821 | int kvm_age_hva(struct kvm *kvm, unsigned long hva) |
| 822 | { |
| 823 | if (!kvm->arch.using_mmu_notifiers) |
| 824 | return 0; |
| 825 | return kvm_handle_hva(kvm, hva, kvm_age_rmapp); |
| 826 | } |
| 827 | |
| 828 | static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp, |
| 829 | unsigned long gfn) |
| 830 | { |
Paul Mackerras | 5551489 | 2011-12-15 02:02:47 +0000 | [diff] [blame] | 831 | struct revmap_entry *rev = kvm->arch.revmap; |
| 832 | unsigned long head, i, j; |
| 833 | unsigned long *hp; |
| 834 | int ret = 1; |
| 835 | |
| 836 | if (*rmapp & KVMPPC_RMAP_REFERENCED) |
| 837 | return 1; |
| 838 | |
| 839 | lock_rmap(rmapp); |
| 840 | if (*rmapp & KVMPPC_RMAP_REFERENCED) |
| 841 | goto out; |
| 842 | |
| 843 | if (*rmapp & KVMPPC_RMAP_PRESENT) { |
| 844 | i = head = *rmapp & KVMPPC_RMAP_INDEX; |
| 845 | do { |
| 846 | hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4)); |
| 847 | j = rev[i].forw; |
| 848 | if (hp[1] & HPTE_R_R) |
| 849 | goto out; |
| 850 | } while ((i = j) != head); |
| 851 | } |
| 852 | ret = 0; |
| 853 | |
| 854 | out: |
| 855 | unlock_rmap(rmapp); |
| 856 | return ret; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 857 | } |
| 858 | |
| 859 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) |
| 860 | { |
| 861 | if (!kvm->arch.using_mmu_notifiers) |
| 862 | return 0; |
| 863 | return kvm_handle_hva(kvm, hva, kvm_test_age_rmapp); |
| 864 | } |
| 865 | |
| 866 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) |
| 867 | { |
| 868 | if (!kvm->arch.using_mmu_notifiers) |
| 869 | return; |
| 870 | kvm_handle_hva(kvm, hva, kvm_unmap_rmapp); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 871 | } |
| 872 | |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame^] | 873 | static int kvm_test_clear_dirty(struct kvm *kvm, unsigned long *rmapp) |
| 874 | { |
| 875 | struct revmap_entry *rev = kvm->arch.revmap; |
| 876 | unsigned long head, i, j; |
| 877 | unsigned long *hptep; |
| 878 | int ret = 0; |
| 879 | |
| 880 | retry: |
| 881 | lock_rmap(rmapp); |
| 882 | if (*rmapp & KVMPPC_RMAP_CHANGED) { |
| 883 | *rmapp &= ~KVMPPC_RMAP_CHANGED; |
| 884 | ret = 1; |
| 885 | } |
| 886 | if (!(*rmapp & KVMPPC_RMAP_PRESENT)) { |
| 887 | unlock_rmap(rmapp); |
| 888 | return ret; |
| 889 | } |
| 890 | |
| 891 | i = head = *rmapp & KVMPPC_RMAP_INDEX; |
| 892 | do { |
| 893 | hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); |
| 894 | j = rev[i].forw; |
| 895 | |
| 896 | if (!(hptep[1] & HPTE_R_C)) |
| 897 | continue; |
| 898 | |
| 899 | if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { |
| 900 | /* unlock rmap before spinning on the HPTE lock */ |
| 901 | unlock_rmap(rmapp); |
| 902 | while (hptep[0] & HPTE_V_HVLOCK) |
| 903 | cpu_relax(); |
| 904 | goto retry; |
| 905 | } |
| 906 | |
| 907 | /* Now check and modify the HPTE */ |
| 908 | if ((hptep[0] & HPTE_V_VALID) && (hptep[1] & HPTE_R_C)) { |
| 909 | /* need to make it temporarily absent to clear C */ |
| 910 | hptep[0] |= HPTE_V_ABSENT; |
| 911 | kvmppc_invalidate_hpte(kvm, hptep, i); |
| 912 | hptep[1] &= ~HPTE_R_C; |
| 913 | eieio(); |
| 914 | hptep[0] = (hptep[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID; |
| 915 | rev[i].guest_rpte |= HPTE_R_C; |
| 916 | ret = 1; |
| 917 | } |
| 918 | hptep[0] &= ~HPTE_V_HVLOCK; |
| 919 | } while ((i = j) != head); |
| 920 | |
| 921 | unlock_rmap(rmapp); |
| 922 | return ret; |
| 923 | } |
| 924 | |
| 925 | long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) |
| 926 | { |
| 927 | unsigned long i; |
| 928 | unsigned long *rmapp, *map; |
| 929 | |
| 930 | preempt_disable(); |
| 931 | rmapp = memslot->rmap; |
| 932 | map = memslot->dirty_bitmap; |
| 933 | for (i = 0; i < memslot->npages; ++i) { |
| 934 | if (kvm_test_clear_dirty(kvm, rmapp)) |
| 935 | __set_bit_le(i, map); |
| 936 | ++rmapp; |
| 937 | } |
| 938 | preempt_enable(); |
| 939 | return 0; |
| 940 | } |
| 941 | |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 942 | void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa, |
| 943 | unsigned long *nb_ret) |
| 944 | { |
| 945 | struct kvm_memory_slot *memslot; |
| 946 | unsigned long gfn = gpa >> PAGE_SHIFT; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 947 | struct page *page, *pages[1]; |
| 948 | int npages; |
| 949 | unsigned long hva, psize, offset; |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 950 | unsigned long pa; |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 951 | unsigned long *physp; |
| 952 | |
| 953 | memslot = gfn_to_memslot(kvm, gfn); |
| 954 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) |
| 955 | return NULL; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 956 | if (!kvm->arch.using_mmu_notifiers) { |
| 957 | physp = kvm->arch.slot_phys[memslot->id]; |
| 958 | if (!physp) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 959 | return NULL; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 960 | physp += gfn - memslot->base_gfn; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 961 | pa = *physp; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 962 | if (!pa) { |
| 963 | if (kvmppc_get_guest_page(kvm, gfn, memslot, |
| 964 | PAGE_SIZE) < 0) |
| 965 | return NULL; |
| 966 | pa = *physp; |
| 967 | } |
| 968 | page = pfn_to_page(pa >> PAGE_SHIFT); |
| 969 | } else { |
| 970 | hva = gfn_to_hva_memslot(memslot, gfn); |
| 971 | npages = get_user_pages_fast(hva, 1, 1, pages); |
| 972 | if (npages < 1) |
| 973 | return NULL; |
| 974 | page = pages[0]; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 975 | } |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 976 | psize = PAGE_SIZE; |
| 977 | if (PageHuge(page)) { |
| 978 | page = compound_head(page); |
| 979 | psize <<= compound_order(page); |
| 980 | } |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 981 | if (!kvm->arch.using_mmu_notifiers) |
| 982 | get_page(page); |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 983 | offset = gpa & (psize - 1); |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 984 | if (nb_ret) |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 985 | *nb_ret = psize - offset; |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 986 | return page_address(page) + offset; |
| 987 | } |
| 988 | |
| 989 | void kvmppc_unpin_guest_page(struct kvm *kvm, void *va) |
| 990 | { |
| 991 | struct page *page = virt_to_page(va); |
| 992 | |
| 993 | page = compound_head(page); |
| 994 | put_page(page); |
| 995 | } |
| 996 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 997 | void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu) |
| 998 | { |
| 999 | struct kvmppc_mmu *mmu = &vcpu->arch.mmu; |
| 1000 | |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 1001 | if (cpu_has_feature(CPU_FTR_ARCH_206)) |
| 1002 | vcpu->arch.slb_nr = 32; /* POWER7 */ |
| 1003 | else |
| 1004 | vcpu->arch.slb_nr = 64; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1005 | |
| 1006 | mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate; |
| 1007 | mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr; |
| 1008 | |
| 1009 | vcpu->arch.hflags |= BOOK3S_HFLAG_SLB; |
| 1010 | } |