Christoffer Dall | 749cf76c | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2012 - Virtual Open Systems and Columbia University |
| 3 | * Author: Christoffer Dall <c.dall@virtualopensystems.com> |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License, version 2, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program; if not, write to the Free Software |
| 16 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 17 | */ |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 18 | |
| 19 | #include <linux/mman.h> |
| 20 | #include <linux/kvm_host.h> |
| 21 | #include <linux/io.h> |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 22 | #include <linux/hugetlb.h> |
Christoffer Dall | 45e96ea | 2013-01-20 18:43:58 -0500 | [diff] [blame] | 23 | #include <trace/events/kvm.h> |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 24 | #include <asm/pgalloc.h> |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 25 | #include <asm/cacheflush.h> |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 26 | #include <asm/kvm_arm.h> |
| 27 | #include <asm/kvm_mmu.h> |
Christoffer Dall | 45e96ea | 2013-01-20 18:43:58 -0500 | [diff] [blame] | 28 | #include <asm/kvm_mmio.h> |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 29 | #include <asm/kvm_asm.h> |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 30 | #include <asm/kvm_emulate.h> |
Marc Zyngier | 1e947ba | 2015-01-29 11:59:54 +0000 | [diff] [blame] | 31 | #include <asm/virt.h> |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 32 | |
| 33 | #include "trace.h" |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 34 | |
| 35 | extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; |
| 36 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 37 | static pgd_t *boot_hyp_pgd; |
Marc Zyngier | 2fb4105 | 2013-04-12 19:12:03 +0100 | [diff] [blame] | 38 | static pgd_t *hyp_pgd; |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 39 | static pgd_t *merged_hyp_pgd; |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 40 | static DEFINE_MUTEX(kvm_hyp_pgd_mutex); |
| 41 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 42 | static unsigned long hyp_idmap_start; |
| 43 | static unsigned long hyp_idmap_end; |
| 44 | static phys_addr_t hyp_idmap_vector; |
| 45 | |
Suzuki K Poulose | 9163ee23 | 2016-03-22 17:01:21 +0000 | [diff] [blame] | 46 | #define S2_PGD_SIZE (PTRS_PER_S2_PGD * sizeof(pgd_t)) |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 47 | #define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t)) |
Mark Salter | 5d4e08c | 2014-03-28 14:25:19 +0000 | [diff] [blame] | 48 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 49 | #define KVM_S2PTE_FLAG_IS_IOMAP (1UL << 0) |
| 50 | #define KVM_S2_FLAG_LOGGING_ACTIVE (1UL << 1) |
| 51 | |
| 52 | static bool memslot_is_logging(struct kvm_memory_slot *memslot) |
| 53 | { |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 54 | return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); |
Mario Smarduch | 7276030 | 2015-01-15 15:59:01 -0800 | [diff] [blame] | 55 | } |
| 56 | |
| 57 | /** |
| 58 | * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8 |
| 59 | * @kvm: pointer to kvm structure. |
| 60 | * |
| 61 | * Interface to HYP function to flush all VM TLB entries |
| 62 | */ |
| 63 | void kvm_flush_remote_tlbs(struct kvm *kvm) |
| 64 | { |
| 65 | kvm_call_hyp(__kvm_tlb_flush_vmid, kvm); |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 66 | } |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 67 | |
Marc Zyngier | 4876276 | 2013-01-28 15:27:00 +0000 | [diff] [blame] | 68 | static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 69 | { |
Suzuki K Poulose | 8684e70 | 2016-03-22 17:14:25 +0000 | [diff] [blame] | 70 | kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 71 | } |
| 72 | |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 73 | /* |
| 74 | * D-Cache management functions. They take the page table entries by |
| 75 | * value, as they are flushing the cache using the kernel mapping (or |
| 76 | * kmap on 32bit). |
| 77 | */ |
| 78 | static void kvm_flush_dcache_pte(pte_t pte) |
| 79 | { |
| 80 | __kvm_flush_dcache_pte(pte); |
| 81 | } |
| 82 | |
| 83 | static void kvm_flush_dcache_pmd(pmd_t pmd) |
| 84 | { |
| 85 | __kvm_flush_dcache_pmd(pmd); |
| 86 | } |
| 87 | |
| 88 | static void kvm_flush_dcache_pud(pud_t pud) |
| 89 | { |
| 90 | __kvm_flush_dcache_pud(pud); |
| 91 | } |
| 92 | |
Ard Biesheuvel | e6fab54 | 2015-11-10 15:11:20 +0100 | [diff] [blame] | 93 | static bool kvm_is_device_pfn(unsigned long pfn) |
| 94 | { |
| 95 | return !pfn_valid(pfn); |
| 96 | } |
| 97 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 98 | /** |
| 99 | * stage2_dissolve_pmd() - clear and flush huge PMD entry |
| 100 | * @kvm: pointer to kvm structure. |
| 101 | * @addr: IPA |
| 102 | * @pmd: pmd pointer for IPA |
| 103 | * |
| 104 | * Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all |
| 105 | * pages in the range dirty. |
| 106 | */ |
| 107 | static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) |
| 108 | { |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 109 | if (!pmd_thp_or_huge(*pmd)) |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 110 | return; |
| 111 | |
| 112 | pmd_clear(pmd); |
| 113 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
| 114 | put_page(virt_to_page(pmd)); |
| 115 | } |
| 116 | |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 117 | static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, |
| 118 | int min, int max) |
| 119 | { |
| 120 | void *page; |
| 121 | |
| 122 | BUG_ON(max > KVM_NR_MEM_OBJS); |
| 123 | if (cache->nobjs >= min) |
| 124 | return 0; |
| 125 | while (cache->nobjs < max) { |
| 126 | page = (void *)__get_free_page(PGALLOC_GFP); |
| 127 | if (!page) |
| 128 | return -ENOMEM; |
| 129 | cache->objects[cache->nobjs++] = page; |
| 130 | } |
| 131 | return 0; |
| 132 | } |
| 133 | |
| 134 | static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) |
| 135 | { |
| 136 | while (mc->nobjs) |
| 137 | free_page((unsigned long)mc->objects[--mc->nobjs]); |
| 138 | } |
| 139 | |
| 140 | static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) |
| 141 | { |
| 142 | void *p; |
| 143 | |
| 144 | BUG_ON(!mc || !mc->nobjs); |
| 145 | p = mc->objects[--mc->nobjs]; |
| 146 | return p; |
| 147 | } |
| 148 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 149 | static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr) |
Marc Zyngier | 979acd5 | 2013-08-06 13:05:48 +0100 | [diff] [blame] | 150 | { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 151 | pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL); |
| 152 | stage2_pgd_clear(pgd); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 153 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 154 | stage2_pud_free(pud_table); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 155 | put_page(virt_to_page(pgd)); |
Marc Zyngier | 979acd5 | 2013-08-06 13:05:48 +0100 | [diff] [blame] | 156 | } |
| 157 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 158 | static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 159 | { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 160 | pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(pud, 0); |
| 161 | VM_BUG_ON(stage2_pud_huge(*pud)); |
| 162 | stage2_pud_clear(pud); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 163 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 164 | stage2_pmd_free(pmd_table); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 165 | put_page(virt_to_page(pud)); |
| 166 | } |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 167 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 168 | static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr) |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 169 | { |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 170 | pte_t *pte_table = pte_offset_kernel(pmd, 0); |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 171 | VM_BUG_ON(pmd_thp_or_huge(*pmd)); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 172 | pmd_clear(pmd); |
| 173 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
| 174 | pte_free_kernel(NULL, pte_table); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 175 | put_page(virt_to_page(pmd)); |
| 176 | } |
| 177 | |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 178 | /* |
| 179 | * Unmapping vs dcache management: |
| 180 | * |
| 181 | * If a guest maps certain memory pages as uncached, all writes will |
| 182 | * bypass the data cache and go directly to RAM. However, the CPUs |
| 183 | * can still speculate reads (not writes) and fill cache lines with |
| 184 | * data. |
| 185 | * |
| 186 | * Those cache lines will be *clean* cache lines though, so a |
| 187 | * clean+invalidate operation is equivalent to an invalidate |
| 188 | * operation, because no cache lines are marked dirty. |
| 189 | * |
| 190 | * Those clean cache lines could be filled prior to an uncached write |
| 191 | * by the guest, and the cache coherent IO subsystem would therefore |
| 192 | * end up writing old data to disk. |
| 193 | * |
| 194 | * This is why right after unmapping a page/section and invalidating |
| 195 | * the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure |
| 196 | * the IO subsystem will never hit in the cache. |
| 197 | */ |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 198 | static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd, |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 199 | phys_addr_t addr, phys_addr_t end) |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 200 | { |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 201 | phys_addr_t start_addr = addr; |
| 202 | pte_t *pte, *start_pte; |
| 203 | |
| 204 | start_pte = pte = pte_offset_kernel(pmd, addr); |
| 205 | do { |
| 206 | if (!pte_none(*pte)) { |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 207 | pte_t old_pte = *pte; |
| 208 | |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 209 | kvm_set_pte(pte, __pte(0)); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 210 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 211 | |
| 212 | /* No need to invalidate the cache for device mappings */ |
Ard Biesheuvel | 0de58f8 | 2015-12-03 09:25:22 +0100 | [diff] [blame] | 213 | if (!kvm_is_device_pfn(pte_pfn(old_pte))) |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 214 | kvm_flush_dcache_pte(old_pte); |
| 215 | |
| 216 | put_page(virt_to_page(pte)); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 217 | } |
| 218 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 219 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 220 | if (stage2_pte_table_empty(start_pte)) |
| 221 | clear_stage2_pmd_entry(kvm, pmd, start_addr); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 222 | } |
| 223 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 224 | static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud, |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 225 | phys_addr_t addr, phys_addr_t end) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 226 | { |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 227 | phys_addr_t next, start_addr = addr; |
| 228 | pmd_t *pmd, *start_pmd; |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 229 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 230 | start_pmd = pmd = stage2_pmd_offset(pud, addr); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 231 | do { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 232 | next = stage2_pmd_addr_end(addr, end); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 233 | if (!pmd_none(*pmd)) { |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 234 | if (pmd_thp_or_huge(*pmd)) { |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 235 | pmd_t old_pmd = *pmd; |
| 236 | |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 237 | pmd_clear(pmd); |
| 238 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 239 | |
| 240 | kvm_flush_dcache_pmd(old_pmd); |
| 241 | |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 242 | put_page(virt_to_page(pmd)); |
| 243 | } else { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 244 | unmap_stage2_ptes(kvm, pmd, addr, next); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 245 | } |
| 246 | } |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 247 | } while (pmd++, addr = next, addr != end); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 248 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 249 | if (stage2_pmd_table_empty(start_pmd)) |
| 250 | clear_stage2_pud_entry(kvm, pud, start_addr); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 251 | } |
| 252 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 253 | static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd, |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 254 | phys_addr_t addr, phys_addr_t end) |
| 255 | { |
| 256 | phys_addr_t next, start_addr = addr; |
| 257 | pud_t *pud, *start_pud; |
| 258 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 259 | start_pud = pud = stage2_pud_offset(pgd, addr); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 260 | do { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 261 | next = stage2_pud_addr_end(addr, end); |
| 262 | if (!stage2_pud_none(*pud)) { |
| 263 | if (stage2_pud_huge(*pud)) { |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 264 | pud_t old_pud = *pud; |
| 265 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 266 | stage2_pud_clear(pud); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 267 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 268 | kvm_flush_dcache_pud(old_pud); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 269 | put_page(virt_to_page(pud)); |
| 270 | } else { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 271 | unmap_stage2_pmds(kvm, pud, addr, next); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 272 | } |
| 273 | } |
| 274 | } while (pud++, addr = next, addr != end); |
| 275 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 276 | if (stage2_pud_table_empty(start_pud)) |
| 277 | clear_stage2_pgd_entry(kvm, pgd, start_addr); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 278 | } |
| 279 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 280 | /** |
| 281 | * unmap_stage2_range -- Clear stage2 page table entries to unmap a range |
| 282 | * @kvm: The VM pointer |
| 283 | * @start: The intermediate physical base address of the range to unmap |
| 284 | * @size: The size of the area to unmap |
| 285 | * |
| 286 | * Clear a range of stage-2 mappings, lowering the various ref-counts. Must |
| 287 | * be called while holding mmu_lock (unless for freeing the stage2 pgd before |
| 288 | * destroying the VM), otherwise another faulting VCPU may come in and mess |
| 289 | * with things behind our backs. |
| 290 | */ |
| 291 | static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size) |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 292 | { |
| 293 | pgd_t *pgd; |
| 294 | phys_addr_t addr = start, end = start + size; |
| 295 | phys_addr_t next; |
| 296 | |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 297 | pgd = kvm->arch.pgd + stage2_pgd_index(addr); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 298 | do { |
Suzuki K Poulose | 7a1c831 | 2016-03-23 12:08:02 +0000 | [diff] [blame] | 299 | next = stage2_pgd_addr_end(addr, end); |
| 300 | if (!stage2_pgd_none(*pgd)) |
| 301 | unmap_stage2_puds(kvm, pgd, addr, next); |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 302 | } while (pgd++, addr = next, addr != end); |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 303 | } |
| 304 | |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 305 | static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd, |
| 306 | phys_addr_t addr, phys_addr_t end) |
| 307 | { |
| 308 | pte_t *pte; |
| 309 | |
| 310 | pte = pte_offset_kernel(pmd, addr); |
| 311 | do { |
Ard Biesheuvel | 0de58f8 | 2015-12-03 09:25:22 +0100 | [diff] [blame] | 312 | if (!pte_none(*pte) && !kvm_is_device_pfn(pte_pfn(*pte))) |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 313 | kvm_flush_dcache_pte(*pte); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 314 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 315 | } |
| 316 | |
| 317 | static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud, |
| 318 | phys_addr_t addr, phys_addr_t end) |
| 319 | { |
| 320 | pmd_t *pmd; |
| 321 | phys_addr_t next; |
| 322 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 323 | pmd = stage2_pmd_offset(pud, addr); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 324 | do { |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 325 | next = stage2_pmd_addr_end(addr, end); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 326 | if (!pmd_none(*pmd)) { |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 327 | if (pmd_thp_or_huge(*pmd)) |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 328 | kvm_flush_dcache_pmd(*pmd); |
| 329 | else |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 330 | stage2_flush_ptes(kvm, pmd, addr, next); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 331 | } |
| 332 | } while (pmd++, addr = next, addr != end); |
| 333 | } |
| 334 | |
| 335 | static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd, |
| 336 | phys_addr_t addr, phys_addr_t end) |
| 337 | { |
| 338 | pud_t *pud; |
| 339 | phys_addr_t next; |
| 340 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 341 | pud = stage2_pud_offset(pgd, addr); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 342 | do { |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 343 | next = stage2_pud_addr_end(addr, end); |
| 344 | if (!stage2_pud_none(*pud)) { |
| 345 | if (stage2_pud_huge(*pud)) |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 346 | kvm_flush_dcache_pud(*pud); |
| 347 | else |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 348 | stage2_flush_pmds(kvm, pud, addr, next); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 349 | } |
| 350 | } while (pud++, addr = next, addr != end); |
| 351 | } |
| 352 | |
| 353 | static void stage2_flush_memslot(struct kvm *kvm, |
| 354 | struct kvm_memory_slot *memslot) |
| 355 | { |
| 356 | phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; |
| 357 | phys_addr_t end = addr + PAGE_SIZE * memslot->npages; |
| 358 | phys_addr_t next; |
| 359 | pgd_t *pgd; |
| 360 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 361 | pgd = kvm->arch.pgd + stage2_pgd_index(addr); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 362 | do { |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 363 | next = stage2_pgd_addr_end(addr, end); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 364 | stage2_flush_puds(kvm, pgd, addr, next); |
| 365 | } while (pgd++, addr = next, addr != end); |
| 366 | } |
| 367 | |
| 368 | /** |
| 369 | * stage2_flush_vm - Invalidate cache for pages mapped in stage 2 |
| 370 | * @kvm: The struct kvm pointer |
| 371 | * |
| 372 | * Go through the stage 2 page tables and invalidate any cache lines |
| 373 | * backing memory already mapped to the VM. |
| 374 | */ |
Marc Zyngier | 3c1e716 | 2014-12-19 16:05:31 +0000 | [diff] [blame] | 375 | static void stage2_flush_vm(struct kvm *kvm) |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 376 | { |
| 377 | struct kvm_memslots *slots; |
| 378 | struct kvm_memory_slot *memslot; |
| 379 | int idx; |
| 380 | |
| 381 | idx = srcu_read_lock(&kvm->srcu); |
| 382 | spin_lock(&kvm->mmu_lock); |
| 383 | |
| 384 | slots = kvm_memslots(kvm); |
| 385 | kvm_for_each_memslot(memslot, slots) |
| 386 | stage2_flush_memslot(kvm, memslot); |
| 387 | |
| 388 | spin_unlock(&kvm->mmu_lock); |
| 389 | srcu_read_unlock(&kvm->srcu, idx); |
| 390 | } |
| 391 | |
Suzuki K Poulose | 64f3249 | 2016-03-22 18:56:21 +0000 | [diff] [blame] | 392 | static void clear_hyp_pgd_entry(pgd_t *pgd) |
| 393 | { |
| 394 | pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL); |
| 395 | pgd_clear(pgd); |
| 396 | pud_free(NULL, pud_table); |
| 397 | put_page(virt_to_page(pgd)); |
| 398 | } |
| 399 | |
| 400 | static void clear_hyp_pud_entry(pud_t *pud) |
| 401 | { |
| 402 | pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0); |
| 403 | VM_BUG_ON(pud_huge(*pud)); |
| 404 | pud_clear(pud); |
| 405 | pmd_free(NULL, pmd_table); |
| 406 | put_page(virt_to_page(pud)); |
| 407 | } |
| 408 | |
| 409 | static void clear_hyp_pmd_entry(pmd_t *pmd) |
| 410 | { |
| 411 | pte_t *pte_table = pte_offset_kernel(pmd, 0); |
| 412 | VM_BUG_ON(pmd_thp_or_huge(*pmd)); |
| 413 | pmd_clear(pmd); |
| 414 | pte_free_kernel(NULL, pte_table); |
| 415 | put_page(virt_to_page(pmd)); |
| 416 | } |
| 417 | |
| 418 | static void unmap_hyp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) |
| 419 | { |
| 420 | pte_t *pte, *start_pte; |
| 421 | |
| 422 | start_pte = pte = pte_offset_kernel(pmd, addr); |
| 423 | do { |
| 424 | if (!pte_none(*pte)) { |
| 425 | kvm_set_pte(pte, __pte(0)); |
| 426 | put_page(virt_to_page(pte)); |
| 427 | } |
| 428 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 429 | |
| 430 | if (hyp_pte_table_empty(start_pte)) |
| 431 | clear_hyp_pmd_entry(pmd); |
| 432 | } |
| 433 | |
| 434 | static void unmap_hyp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) |
| 435 | { |
| 436 | phys_addr_t next; |
| 437 | pmd_t *pmd, *start_pmd; |
| 438 | |
| 439 | start_pmd = pmd = pmd_offset(pud, addr); |
| 440 | do { |
| 441 | next = pmd_addr_end(addr, end); |
| 442 | /* Hyp doesn't use huge pmds */ |
| 443 | if (!pmd_none(*pmd)) |
| 444 | unmap_hyp_ptes(pmd, addr, next); |
| 445 | } while (pmd++, addr = next, addr != end); |
| 446 | |
| 447 | if (hyp_pmd_table_empty(start_pmd)) |
| 448 | clear_hyp_pud_entry(pud); |
| 449 | } |
| 450 | |
| 451 | static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end) |
| 452 | { |
| 453 | phys_addr_t next; |
| 454 | pud_t *pud, *start_pud; |
| 455 | |
| 456 | start_pud = pud = pud_offset(pgd, addr); |
| 457 | do { |
| 458 | next = pud_addr_end(addr, end); |
| 459 | /* Hyp doesn't use huge puds */ |
| 460 | if (!pud_none(*pud)) |
| 461 | unmap_hyp_pmds(pud, addr, next); |
| 462 | } while (pud++, addr = next, addr != end); |
| 463 | |
| 464 | if (hyp_pud_table_empty(start_pud)) |
| 465 | clear_hyp_pgd_entry(pgd); |
| 466 | } |
| 467 | |
| 468 | static void unmap_hyp_range(pgd_t *pgdp, phys_addr_t start, u64 size) |
| 469 | { |
| 470 | pgd_t *pgd; |
| 471 | phys_addr_t addr = start, end = start + size; |
| 472 | phys_addr_t next; |
| 473 | |
| 474 | /* |
| 475 | * We don't unmap anything from HYP, except at the hyp tear down. |
| 476 | * Hence, we don't have to invalidate the TLBs here. |
| 477 | */ |
| 478 | pgd = pgdp + pgd_index(addr); |
| 479 | do { |
| 480 | next = pgd_addr_end(addr, end); |
| 481 | if (!pgd_none(*pgd)) |
| 482 | unmap_hyp_puds(pgd, addr, next); |
| 483 | } while (pgd++, addr = next, addr != end); |
| 484 | } |
| 485 | |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 486 | /** |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 487 | * free_boot_hyp_pgd - free HYP boot page tables |
| 488 | * |
| 489 | * Free the HYP boot page tables. The bounce page is also freed. |
| 490 | */ |
| 491 | void free_boot_hyp_pgd(void) |
| 492 | { |
| 493 | mutex_lock(&kvm_hyp_pgd_mutex); |
| 494 | |
| 495 | if (boot_hyp_pgd) { |
Suzuki K Poulose | 64f3249 | 2016-03-22 18:56:21 +0000 | [diff] [blame] | 496 | unmap_hyp_range(boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE); |
| 497 | unmap_hyp_range(boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE); |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 498 | free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order); |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 499 | boot_hyp_pgd = NULL; |
| 500 | } |
| 501 | |
| 502 | if (hyp_pgd) |
Suzuki K Poulose | 64f3249 | 2016-03-22 18:56:21 +0000 | [diff] [blame] | 503 | unmap_hyp_range(hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE); |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 504 | |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 505 | mutex_unlock(&kvm_hyp_pgd_mutex); |
| 506 | } |
| 507 | |
| 508 | /** |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 509 | * free_hyp_pgds - free Hyp-mode page tables |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 510 | * |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 511 | * Assumes hyp_pgd is a page table used strictly in Hyp-mode and |
| 512 | * therefore contains either mappings in the kernel memory area (above |
| 513 | * PAGE_OFFSET), or device mappings in the vmalloc range (from |
| 514 | * VMALLOC_START to VMALLOC_END). |
| 515 | * |
| 516 | * boot_hyp_pgd should only map two pages for the init code. |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 517 | */ |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 518 | void free_hyp_pgds(void) |
Marc Zyngier | 000d399 | 2013-03-05 02:43:17 +0000 | [diff] [blame] | 519 | { |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 520 | unsigned long addr; |
| 521 | |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 522 | free_boot_hyp_pgd(); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 523 | |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 524 | mutex_lock(&kvm_hyp_pgd_mutex); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 525 | |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 526 | if (hyp_pgd) { |
| 527 | for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE) |
Suzuki K Poulose | 64f3249 | 2016-03-22 18:56:21 +0000 | [diff] [blame] | 528 | unmap_hyp_range(hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 529 | for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE) |
Suzuki K Poulose | 64f3249 | 2016-03-22 18:56:21 +0000 | [diff] [blame] | 530 | unmap_hyp_range(hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); |
Marc Zyngier | d4cb9df5 | 2013-05-14 12:11:34 +0100 | [diff] [blame] | 531 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 532 | free_pages((unsigned long)hyp_pgd, hyp_pgd_order); |
Marc Zyngier | d157f4a | 2013-04-12 19:12:07 +0100 | [diff] [blame] | 533 | hyp_pgd = NULL; |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 534 | } |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 535 | if (merged_hyp_pgd) { |
| 536 | clear_page(merged_hyp_pgd); |
| 537 | free_page((unsigned long)merged_hyp_pgd); |
| 538 | merged_hyp_pgd = NULL; |
| 539 | } |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 540 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 541 | mutex_unlock(&kvm_hyp_pgd_mutex); |
| 542 | } |
| 543 | |
| 544 | static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start, |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 545 | unsigned long end, unsigned long pfn, |
| 546 | pgprot_t prot) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 547 | { |
| 548 | pte_t *pte; |
| 549 | unsigned long addr; |
| 550 | |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 551 | addr = start; |
| 552 | do { |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 553 | pte = pte_offset_kernel(pmd, addr); |
| 554 | kvm_set_pte(pte, pfn_pte(pfn, prot)); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 555 | get_page(virt_to_page(pte)); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 556 | kvm_flush_dcache_to_poc(pte, sizeof(*pte)); |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 557 | pfn++; |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 558 | } while (addr += PAGE_SIZE, addr != end); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 559 | } |
| 560 | |
| 561 | static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start, |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 562 | unsigned long end, unsigned long pfn, |
| 563 | pgprot_t prot) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 564 | { |
| 565 | pmd_t *pmd; |
| 566 | pte_t *pte; |
| 567 | unsigned long addr, next; |
| 568 | |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 569 | addr = start; |
| 570 | do { |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 571 | pmd = pmd_offset(pud, addr); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 572 | |
| 573 | BUG_ON(pmd_sect(*pmd)); |
| 574 | |
| 575 | if (pmd_none(*pmd)) { |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 576 | pte = pte_alloc_one_kernel(NULL, addr); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 577 | if (!pte) { |
| 578 | kvm_err("Cannot allocate Hyp pte\n"); |
| 579 | return -ENOMEM; |
| 580 | } |
| 581 | pmd_populate_kernel(NULL, pmd, pte); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 582 | get_page(virt_to_page(pmd)); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 583 | kvm_flush_dcache_to_poc(pmd, sizeof(*pmd)); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 584 | } |
| 585 | |
| 586 | next = pmd_addr_end(addr, end); |
| 587 | |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 588 | create_hyp_pte_mappings(pmd, addr, next, pfn, prot); |
| 589 | pfn += (next - addr) >> PAGE_SHIFT; |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 590 | } while (addr = next, addr != end); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 591 | |
| 592 | return 0; |
| 593 | } |
| 594 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 595 | static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start, |
| 596 | unsigned long end, unsigned long pfn, |
| 597 | pgprot_t prot) |
| 598 | { |
| 599 | pud_t *pud; |
| 600 | pmd_t *pmd; |
| 601 | unsigned long addr, next; |
| 602 | int ret; |
| 603 | |
| 604 | addr = start; |
| 605 | do { |
| 606 | pud = pud_offset(pgd, addr); |
| 607 | |
| 608 | if (pud_none_or_clear_bad(pud)) { |
| 609 | pmd = pmd_alloc_one(NULL, addr); |
| 610 | if (!pmd) { |
| 611 | kvm_err("Cannot allocate Hyp pmd\n"); |
| 612 | return -ENOMEM; |
| 613 | } |
| 614 | pud_populate(NULL, pud, pmd); |
| 615 | get_page(virt_to_page(pud)); |
| 616 | kvm_flush_dcache_to_poc(pud, sizeof(*pud)); |
| 617 | } |
| 618 | |
| 619 | next = pud_addr_end(addr, end); |
| 620 | ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot); |
| 621 | if (ret) |
| 622 | return ret; |
| 623 | pfn += (next - addr) >> PAGE_SHIFT; |
| 624 | } while (addr = next, addr != end); |
| 625 | |
| 626 | return 0; |
| 627 | } |
| 628 | |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 629 | static int __create_hyp_mappings(pgd_t *pgdp, |
| 630 | unsigned long start, unsigned long end, |
| 631 | unsigned long pfn, pgprot_t prot) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 632 | { |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 633 | pgd_t *pgd; |
| 634 | pud_t *pud; |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 635 | unsigned long addr, next; |
| 636 | int err = 0; |
| 637 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 638 | mutex_lock(&kvm_hyp_pgd_mutex); |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 639 | addr = start & PAGE_MASK; |
| 640 | end = PAGE_ALIGN(end); |
| 641 | do { |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 642 | pgd = pgdp + pgd_index(addr); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 643 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 644 | if (pgd_none(*pgd)) { |
| 645 | pud = pud_alloc_one(NULL, addr); |
| 646 | if (!pud) { |
| 647 | kvm_err("Cannot allocate Hyp pud\n"); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 648 | err = -ENOMEM; |
| 649 | goto out; |
| 650 | } |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 651 | pgd_populate(NULL, pgd, pud); |
| 652 | get_page(virt_to_page(pgd)); |
| 653 | kvm_flush_dcache_to_poc(pgd, sizeof(*pgd)); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 654 | } |
| 655 | |
| 656 | next = pgd_addr_end(addr, end); |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 657 | err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 658 | if (err) |
| 659 | goto out; |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 660 | pfn += (next - addr) >> PAGE_SHIFT; |
Marc Zyngier | 3562c76 | 2013-04-12 19:12:02 +0100 | [diff] [blame] | 661 | } while (addr = next, addr != end); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 662 | out: |
| 663 | mutex_unlock(&kvm_hyp_pgd_mutex); |
| 664 | return err; |
| 665 | } |
| 666 | |
Christoffer Dall | 40c2729 | 2013-11-15 13:14:12 -0800 | [diff] [blame] | 667 | static phys_addr_t kvm_kaddr_to_phys(void *kaddr) |
| 668 | { |
| 669 | if (!is_vmalloc_addr(kaddr)) { |
| 670 | BUG_ON(!virt_addr_valid(kaddr)); |
| 671 | return __pa(kaddr); |
| 672 | } else { |
| 673 | return page_to_phys(vmalloc_to_page(kaddr)) + |
| 674 | offset_in_page(kaddr); |
| 675 | } |
| 676 | } |
| 677 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 678 | /** |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 679 | * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 680 | * @from: The virtual kernel start address of the range |
| 681 | * @to: The virtual kernel end address of the range (exclusive) |
| 682 | * |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 683 | * The same virtual address as the kernel virtual address is also used |
| 684 | * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying |
| 685 | * physical pages. |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 686 | */ |
| 687 | int create_hyp_mappings(void *from, void *to) |
| 688 | { |
Christoffer Dall | 40c2729 | 2013-11-15 13:14:12 -0800 | [diff] [blame] | 689 | phys_addr_t phys_addr; |
| 690 | unsigned long virt_addr; |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 691 | unsigned long start = KERN_TO_HYP((unsigned long)from); |
| 692 | unsigned long end = KERN_TO_HYP((unsigned long)to); |
| 693 | |
Marc Zyngier | 1e947ba | 2015-01-29 11:59:54 +0000 | [diff] [blame] | 694 | if (is_kernel_in_hyp_mode()) |
| 695 | return 0; |
| 696 | |
Christoffer Dall | 40c2729 | 2013-11-15 13:14:12 -0800 | [diff] [blame] | 697 | start = start & PAGE_MASK; |
| 698 | end = PAGE_ALIGN(end); |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 699 | |
Christoffer Dall | 40c2729 | 2013-11-15 13:14:12 -0800 | [diff] [blame] | 700 | for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) { |
| 701 | int err; |
| 702 | |
| 703 | phys_addr = kvm_kaddr_to_phys(from + virt_addr - start); |
| 704 | err = __create_hyp_mappings(hyp_pgd, virt_addr, |
| 705 | virt_addr + PAGE_SIZE, |
| 706 | __phys_to_pfn(phys_addr), |
| 707 | PAGE_HYP); |
| 708 | if (err) |
| 709 | return err; |
| 710 | } |
| 711 | |
| 712 | return 0; |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 713 | } |
| 714 | |
| 715 | /** |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 716 | * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode |
| 717 | * @from: The kernel start VA of the range |
| 718 | * @to: The kernel end VA of the range (exclusive) |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 719 | * @phys_addr: The physical start address which gets mapped |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 720 | * |
| 721 | * The resulting HYP VA is the same as the kernel VA, modulo |
| 722 | * HYP_PAGE_OFFSET. |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 723 | */ |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 724 | int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr) |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 725 | { |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 726 | unsigned long start = KERN_TO_HYP((unsigned long)from); |
| 727 | unsigned long end = KERN_TO_HYP((unsigned long)to); |
| 728 | |
Marc Zyngier | 1e947ba | 2015-01-29 11:59:54 +0000 | [diff] [blame] | 729 | if (is_kernel_in_hyp_mode()) |
| 730 | return 0; |
| 731 | |
Marc Zyngier | 6060df8 | 2013-04-12 19:12:01 +0100 | [diff] [blame] | 732 | /* Check for a valid kernel IO mapping */ |
| 733 | if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1)) |
| 734 | return -EINVAL; |
| 735 | |
| 736 | return __create_hyp_mappings(hyp_pgd, start, end, |
| 737 | __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 738 | } |
| 739 | |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 740 | /** |
| 741 | * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation. |
| 742 | * @kvm: The KVM struct pointer for the VM. |
| 743 | * |
Vladimir Murzin | 9d4dc688 | 2015-11-16 11:28:16 +0000 | [diff] [blame] | 744 | * Allocates only the stage-2 HW PGD level table(s) (can support either full |
| 745 | * 40-bit input addresses or limited to 32-bit input addresses). Clears the |
| 746 | * allocated pages. |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 747 | * |
| 748 | * Note we don't need locking here as this is only called when the VM is |
| 749 | * created, which can only be done once. |
| 750 | */ |
| 751 | int kvm_alloc_stage2_pgd(struct kvm *kvm) |
| 752 | { |
| 753 | pgd_t *pgd; |
| 754 | |
| 755 | if (kvm->arch.pgd != NULL) { |
| 756 | kvm_err("kvm_arch already initialized?\n"); |
| 757 | return -EINVAL; |
| 758 | } |
| 759 | |
Suzuki K Poulose | 9163ee23 | 2016-03-22 17:01:21 +0000 | [diff] [blame] | 760 | /* Allocate the HW PGD, making sure that each page gets its own refcount */ |
| 761 | pgd = alloc_pages_exact(S2_PGD_SIZE, GFP_KERNEL | __GFP_ZERO); |
| 762 | if (!pgd) |
Marc Zyngier | a987370 | 2015-03-10 19:06:59 +0000 | [diff] [blame] | 763 | return -ENOMEM; |
| 764 | |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 765 | kvm_clean_pgd(pgd); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 766 | kvm->arch.pgd = pgd; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 767 | return 0; |
| 768 | } |
| 769 | |
Christoffer Dall | 957db10 | 2014-11-27 10:35:03 +0100 | [diff] [blame] | 770 | static void stage2_unmap_memslot(struct kvm *kvm, |
| 771 | struct kvm_memory_slot *memslot) |
| 772 | { |
| 773 | hva_t hva = memslot->userspace_addr; |
| 774 | phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; |
| 775 | phys_addr_t size = PAGE_SIZE * memslot->npages; |
| 776 | hva_t reg_end = hva + size; |
| 777 | |
| 778 | /* |
| 779 | * A memory region could potentially cover multiple VMAs, and any holes |
| 780 | * between them, so iterate over all of them to find out if we should |
| 781 | * unmap any of them. |
| 782 | * |
| 783 | * +--------------------------------------------+ |
| 784 | * +---------------+----------------+ +----------------+ |
| 785 | * | : VMA 1 | VMA 2 | | VMA 3 : | |
| 786 | * +---------------+----------------+ +----------------+ |
| 787 | * | memory region | |
| 788 | * +--------------------------------------------+ |
| 789 | */ |
| 790 | do { |
| 791 | struct vm_area_struct *vma = find_vma(current->mm, hva); |
| 792 | hva_t vm_start, vm_end; |
| 793 | |
| 794 | if (!vma || vma->vm_start >= reg_end) |
| 795 | break; |
| 796 | |
| 797 | /* |
| 798 | * Take the intersection of this VMA with the memory region |
| 799 | */ |
| 800 | vm_start = max(hva, vma->vm_start); |
| 801 | vm_end = min(reg_end, vma->vm_end); |
| 802 | |
| 803 | if (!(vma->vm_flags & VM_PFNMAP)) { |
| 804 | gpa_t gpa = addr + (vm_start - memslot->userspace_addr); |
| 805 | unmap_stage2_range(kvm, gpa, vm_end - vm_start); |
| 806 | } |
| 807 | hva = vm_end; |
| 808 | } while (hva < reg_end); |
| 809 | } |
| 810 | |
| 811 | /** |
| 812 | * stage2_unmap_vm - Unmap Stage-2 RAM mappings |
| 813 | * @kvm: The struct kvm pointer |
| 814 | * |
| 815 | * Go through the memregions and unmap any reguler RAM |
| 816 | * backing memory already mapped to the VM. |
| 817 | */ |
| 818 | void stage2_unmap_vm(struct kvm *kvm) |
| 819 | { |
| 820 | struct kvm_memslots *slots; |
| 821 | struct kvm_memory_slot *memslot; |
| 822 | int idx; |
| 823 | |
| 824 | idx = srcu_read_lock(&kvm->srcu); |
| 825 | spin_lock(&kvm->mmu_lock); |
| 826 | |
| 827 | slots = kvm_memslots(kvm); |
| 828 | kvm_for_each_memslot(memslot, slots) |
| 829 | stage2_unmap_memslot(kvm, memslot); |
| 830 | |
| 831 | spin_unlock(&kvm->mmu_lock); |
| 832 | srcu_read_unlock(&kvm->srcu, idx); |
| 833 | } |
| 834 | |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 835 | /** |
| 836 | * kvm_free_stage2_pgd - free all stage-2 tables |
| 837 | * @kvm: The KVM struct pointer for the VM. |
| 838 | * |
| 839 | * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all |
| 840 | * underlying level-2 and level-3 tables before freeing the actual level-1 table |
| 841 | * and setting the struct pointer to NULL. |
| 842 | * |
| 843 | * Note we don't need locking here as this is only called when the VM is |
| 844 | * destroyed, which can only be done once. |
| 845 | */ |
| 846 | void kvm_free_stage2_pgd(struct kvm *kvm) |
| 847 | { |
| 848 | if (kvm->arch.pgd == NULL) |
| 849 | return; |
| 850 | |
| 851 | unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE); |
Suzuki K Poulose | 9163ee23 | 2016-03-22 17:01:21 +0000 | [diff] [blame] | 852 | /* Free the HW pgd, one page at a time */ |
| 853 | free_pages_exact(kvm->arch.pgd, S2_PGD_SIZE); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 854 | kvm->arch.pgd = NULL; |
| 855 | } |
| 856 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 857 | static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, |
| 858 | phys_addr_t addr) |
| 859 | { |
| 860 | pgd_t *pgd; |
| 861 | pud_t *pud; |
| 862 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 863 | pgd = kvm->arch.pgd + stage2_pgd_index(addr); |
| 864 | if (WARN_ON(stage2_pgd_none(*pgd))) { |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 865 | if (!cache) |
| 866 | return NULL; |
| 867 | pud = mmu_memory_cache_alloc(cache); |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 868 | stage2_pgd_populate(pgd, pud); |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 869 | get_page(virt_to_page(pgd)); |
| 870 | } |
| 871 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 872 | return stage2_pud_offset(pgd, addr); |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 873 | } |
| 874 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 875 | static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, |
| 876 | phys_addr_t addr) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 877 | { |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 878 | pud_t *pud; |
| 879 | pmd_t *pmd; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 880 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 881 | pud = stage2_get_pud(kvm, cache, addr); |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 882 | if (stage2_pud_none(*pud)) { |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 883 | if (!cache) |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 884 | return NULL; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 885 | pmd = mmu_memory_cache_alloc(cache); |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 886 | stage2_pud_populate(pud, pmd); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 887 | get_page(virt_to_page(pud)); |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 888 | } |
| 889 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 890 | return stage2_pmd_offset(pud, addr); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 891 | } |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 892 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 893 | static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache |
| 894 | *cache, phys_addr_t addr, const pmd_t *new_pmd) |
| 895 | { |
| 896 | pmd_t *pmd, old_pmd; |
| 897 | |
| 898 | pmd = stage2_get_pmd(kvm, cache, addr); |
| 899 | VM_BUG_ON(!pmd); |
| 900 | |
| 901 | /* |
| 902 | * Mapping in huge pages should only happen through a fault. If a |
| 903 | * page is merged into a transparent huge page, the individual |
| 904 | * subpages of that huge page should be unmapped through MMU |
| 905 | * notifiers before we get here. |
| 906 | * |
| 907 | * Merging of CompoundPages is not supported; they should become |
| 908 | * splitting first, unmapped, merged, and mapped back in on-demand. |
| 909 | */ |
| 910 | VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd)); |
| 911 | |
| 912 | old_pmd = *pmd; |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 913 | if (pmd_present(old_pmd)) { |
| 914 | pmd_clear(pmd); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 915 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 916 | } else { |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 917 | get_page(virt_to_page(pmd)); |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 918 | } |
| 919 | |
| 920 | kvm_set_pmd(pmd, *new_pmd); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 921 | return 0; |
| 922 | } |
| 923 | |
| 924 | static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 925 | phys_addr_t addr, const pte_t *new_pte, |
| 926 | unsigned long flags) |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 927 | { |
| 928 | pmd_t *pmd; |
| 929 | pte_t *pte, old_pte; |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 930 | bool iomap = flags & KVM_S2PTE_FLAG_IS_IOMAP; |
| 931 | bool logging_active = flags & KVM_S2_FLAG_LOGGING_ACTIVE; |
| 932 | |
| 933 | VM_BUG_ON(logging_active && !cache); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 934 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 935 | /* Create stage-2 page table mapping - Levels 0 and 1 */ |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 936 | pmd = stage2_get_pmd(kvm, cache, addr); |
| 937 | if (!pmd) { |
| 938 | /* |
| 939 | * Ignore calls from kvm_set_spte_hva for unallocated |
| 940 | * address ranges. |
| 941 | */ |
| 942 | return 0; |
| 943 | } |
| 944 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 945 | /* |
| 946 | * While dirty page logging - dissolve huge PMD, then continue on to |
| 947 | * allocate page. |
| 948 | */ |
| 949 | if (logging_active) |
| 950 | stage2_dissolve_pmd(kvm, addr, pmd); |
| 951 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 952 | /* Create stage-2 page mappings - Level 2 */ |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 953 | if (pmd_none(*pmd)) { |
| 954 | if (!cache) |
| 955 | return 0; /* ignore calls from kvm_set_spte_hva */ |
| 956 | pte = mmu_memory_cache_alloc(cache); |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 957 | kvm_clean_pte(pte); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 958 | pmd_populate_kernel(NULL, pmd, pte); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 959 | get_page(virt_to_page(pmd)); |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 960 | } |
| 961 | |
| 962 | pte = pte_offset_kernel(pmd, addr); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 963 | |
| 964 | if (iomap && pte_present(*pte)) |
| 965 | return -EFAULT; |
| 966 | |
| 967 | /* Create 2nd stage page table mapping - Level 3 */ |
| 968 | old_pte = *pte; |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 969 | if (pte_present(old_pte)) { |
| 970 | kvm_set_pte(pte, __pte(0)); |
Marc Zyngier | 4876276 | 2013-01-28 15:27:00 +0000 | [diff] [blame] | 971 | kvm_tlb_flush_vmid_ipa(kvm, addr); |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 972 | } else { |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 973 | get_page(virt_to_page(pte)); |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 974 | } |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 975 | |
Marc Zyngier | d4b9e07 | 2016-04-28 16:16:31 +0100 | [diff] [blame] | 976 | kvm_set_pte(pte, *new_pte); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 977 | return 0; |
| 978 | } |
| 979 | |
| 980 | /** |
| 981 | * kvm_phys_addr_ioremap - map a device range to guest IPA |
| 982 | * |
| 983 | * @kvm: The KVM pointer |
| 984 | * @guest_ipa: The IPA at which to insert the mapping |
| 985 | * @pa: The physical address of the device |
| 986 | * @size: The size of the mapping |
| 987 | */ |
| 988 | int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, |
Ard Biesheuvel | c40f2f8 | 2014-09-17 14:56:18 -0700 | [diff] [blame] | 989 | phys_addr_t pa, unsigned long size, bool writable) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 990 | { |
| 991 | phys_addr_t addr, end; |
| 992 | int ret = 0; |
| 993 | unsigned long pfn; |
| 994 | struct kvm_mmu_memory_cache cache = { 0, }; |
| 995 | |
| 996 | end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK; |
| 997 | pfn = __phys_to_pfn(pa); |
| 998 | |
| 999 | for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) { |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 1000 | pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1001 | |
Ard Biesheuvel | c40f2f8 | 2014-09-17 14:56:18 -0700 | [diff] [blame] | 1002 | if (writable) |
| 1003 | kvm_set_s2pte_writable(&pte); |
| 1004 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 1005 | ret = mmu_topup_memory_cache(&cache, KVM_MMU_CACHE_MIN_PAGES, |
| 1006 | KVM_NR_MEM_OBJS); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1007 | if (ret) |
| 1008 | goto out; |
| 1009 | spin_lock(&kvm->mmu_lock); |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1010 | ret = stage2_set_pte(kvm, &cache, addr, &pte, |
| 1011 | KVM_S2PTE_FLAG_IS_IOMAP); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1012 | spin_unlock(&kvm->mmu_lock); |
| 1013 | if (ret) |
| 1014 | goto out; |
| 1015 | |
| 1016 | pfn++; |
| 1017 | } |
| 1018 | |
| 1019 | out: |
| 1020 | mmu_free_memory_cache(&cache); |
| 1021 | return ret; |
| 1022 | } |
| 1023 | |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 1024 | static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap) |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1025 | { |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 1026 | kvm_pfn_t pfn = *pfnp; |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1027 | gfn_t gfn = *ipap >> PAGE_SHIFT; |
| 1028 | |
| 1029 | if (PageTransCompound(pfn_to_page(pfn))) { |
| 1030 | unsigned long mask; |
| 1031 | /* |
| 1032 | * The address we faulted on is backed by a transparent huge |
| 1033 | * page. However, because we map the compound huge page and |
| 1034 | * not the individual tail page, we need to transfer the |
| 1035 | * refcount to the head page. We have to be careful that the |
| 1036 | * THP doesn't start to split while we are adjusting the |
| 1037 | * refcounts. |
| 1038 | * |
| 1039 | * We are sure this doesn't happen, because mmu_notifier_retry |
| 1040 | * was successful and we are holding the mmu_lock, so if this |
| 1041 | * THP is trying to split, it will be blocked in the mmu |
| 1042 | * notifier before touching any of the pages, specifically |
| 1043 | * before being able to call __split_huge_page_refcount(). |
| 1044 | * |
| 1045 | * We can therefore safely transfer the refcount from PG_tail |
| 1046 | * to PG_head and switch the pfn from a tail page to the head |
| 1047 | * page accordingly. |
| 1048 | */ |
| 1049 | mask = PTRS_PER_PMD - 1; |
| 1050 | VM_BUG_ON((gfn & mask) != (pfn & mask)); |
| 1051 | if (pfn & mask) { |
| 1052 | *ipap &= PMD_MASK; |
| 1053 | kvm_release_pfn_clean(pfn); |
| 1054 | pfn &= ~mask; |
| 1055 | kvm_get_pfn(pfn); |
| 1056 | *pfnp = pfn; |
| 1057 | } |
| 1058 | |
| 1059 | return true; |
| 1060 | } |
| 1061 | |
| 1062 | return false; |
| 1063 | } |
| 1064 | |
Ard Biesheuvel | a7d079c | 2014-09-09 11:27:09 +0100 | [diff] [blame] | 1065 | static bool kvm_is_write_fault(struct kvm_vcpu *vcpu) |
| 1066 | { |
| 1067 | if (kvm_vcpu_trap_is_iabt(vcpu)) |
| 1068 | return false; |
| 1069 | |
| 1070 | return kvm_vcpu_dabt_iswrite(vcpu); |
| 1071 | } |
| 1072 | |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1073 | /** |
| 1074 | * stage2_wp_ptes - write protect PMD range |
| 1075 | * @pmd: pointer to pmd entry |
| 1076 | * @addr: range start address |
| 1077 | * @end: range end address |
| 1078 | */ |
| 1079 | static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) |
| 1080 | { |
| 1081 | pte_t *pte; |
| 1082 | |
| 1083 | pte = pte_offset_kernel(pmd, addr); |
| 1084 | do { |
| 1085 | if (!pte_none(*pte)) { |
| 1086 | if (!kvm_s2pte_readonly(pte)) |
| 1087 | kvm_set_s2pte_readonly(pte); |
| 1088 | } |
| 1089 | } while (pte++, addr += PAGE_SIZE, addr != end); |
| 1090 | } |
| 1091 | |
| 1092 | /** |
| 1093 | * stage2_wp_pmds - write protect PUD range |
| 1094 | * @pud: pointer to pud entry |
| 1095 | * @addr: range start address |
| 1096 | * @end: range end address |
| 1097 | */ |
| 1098 | static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) |
| 1099 | { |
| 1100 | pmd_t *pmd; |
| 1101 | phys_addr_t next; |
| 1102 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1103 | pmd = stage2_pmd_offset(pud, addr); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1104 | |
| 1105 | do { |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1106 | next = stage2_pmd_addr_end(addr, end); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1107 | if (!pmd_none(*pmd)) { |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 1108 | if (pmd_thp_or_huge(*pmd)) { |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1109 | if (!kvm_s2pmd_readonly(pmd)) |
| 1110 | kvm_set_s2pmd_readonly(pmd); |
| 1111 | } else { |
| 1112 | stage2_wp_ptes(pmd, addr, next); |
| 1113 | } |
| 1114 | } |
| 1115 | } while (pmd++, addr = next, addr != end); |
| 1116 | } |
| 1117 | |
| 1118 | /** |
| 1119 | * stage2_wp_puds - write protect PGD range |
| 1120 | * @pgd: pointer to pgd entry |
| 1121 | * @addr: range start address |
| 1122 | * @end: range end address |
| 1123 | * |
| 1124 | * Process PUD entries, for a huge PUD we cause a panic. |
| 1125 | */ |
| 1126 | static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end) |
| 1127 | { |
| 1128 | pud_t *pud; |
| 1129 | phys_addr_t next; |
| 1130 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1131 | pud = stage2_pud_offset(pgd, addr); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1132 | do { |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1133 | next = stage2_pud_addr_end(addr, end); |
| 1134 | if (!stage2_pud_none(*pud)) { |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1135 | /* TODO:PUD not supported, revisit later if supported */ |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1136 | BUG_ON(stage2_pud_huge(*pud)); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1137 | stage2_wp_pmds(pud, addr, next); |
| 1138 | } |
| 1139 | } while (pud++, addr = next, addr != end); |
| 1140 | } |
| 1141 | |
| 1142 | /** |
| 1143 | * stage2_wp_range() - write protect stage2 memory region range |
| 1144 | * @kvm: The KVM pointer |
| 1145 | * @addr: Start address of range |
| 1146 | * @end: End address of range |
| 1147 | */ |
| 1148 | static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) |
| 1149 | { |
| 1150 | pgd_t *pgd; |
| 1151 | phys_addr_t next; |
| 1152 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1153 | pgd = kvm->arch.pgd + stage2_pgd_index(addr); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1154 | do { |
| 1155 | /* |
| 1156 | * Release kvm_mmu_lock periodically if the memory region is |
| 1157 | * large. Otherwise, we may see kernel panics with |
Christoffer Dall | 227ea81 | 2015-01-23 10:49:31 +0100 | [diff] [blame] | 1158 | * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR, |
| 1159 | * CONFIG_LOCKDEP. Additionally, holding the lock too long |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1160 | * will also starve other vCPUs. |
| 1161 | */ |
| 1162 | if (need_resched() || spin_needbreak(&kvm->mmu_lock)) |
| 1163 | cond_resched_lock(&kvm->mmu_lock); |
| 1164 | |
Suzuki K Poulose | 70fd190 | 2016-03-22 18:33:45 +0000 | [diff] [blame] | 1165 | next = stage2_pgd_addr_end(addr, end); |
| 1166 | if (stage2_pgd_present(*pgd)) |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1167 | stage2_wp_puds(pgd, addr, next); |
| 1168 | } while (pgd++, addr = next, addr != end); |
| 1169 | } |
| 1170 | |
| 1171 | /** |
| 1172 | * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot |
| 1173 | * @kvm: The KVM pointer |
| 1174 | * @slot: The memory slot to write protect |
| 1175 | * |
| 1176 | * Called to start logging dirty pages after memory region |
| 1177 | * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns |
| 1178 | * all present PMD and PTEs are write protected in the memory region. |
| 1179 | * Afterwards read of dirty page log can be called. |
| 1180 | * |
| 1181 | * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired, |
| 1182 | * serializing operations for VM memory regions. |
| 1183 | */ |
| 1184 | void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) |
| 1185 | { |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 1186 | struct kvm_memslots *slots = kvm_memslots(kvm); |
| 1187 | struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1188 | phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; |
| 1189 | phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; |
| 1190 | |
| 1191 | spin_lock(&kvm->mmu_lock); |
| 1192 | stage2_wp_range(kvm, start, end); |
| 1193 | spin_unlock(&kvm->mmu_lock); |
| 1194 | kvm_flush_remote_tlbs(kvm); |
| 1195 | } |
Mario Smarduch | 53c810c | 2015-01-15 15:58:57 -0800 | [diff] [blame] | 1196 | |
| 1197 | /** |
Kai Huang | 3b0f1d0 | 2015-01-28 10:54:23 +0800 | [diff] [blame] | 1198 | * kvm_mmu_write_protect_pt_masked() - write protect dirty pages |
Mario Smarduch | 53c810c | 2015-01-15 15:58:57 -0800 | [diff] [blame] | 1199 | * @kvm: The KVM pointer |
| 1200 | * @slot: The memory slot associated with mask |
| 1201 | * @gfn_offset: The gfn offset in memory slot |
| 1202 | * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory |
| 1203 | * slot to be write protected |
| 1204 | * |
| 1205 | * Walks bits set in mask write protects the associated pte's. Caller must |
| 1206 | * acquire kvm_mmu_lock. |
| 1207 | */ |
Kai Huang | 3b0f1d0 | 2015-01-28 10:54:23 +0800 | [diff] [blame] | 1208 | static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, |
Mario Smarduch | 53c810c | 2015-01-15 15:58:57 -0800 | [diff] [blame] | 1209 | struct kvm_memory_slot *slot, |
| 1210 | gfn_t gfn_offset, unsigned long mask) |
| 1211 | { |
| 1212 | phys_addr_t base_gfn = slot->base_gfn + gfn_offset; |
| 1213 | phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; |
| 1214 | phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; |
| 1215 | |
| 1216 | stage2_wp_range(kvm, start, end); |
| 1217 | } |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1218 | |
Kai Huang | 3b0f1d0 | 2015-01-28 10:54:23 +0800 | [diff] [blame] | 1219 | /* |
| 1220 | * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected |
| 1221 | * dirty pages. |
| 1222 | * |
| 1223 | * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to |
| 1224 | * enable dirty logging for them. |
| 1225 | */ |
| 1226 | void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, |
| 1227 | struct kvm_memory_slot *slot, |
| 1228 | gfn_t gfn_offset, unsigned long mask) |
| 1229 | { |
| 1230 | kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); |
| 1231 | } |
| 1232 | |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 1233 | static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn, |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 1234 | unsigned long size, bool uncached) |
| 1235 | { |
| 1236 | __coherent_cache_guest_page(vcpu, pfn, size, uncached); |
| 1237 | } |
| 1238 | |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1239 | static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, |
Christoffer Dall | 9804788 | 2014-08-19 12:18:04 +0200 | [diff] [blame] | 1240 | struct kvm_memory_slot *memslot, unsigned long hva, |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1241 | unsigned long fault_status) |
| 1242 | { |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1243 | int ret; |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1244 | bool write_fault, writable, hugetlb = false, force_pte = false; |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1245 | unsigned long mmu_seq; |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1246 | gfn_t gfn = fault_ipa >> PAGE_SHIFT; |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1247 | struct kvm *kvm = vcpu->kvm; |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1248 | struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1249 | struct vm_area_struct *vma; |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 1250 | kvm_pfn_t pfn; |
Kim Phillips | b886576 | 2014-06-26 01:45:51 +0100 | [diff] [blame] | 1251 | pgprot_t mem_type = PAGE_S2; |
Laszlo Ersek | 840f4bf | 2014-11-17 14:58:52 +0000 | [diff] [blame] | 1252 | bool fault_ipa_uncached; |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1253 | bool logging_active = memslot_is_logging(memslot); |
| 1254 | unsigned long flags = 0; |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1255 | |
Ard Biesheuvel | a7d079c | 2014-09-09 11:27:09 +0100 | [diff] [blame] | 1256 | write_fault = kvm_is_write_fault(vcpu); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1257 | if (fault_status == FSC_PERM && !write_fault) { |
| 1258 | kvm_err("Unexpected L2 read permission error\n"); |
| 1259 | return -EFAULT; |
| 1260 | } |
| 1261 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1262 | /* Let's check if we will get back a huge page backed by hugetlbfs */ |
| 1263 | down_read(¤t->mm->mmap_sem); |
| 1264 | vma = find_vma_intersection(current->mm, hva, hva + 1); |
Ard Biesheuvel | 37b5440 | 2014-09-17 14:56:17 -0700 | [diff] [blame] | 1265 | if (unlikely(!vma)) { |
| 1266 | kvm_err("Failed to find VMA for hva 0x%lx\n", hva); |
| 1267 | up_read(¤t->mm->mmap_sem); |
| 1268 | return -EFAULT; |
| 1269 | } |
| 1270 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1271 | if (is_vm_hugetlb_page(vma) && !logging_active) { |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1272 | hugetlb = true; |
| 1273 | gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT; |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1274 | } else { |
| 1275 | /* |
Marc Zyngier | 136d737 | 2013-12-13 16:56:06 +0000 | [diff] [blame] | 1276 | * Pages belonging to memslots that don't have the same |
| 1277 | * alignment for userspace and IPA cannot be mapped using |
| 1278 | * block descriptors even if the pages belong to a THP for |
| 1279 | * the process, because the stage-2 block descriptor will |
| 1280 | * cover more than a single THP and we loose atomicity for |
| 1281 | * unmapping, updates, and splits of the THP or other pages |
| 1282 | * in the stage-2 block range. |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1283 | */ |
Marc Zyngier | 136d737 | 2013-12-13 16:56:06 +0000 | [diff] [blame] | 1284 | if ((memslot->userspace_addr & ~PMD_MASK) != |
| 1285 | ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK)) |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1286 | force_pte = true; |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1287 | } |
| 1288 | up_read(¤t->mm->mmap_sem); |
| 1289 | |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1290 | /* We need minimum second+third level pages */ |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 1291 | ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, |
| 1292 | KVM_NR_MEM_OBJS); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1293 | if (ret) |
| 1294 | return ret; |
| 1295 | |
| 1296 | mmu_seq = vcpu->kvm->mmu_notifier_seq; |
| 1297 | /* |
| 1298 | * Ensure the read of mmu_notifier_seq happens before we call |
| 1299 | * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk |
| 1300 | * the page we just got a reference to gets unmapped before we have a |
| 1301 | * chance to grab the mmu_lock, which ensure that if the page gets |
| 1302 | * unmapped afterwards, the call to kvm_unmap_hva will take it away |
| 1303 | * from us again properly. This smp_rmb() interacts with the smp_wmb() |
| 1304 | * in kvm_mmu_notifier_invalidate_<page|range_end>. |
| 1305 | */ |
| 1306 | smp_rmb(); |
| 1307 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1308 | pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1309 | if (is_error_pfn(pfn)) |
| 1310 | return -EFAULT; |
| 1311 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1312 | if (kvm_is_device_pfn(pfn)) { |
Kim Phillips | b886576 | 2014-06-26 01:45:51 +0100 | [diff] [blame] | 1313 | mem_type = PAGE_S2_DEVICE; |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1314 | flags |= KVM_S2PTE_FLAG_IS_IOMAP; |
| 1315 | } else if (logging_active) { |
| 1316 | /* |
| 1317 | * Faults on pages in a memslot with logging enabled |
| 1318 | * should not be mapped with huge pages (it introduces churn |
| 1319 | * and performance degradation), so force a pte mapping. |
| 1320 | */ |
| 1321 | force_pte = true; |
| 1322 | flags |= KVM_S2_FLAG_LOGGING_ACTIVE; |
| 1323 | |
| 1324 | /* |
| 1325 | * Only actually map the page as writable if this was a write |
| 1326 | * fault. |
| 1327 | */ |
| 1328 | if (!write_fault) |
| 1329 | writable = false; |
| 1330 | } |
Kim Phillips | b886576 | 2014-06-26 01:45:51 +0100 | [diff] [blame] | 1331 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1332 | spin_lock(&kvm->mmu_lock); |
| 1333 | if (mmu_notifier_retry(kvm, mmu_seq)) |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1334 | goto out_unlock; |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1335 | |
Christoffer Dall | 9b5fdb9 | 2013-10-02 15:32:01 -0700 | [diff] [blame] | 1336 | if (!hugetlb && !force_pte) |
| 1337 | hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1338 | |
Ard Biesheuvel | 849260c | 2014-11-17 14:58:53 +0000 | [diff] [blame] | 1339 | fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT; |
Laszlo Ersek | 840f4bf | 2014-11-17 14:58:52 +0000 | [diff] [blame] | 1340 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1341 | if (hugetlb) { |
Kim Phillips | b886576 | 2014-06-26 01:45:51 +0100 | [diff] [blame] | 1342 | pmd_t new_pmd = pfn_pmd(pfn, mem_type); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1343 | new_pmd = pmd_mkhuge(new_pmd); |
| 1344 | if (writable) { |
| 1345 | kvm_set_s2pmd_writable(&new_pmd); |
| 1346 | kvm_set_pfn_dirty(pfn); |
| 1347 | } |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 1348 | coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1349 | ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd); |
| 1350 | } else { |
Kim Phillips | b886576 | 2014-06-26 01:45:51 +0100 | [diff] [blame] | 1351 | pte_t new_pte = pfn_pte(pfn, mem_type); |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1352 | |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1353 | if (writable) { |
| 1354 | kvm_set_s2pte_writable(&new_pte); |
| 1355 | kvm_set_pfn_dirty(pfn); |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1356 | mark_page_dirty(kvm, gfn); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1357 | } |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 1358 | coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached); |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1359 | ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1360 | } |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1361 | |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1362 | out_unlock: |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1363 | spin_unlock(&kvm->mmu_lock); |
Marc Zyngier | 35307b9 | 2015-03-12 18:16:51 +0000 | [diff] [blame] | 1364 | kvm_set_pfn_accessed(pfn); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1365 | kvm_release_pfn_clean(pfn); |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 1366 | return ret; |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1367 | } |
| 1368 | |
Marc Zyngier | aeda913 | 2015-03-12 18:16:52 +0000 | [diff] [blame] | 1369 | /* |
| 1370 | * Resolve the access fault by making the page young again. |
| 1371 | * Note that because the faulting entry is guaranteed not to be |
| 1372 | * cached in the TLB, we don't need to invalidate anything. |
| 1373 | */ |
| 1374 | static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) |
| 1375 | { |
| 1376 | pmd_t *pmd; |
| 1377 | pte_t *pte; |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 1378 | kvm_pfn_t pfn; |
Marc Zyngier | aeda913 | 2015-03-12 18:16:52 +0000 | [diff] [blame] | 1379 | bool pfn_valid = false; |
| 1380 | |
| 1381 | trace_kvm_access_fault(fault_ipa); |
| 1382 | |
| 1383 | spin_lock(&vcpu->kvm->mmu_lock); |
| 1384 | |
| 1385 | pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa); |
| 1386 | if (!pmd || pmd_none(*pmd)) /* Nothing there */ |
| 1387 | goto out; |
| 1388 | |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 1389 | if (pmd_thp_or_huge(*pmd)) { /* THP, HugeTLB */ |
Marc Zyngier | aeda913 | 2015-03-12 18:16:52 +0000 | [diff] [blame] | 1390 | *pmd = pmd_mkyoung(*pmd); |
| 1391 | pfn = pmd_pfn(*pmd); |
| 1392 | pfn_valid = true; |
| 1393 | goto out; |
| 1394 | } |
| 1395 | |
| 1396 | pte = pte_offset_kernel(pmd, fault_ipa); |
| 1397 | if (pte_none(*pte)) /* Nothing there either */ |
| 1398 | goto out; |
| 1399 | |
| 1400 | *pte = pte_mkyoung(*pte); /* Just a page... */ |
| 1401 | pfn = pte_pfn(*pte); |
| 1402 | pfn_valid = true; |
| 1403 | out: |
| 1404 | spin_unlock(&vcpu->kvm->mmu_lock); |
| 1405 | if (pfn_valid) |
| 1406 | kvm_set_pfn_accessed(pfn); |
| 1407 | } |
| 1408 | |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1409 | /** |
| 1410 | * kvm_handle_guest_abort - handles all 2nd stage aborts |
| 1411 | * @vcpu: the VCPU pointer |
| 1412 | * @run: the kvm_run structure |
| 1413 | * |
| 1414 | * Any abort that gets to the host is almost guaranteed to be caused by a |
| 1415 | * missing second stage translation table entry, which can mean that either the |
| 1416 | * guest simply needs more memory and we must allocate an appropriate page or it |
| 1417 | * can mean that the guest tried to access I/O memory, which is emulated by user |
| 1418 | * space. The distinction is based on the IPA causing the fault and whether this |
| 1419 | * memory region has been registered as standard RAM by user space. |
| 1420 | */ |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1421 | int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) |
| 1422 | { |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1423 | unsigned long fault_status; |
| 1424 | phys_addr_t fault_ipa; |
| 1425 | struct kvm_memory_slot *memslot; |
Christoffer Dall | 9804788 | 2014-08-19 12:18:04 +0200 | [diff] [blame] | 1426 | unsigned long hva; |
| 1427 | bool is_iabt, write_fault, writable; |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1428 | gfn_t gfn; |
| 1429 | int ret, idx; |
| 1430 | |
Marc Zyngier | 52d1dba | 2012-10-15 10:33:38 +0100 | [diff] [blame] | 1431 | is_iabt = kvm_vcpu_trap_is_iabt(vcpu); |
Marc Zyngier | 7393b59 | 2012-09-17 19:27:09 +0100 | [diff] [blame] | 1432 | fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1433 | |
Marc Zyngier | 7393b59 | 2012-09-17 19:27:09 +0100 | [diff] [blame] | 1434 | trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu), |
| 1435 | kvm_vcpu_get_hfar(vcpu), fault_ipa); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1436 | |
| 1437 | /* Check the stage-2 fault is trans. fault or write fault */ |
Christoffer Dall | 0496daa5 | 2014-09-26 12:29:34 +0200 | [diff] [blame] | 1438 | fault_status = kvm_vcpu_trap_get_fault_type(vcpu); |
Marc Zyngier | 35307b9 | 2015-03-12 18:16:51 +0000 | [diff] [blame] | 1439 | if (fault_status != FSC_FAULT && fault_status != FSC_PERM && |
| 1440 | fault_status != FSC_ACCESS) { |
Christoffer Dall | 0496daa5 | 2014-09-26 12:29:34 +0200 | [diff] [blame] | 1441 | kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", |
| 1442 | kvm_vcpu_trap_get_class(vcpu), |
| 1443 | (unsigned long)kvm_vcpu_trap_get_fault(vcpu), |
| 1444 | (unsigned long)kvm_vcpu_get_hsr(vcpu)); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1445 | return -EFAULT; |
| 1446 | } |
| 1447 | |
| 1448 | idx = srcu_read_lock(&vcpu->kvm->srcu); |
| 1449 | |
| 1450 | gfn = fault_ipa >> PAGE_SHIFT; |
Christoffer Dall | 9804788 | 2014-08-19 12:18:04 +0200 | [diff] [blame] | 1451 | memslot = gfn_to_memslot(vcpu->kvm, gfn); |
| 1452 | hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); |
Ard Biesheuvel | a7d079c | 2014-09-09 11:27:09 +0100 | [diff] [blame] | 1453 | write_fault = kvm_is_write_fault(vcpu); |
Christoffer Dall | 9804788 | 2014-08-19 12:18:04 +0200 | [diff] [blame] | 1454 | if (kvm_is_error_hva(hva) || (write_fault && !writable)) { |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1455 | if (is_iabt) { |
| 1456 | /* Prefetch Abort on I/O address */ |
Marc Zyngier | 7393b59 | 2012-09-17 19:27:09 +0100 | [diff] [blame] | 1457 | kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu)); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1458 | ret = 1; |
| 1459 | goto out_unlock; |
| 1460 | } |
| 1461 | |
Marc Zyngier | cfe3950 | 2012-12-12 14:42:09 +0000 | [diff] [blame] | 1462 | /* |
Marc Zyngier | 57c841f | 2016-01-29 15:01:28 +0000 | [diff] [blame] | 1463 | * Check for a cache maintenance operation. Since we |
| 1464 | * ended-up here, we know it is outside of any memory |
| 1465 | * slot. But we can't find out if that is for a device, |
| 1466 | * or if the guest is just being stupid. The only thing |
| 1467 | * we know for sure is that this range cannot be cached. |
| 1468 | * |
| 1469 | * So let's assume that the guest is just being |
| 1470 | * cautious, and skip the instruction. |
| 1471 | */ |
| 1472 | if (kvm_vcpu_dabt_is_cm(vcpu)) { |
| 1473 | kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); |
| 1474 | ret = 1; |
| 1475 | goto out_unlock; |
| 1476 | } |
| 1477 | |
| 1478 | /* |
Marc Zyngier | cfe3950 | 2012-12-12 14:42:09 +0000 | [diff] [blame] | 1479 | * The IPA is reported as [MAX:12], so we need to |
| 1480 | * complement it with the bottom 12 bits from the |
| 1481 | * faulting VA. This is always 12 bits, irrespective |
| 1482 | * of the page size. |
| 1483 | */ |
| 1484 | fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); |
Christoffer Dall | 45e96ea | 2013-01-20 18:43:58 -0500 | [diff] [blame] | 1485 | ret = io_mem_abort(vcpu, run, fault_ipa); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1486 | goto out_unlock; |
| 1487 | } |
| 1488 | |
Christoffer Dall | c3058d5 | 2014-10-10 12:14:29 +0200 | [diff] [blame] | 1489 | /* Userspace should not be able to register out-of-bounds IPAs */ |
| 1490 | VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE); |
| 1491 | |
Marc Zyngier | aeda913 | 2015-03-12 18:16:52 +0000 | [diff] [blame] | 1492 | if (fault_status == FSC_ACCESS) { |
| 1493 | handle_access_fault(vcpu, fault_ipa); |
| 1494 | ret = 1; |
| 1495 | goto out_unlock; |
| 1496 | } |
| 1497 | |
Christoffer Dall | 9804788 | 2014-08-19 12:18:04 +0200 | [diff] [blame] | 1498 | ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 1499 | if (ret == 0) |
| 1500 | ret = 1; |
| 1501 | out_unlock: |
| 1502 | srcu_read_unlock(&vcpu->kvm->srcu, idx); |
| 1503 | return ret; |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1504 | } |
| 1505 | |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1506 | static int handle_hva_to_gpa(struct kvm *kvm, |
| 1507 | unsigned long start, |
| 1508 | unsigned long end, |
| 1509 | int (*handler)(struct kvm *kvm, |
| 1510 | gpa_t gpa, void *data), |
| 1511 | void *data) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1512 | { |
| 1513 | struct kvm_memslots *slots; |
| 1514 | struct kvm_memory_slot *memslot; |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1515 | int ret = 0; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1516 | |
| 1517 | slots = kvm_memslots(kvm); |
| 1518 | |
| 1519 | /* we only care about the pages that the guest sees */ |
| 1520 | kvm_for_each_memslot(memslot, slots) { |
| 1521 | unsigned long hva_start, hva_end; |
| 1522 | gfn_t gfn, gfn_end; |
| 1523 | |
| 1524 | hva_start = max(start, memslot->userspace_addr); |
| 1525 | hva_end = min(end, memslot->userspace_addr + |
| 1526 | (memslot->npages << PAGE_SHIFT)); |
| 1527 | if (hva_start >= hva_end) |
| 1528 | continue; |
| 1529 | |
| 1530 | /* |
| 1531 | * {gfn(page) | page intersects with [hva_start, hva_end)} = |
| 1532 | * {gfn_start, gfn_start+1, ..., gfn_end-1}. |
| 1533 | */ |
| 1534 | gfn = hva_to_gfn_memslot(hva_start, memslot); |
| 1535 | gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); |
| 1536 | |
| 1537 | for (; gfn < gfn_end; ++gfn) { |
| 1538 | gpa_t gpa = gfn << PAGE_SHIFT; |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1539 | ret |= handler(kvm, gpa, data); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1540 | } |
| 1541 | } |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1542 | |
| 1543 | return ret; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1544 | } |
| 1545 | |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1546 | static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1547 | { |
| 1548 | unmap_stage2_range(kvm, gpa, PAGE_SIZE); |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1549 | return 0; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1550 | } |
| 1551 | |
| 1552 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) |
| 1553 | { |
| 1554 | unsigned long end = hva + PAGE_SIZE; |
| 1555 | |
| 1556 | if (!kvm->arch.pgd) |
| 1557 | return 0; |
| 1558 | |
| 1559 | trace_kvm_unmap_hva(hva); |
| 1560 | handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL); |
| 1561 | return 0; |
| 1562 | } |
| 1563 | |
| 1564 | int kvm_unmap_hva_range(struct kvm *kvm, |
| 1565 | unsigned long start, unsigned long end) |
| 1566 | { |
| 1567 | if (!kvm->arch.pgd) |
| 1568 | return 0; |
| 1569 | |
| 1570 | trace_kvm_unmap_hva_range(start, end); |
| 1571 | handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); |
| 1572 | return 0; |
| 1573 | } |
| 1574 | |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1575 | static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data) |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1576 | { |
| 1577 | pte_t *pte = (pte_t *)data; |
| 1578 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1579 | /* |
| 1580 | * We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE |
| 1581 | * flag clear because MMU notifiers will have unmapped a huge PMD before |
| 1582 | * calling ->change_pte() (which in turn calls kvm_set_spte_hva()) and |
| 1583 | * therefore stage2_set_pte() never needs to clear out a huge PMD |
| 1584 | * through this calling path. |
| 1585 | */ |
| 1586 | stage2_set_pte(kvm, NULL, gpa, pte, 0); |
Marc Zyngier | 1d2ebac | 2015-03-12 18:16:50 +0000 | [diff] [blame] | 1587 | return 0; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1588 | } |
| 1589 | |
| 1590 | |
| 1591 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) |
| 1592 | { |
| 1593 | unsigned long end = hva + PAGE_SIZE; |
| 1594 | pte_t stage2_pte; |
| 1595 | |
| 1596 | if (!kvm->arch.pgd) |
| 1597 | return; |
| 1598 | |
| 1599 | trace_kvm_set_spte_hva(hva); |
| 1600 | stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2); |
| 1601 | handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte); |
| 1602 | } |
| 1603 | |
Marc Zyngier | 35307b9 | 2015-03-12 18:16:51 +0000 | [diff] [blame] | 1604 | static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) |
| 1605 | { |
| 1606 | pmd_t *pmd; |
| 1607 | pte_t *pte; |
| 1608 | |
| 1609 | pmd = stage2_get_pmd(kvm, NULL, gpa); |
| 1610 | if (!pmd || pmd_none(*pmd)) /* Nothing there */ |
| 1611 | return 0; |
| 1612 | |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 1613 | if (pmd_thp_or_huge(*pmd)) { /* THP, HugeTLB */ |
Marc Zyngier | 35307b9 | 2015-03-12 18:16:51 +0000 | [diff] [blame] | 1614 | if (pmd_young(*pmd)) { |
| 1615 | *pmd = pmd_mkold(*pmd); |
| 1616 | return 1; |
| 1617 | } |
| 1618 | |
| 1619 | return 0; |
| 1620 | } |
| 1621 | |
| 1622 | pte = pte_offset_kernel(pmd, gpa); |
| 1623 | if (pte_none(*pte)) |
| 1624 | return 0; |
| 1625 | |
| 1626 | if (pte_young(*pte)) { |
| 1627 | *pte = pte_mkold(*pte); /* Just a page... */ |
| 1628 | return 1; |
| 1629 | } |
| 1630 | |
| 1631 | return 0; |
| 1632 | } |
| 1633 | |
| 1634 | static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) |
| 1635 | { |
| 1636 | pmd_t *pmd; |
| 1637 | pte_t *pte; |
| 1638 | |
| 1639 | pmd = stage2_get_pmd(kvm, NULL, gpa); |
| 1640 | if (!pmd || pmd_none(*pmd)) /* Nothing there */ |
| 1641 | return 0; |
| 1642 | |
Suzuki K Poulose | bbb3b6b | 2016-03-01 12:00:39 +0000 | [diff] [blame] | 1643 | if (pmd_thp_or_huge(*pmd)) /* THP, HugeTLB */ |
Marc Zyngier | 35307b9 | 2015-03-12 18:16:51 +0000 | [diff] [blame] | 1644 | return pmd_young(*pmd); |
| 1645 | |
| 1646 | pte = pte_offset_kernel(pmd, gpa); |
| 1647 | if (!pte_none(*pte)) /* Just a page... */ |
| 1648 | return pte_young(*pte); |
| 1649 | |
| 1650 | return 0; |
| 1651 | } |
| 1652 | |
| 1653 | int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) |
| 1654 | { |
| 1655 | trace_kvm_age_hva(start, end); |
| 1656 | return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); |
| 1657 | } |
| 1658 | |
| 1659 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) |
| 1660 | { |
| 1661 | trace_kvm_test_age_hva(hva); |
| 1662 | return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); |
| 1663 | } |
| 1664 | |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1665 | void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) |
| 1666 | { |
| 1667 | mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); |
| 1668 | } |
| 1669 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1670 | phys_addr_t kvm_mmu_get_httbr(void) |
| 1671 | { |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 1672 | if (__kvm_cpu_uses_extended_idmap()) |
| 1673 | return virt_to_phys(merged_hyp_pgd); |
| 1674 | else |
| 1675 | return virt_to_phys(hyp_pgd); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1676 | } |
| 1677 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1678 | phys_addr_t kvm_mmu_get_boot_httbr(void) |
| 1679 | { |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 1680 | if (__kvm_cpu_uses_extended_idmap()) |
| 1681 | return virt_to_phys(merged_hyp_pgd); |
| 1682 | else |
| 1683 | return virt_to_phys(boot_hyp_pgd); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1684 | } |
| 1685 | |
| 1686 | phys_addr_t kvm_get_idmap_vector(void) |
| 1687 | { |
| 1688 | return hyp_idmap_vector; |
| 1689 | } |
| 1690 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1691 | int kvm_mmu_init(void) |
| 1692 | { |
Marc Zyngier | 2fb4105 | 2013-04-12 19:12:03 +0100 | [diff] [blame] | 1693 | int err; |
| 1694 | |
Santosh Shilimkar | 4fda342 | 2013-11-19 14:59:12 -0500 | [diff] [blame] | 1695 | hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start); |
| 1696 | hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end); |
| 1697 | hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1698 | |
Ard Biesheuvel | 06f75a1 | 2015-03-19 16:42:26 +0000 | [diff] [blame] | 1699 | /* |
| 1700 | * We rely on the linker script to ensure at build time that the HYP |
| 1701 | * init code does not cross a page boundary. |
| 1702 | */ |
| 1703 | BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1704 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 1705 | hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); |
| 1706 | boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); |
Mark Salter | 5d4e08c | 2014-03-28 14:25:19 +0000 | [diff] [blame] | 1707 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1708 | if (!hyp_pgd || !boot_hyp_pgd) { |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1709 | kvm_err("Hyp mode PGD not allocated\n"); |
Marc Zyngier | 2fb4105 | 2013-04-12 19:12:03 +0100 | [diff] [blame] | 1710 | err = -ENOMEM; |
| 1711 | goto out; |
| 1712 | } |
| 1713 | |
| 1714 | /* Create the idmap in the boot page tables */ |
| 1715 | err = __create_hyp_mappings(boot_hyp_pgd, |
| 1716 | hyp_idmap_start, hyp_idmap_end, |
| 1717 | __phys_to_pfn(hyp_idmap_start), |
| 1718 | PAGE_HYP); |
| 1719 | |
| 1720 | if (err) { |
| 1721 | kvm_err("Failed to idmap %lx-%lx\n", |
| 1722 | hyp_idmap_start, hyp_idmap_end); |
| 1723 | goto out; |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1724 | } |
| 1725 | |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 1726 | if (__kvm_cpu_uses_extended_idmap()) { |
| 1727 | merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); |
| 1728 | if (!merged_hyp_pgd) { |
| 1729 | kvm_err("Failed to allocate extra HYP pgd\n"); |
| 1730 | goto out; |
| 1731 | } |
| 1732 | __kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd, |
| 1733 | hyp_idmap_start); |
| 1734 | return 0; |
| 1735 | } |
| 1736 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 1737 | /* Map the very same page at the trampoline VA */ |
| 1738 | err = __create_hyp_mappings(boot_hyp_pgd, |
| 1739 | TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE, |
| 1740 | __phys_to_pfn(hyp_idmap_start), |
| 1741 | PAGE_HYP); |
| 1742 | if (err) { |
| 1743 | kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n", |
| 1744 | TRAMPOLINE_VA); |
| 1745 | goto out; |
| 1746 | } |
| 1747 | |
| 1748 | /* Map the same page again into the runtime page tables */ |
| 1749 | err = __create_hyp_mappings(hyp_pgd, |
| 1750 | TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE, |
| 1751 | __phys_to_pfn(hyp_idmap_start), |
| 1752 | PAGE_HYP); |
| 1753 | if (err) { |
| 1754 | kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n", |
| 1755 | TRAMPOLINE_VA); |
| 1756 | goto out; |
| 1757 | } |
| 1758 | |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 1759 | return 0; |
Marc Zyngier | 2fb4105 | 2013-04-12 19:12:03 +0100 | [diff] [blame] | 1760 | out: |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 1761 | free_hyp_pgds(); |
Marc Zyngier | 2fb4105 | 2013-04-12 19:12:03 +0100 | [diff] [blame] | 1762 | return err; |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1763 | } |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1764 | |
| 1765 | void kvm_arch_commit_memory_region(struct kvm *kvm, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 1766 | const struct kvm_userspace_memory_region *mem, |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1767 | const struct kvm_memory_slot *old, |
Paolo Bonzini | f36f3f2 | 2015-05-18 13:20:23 +0200 | [diff] [blame] | 1768 | const struct kvm_memory_slot *new, |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1769 | enum kvm_mr_change change) |
| 1770 | { |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 1771 | /* |
| 1772 | * At this point memslot has been committed and there is an |
| 1773 | * allocated dirty_bitmap[], dirty pages will be be tracked while the |
| 1774 | * memory slot is write protected. |
| 1775 | */ |
| 1776 | if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) |
| 1777 | kvm_mmu_wp_memory_region(kvm, mem->slot); |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1778 | } |
| 1779 | |
| 1780 | int kvm_arch_prepare_memory_region(struct kvm *kvm, |
| 1781 | struct kvm_memory_slot *memslot, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 1782 | const struct kvm_userspace_memory_region *mem, |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1783 | enum kvm_mr_change change) |
| 1784 | { |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1785 | hva_t hva = mem->userspace_addr; |
| 1786 | hva_t reg_end = hva + mem->memory_size; |
| 1787 | bool writable = !(mem->flags & KVM_MEM_READONLY); |
| 1788 | int ret = 0; |
| 1789 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1790 | if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && |
| 1791 | change != KVM_MR_FLAGS_ONLY) |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1792 | return 0; |
| 1793 | |
| 1794 | /* |
Christoffer Dall | c3058d5 | 2014-10-10 12:14:29 +0200 | [diff] [blame] | 1795 | * Prevent userspace from creating a memory region outside of the IPA |
| 1796 | * space addressable by the KVM guest IPA space. |
| 1797 | */ |
| 1798 | if (memslot->base_gfn + memslot->npages >= |
| 1799 | (KVM_PHYS_SIZE >> PAGE_SHIFT)) |
| 1800 | return -EFAULT; |
| 1801 | |
| 1802 | /* |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1803 | * A memory region could potentially cover multiple VMAs, and any holes |
| 1804 | * between them, so iterate over all of them to find out if we can map |
| 1805 | * any of them right now. |
| 1806 | * |
| 1807 | * +--------------------------------------------+ |
| 1808 | * +---------------+----------------+ +----------------+ |
| 1809 | * | : VMA 1 | VMA 2 | | VMA 3 : | |
| 1810 | * +---------------+----------------+ +----------------+ |
| 1811 | * | memory region | |
| 1812 | * +--------------------------------------------+ |
| 1813 | */ |
| 1814 | do { |
| 1815 | struct vm_area_struct *vma = find_vma(current->mm, hva); |
| 1816 | hva_t vm_start, vm_end; |
| 1817 | |
| 1818 | if (!vma || vma->vm_start >= reg_end) |
| 1819 | break; |
| 1820 | |
| 1821 | /* |
| 1822 | * Mapping a read-only VMA is only allowed if the |
| 1823 | * memory region is configured as read-only. |
| 1824 | */ |
| 1825 | if (writable && !(vma->vm_flags & VM_WRITE)) { |
| 1826 | ret = -EPERM; |
| 1827 | break; |
| 1828 | } |
| 1829 | |
| 1830 | /* |
| 1831 | * Take the intersection of this VMA with the memory region |
| 1832 | */ |
| 1833 | vm_start = max(hva, vma->vm_start); |
| 1834 | vm_end = min(reg_end, vma->vm_end); |
| 1835 | |
| 1836 | if (vma->vm_flags & VM_PFNMAP) { |
| 1837 | gpa_t gpa = mem->guest_phys_addr + |
| 1838 | (vm_start - mem->userspace_addr); |
Marek Majtyka | ca09f02 | 2015-09-16 12:04:55 +0200 | [diff] [blame] | 1839 | phys_addr_t pa; |
| 1840 | |
| 1841 | pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; |
| 1842 | pa += vm_start - vma->vm_start; |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1843 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1844 | /* IO region dirty page logging not allowed */ |
| 1845 | if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) |
| 1846 | return -EINVAL; |
| 1847 | |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1848 | ret = kvm_phys_addr_ioremap(kvm, gpa, pa, |
| 1849 | vm_end - vm_start, |
| 1850 | writable); |
| 1851 | if (ret) |
| 1852 | break; |
| 1853 | } |
| 1854 | hva = vm_end; |
| 1855 | } while (hva < reg_end); |
| 1856 | |
Mario Smarduch | 15a49a4 | 2015-01-15 15:58:58 -0800 | [diff] [blame] | 1857 | if (change == KVM_MR_FLAGS_ONLY) |
| 1858 | return ret; |
| 1859 | |
Ard Biesheuvel | 849260c | 2014-11-17 14:58:53 +0000 | [diff] [blame] | 1860 | spin_lock(&kvm->mmu_lock); |
| 1861 | if (ret) |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1862 | unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size); |
Ard Biesheuvel | 849260c | 2014-11-17 14:58:53 +0000 | [diff] [blame] | 1863 | else |
| 1864 | stage2_flush_memslot(kvm, memslot); |
| 1865 | spin_unlock(&kvm->mmu_lock); |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1866 | return ret; |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1867 | } |
| 1868 | |
| 1869 | void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, |
| 1870 | struct kvm_memory_slot *dont) |
| 1871 | { |
| 1872 | } |
| 1873 | |
| 1874 | int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, |
| 1875 | unsigned long npages) |
| 1876 | { |
Ard Biesheuvel | 849260c | 2014-11-17 14:58:53 +0000 | [diff] [blame] | 1877 | /* |
| 1878 | * Readonly memslots are not incoherent with the caches by definition, |
| 1879 | * but in practice, they are used mostly to emulate ROMs or NOR flashes |
| 1880 | * that the guest may consider devices and hence map as uncached. |
| 1881 | * To prevent incoherency issues in these cases, tag all readonly |
| 1882 | * regions as incoherent. |
| 1883 | */ |
| 1884 | if (slot->flags & KVM_MEM_READONLY) |
| 1885 | slot->flags |= KVM_MEMSLOT_INCOHERENT; |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1886 | return 0; |
| 1887 | } |
| 1888 | |
Paolo Bonzini | 15f4601 | 2015-05-17 21:26:08 +0200 | [diff] [blame] | 1889 | void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1890 | { |
| 1891 | } |
| 1892 | |
| 1893 | void kvm_arch_flush_shadow_all(struct kvm *kvm) |
| 1894 | { |
| 1895 | } |
| 1896 | |
| 1897 | void kvm_arch_flush_shadow_memslot(struct kvm *kvm, |
| 1898 | struct kvm_memory_slot *slot) |
| 1899 | { |
Ard Biesheuvel | 8eef912 | 2014-10-10 17:00:32 +0200 | [diff] [blame] | 1900 | gpa_t gpa = slot->base_gfn << PAGE_SHIFT; |
| 1901 | phys_addr_t size = slot->npages << PAGE_SHIFT; |
| 1902 | |
| 1903 | spin_lock(&kvm->mmu_lock); |
| 1904 | unmap_stage2_range(kvm, gpa, size); |
| 1905 | spin_unlock(&kvm->mmu_lock); |
Eric Auger | df6ce24 | 2014-06-06 11:10:23 +0200 | [diff] [blame] | 1906 | } |
Marc Zyngier | 3c1e716 | 2014-12-19 16:05:31 +0000 | [diff] [blame] | 1907 | |
| 1908 | /* |
| 1909 | * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). |
| 1910 | * |
| 1911 | * Main problems: |
| 1912 | * - S/W ops are local to a CPU (not broadcast) |
| 1913 | * - We have line migration behind our back (speculation) |
| 1914 | * - System caches don't support S/W at all (damn!) |
| 1915 | * |
| 1916 | * In the face of the above, the best we can do is to try and convert |
| 1917 | * S/W ops to VA ops. Because the guest is not allowed to infer the |
| 1918 | * S/W to PA mapping, it can only use S/W to nuke the whole cache, |
| 1919 | * which is a rather good thing for us. |
| 1920 | * |
| 1921 | * Also, it is only used when turning caches on/off ("The expected |
| 1922 | * usage of the cache maintenance instructions that operate by set/way |
| 1923 | * is associated with the cache maintenance instructions associated |
| 1924 | * with the powerdown and powerup of caches, if this is required by |
| 1925 | * the implementation."). |
| 1926 | * |
| 1927 | * We use the following policy: |
| 1928 | * |
| 1929 | * - If we trap a S/W operation, we enable VM trapping to detect |
| 1930 | * caches being turned on/off, and do a full clean. |
| 1931 | * |
| 1932 | * - We flush the caches on both caches being turned on and off. |
| 1933 | * |
| 1934 | * - Once the caches are enabled, we stop trapping VM ops. |
| 1935 | */ |
| 1936 | void kvm_set_way_flush(struct kvm_vcpu *vcpu) |
| 1937 | { |
| 1938 | unsigned long hcr = vcpu_get_hcr(vcpu); |
| 1939 | |
| 1940 | /* |
| 1941 | * If this is the first time we do a S/W operation |
| 1942 | * (i.e. HCR_TVM not set) flush the whole memory, and set the |
| 1943 | * VM trapping. |
| 1944 | * |
| 1945 | * Otherwise, rely on the VM trapping to wait for the MMU + |
| 1946 | * Caches to be turned off. At that point, we'll be able to |
| 1947 | * clean the caches again. |
| 1948 | */ |
| 1949 | if (!(hcr & HCR_TVM)) { |
| 1950 | trace_kvm_set_way_flush(*vcpu_pc(vcpu), |
| 1951 | vcpu_has_cache_enabled(vcpu)); |
| 1952 | stage2_flush_vm(vcpu->kvm); |
| 1953 | vcpu_set_hcr(vcpu, hcr | HCR_TVM); |
| 1954 | } |
| 1955 | } |
| 1956 | |
| 1957 | void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) |
| 1958 | { |
| 1959 | bool now_enabled = vcpu_has_cache_enabled(vcpu); |
| 1960 | |
| 1961 | /* |
| 1962 | * If switching the MMU+caches on, need to invalidate the caches. |
| 1963 | * If switching it off, need to clean the caches. |
| 1964 | * Clean + invalidate does the trick always. |
| 1965 | */ |
| 1966 | if (now_enabled != was_enabled) |
| 1967 | stage2_flush_vm(vcpu->kvm); |
| 1968 | |
| 1969 | /* Caches are now on, stop trapping VM ops (until a S/W op) */ |
| 1970 | if (now_enabled) |
| 1971 | vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) & ~HCR_TVM); |
| 1972 | |
| 1973 | trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); |
| 1974 | } |