blob: 7789857d147034b8cbfd1e44d599f81db085215b [file] [log] [blame]
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
Christoffer Dall342cd0a2013-01-20 18:28:06 -050018
19#include <linux/mman.h>
20#include <linux/kvm_host.h>
21#include <linux/io.h>
Christoffer Dallad361f02012-11-01 17:14:45 +010022#include <linux/hugetlb.h>
Christoffer Dall45e96ea2013-01-20 18:43:58 -050023#include <trace/events/kvm.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050024#include <asm/pgalloc.h>
Christoffer Dall94f8e642013-01-20 18:28:12 -050025#include <asm/cacheflush.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050026#include <asm/kvm_arm.h>
27#include <asm/kvm_mmu.h>
Christoffer Dall45e96ea2013-01-20 18:43:58 -050028#include <asm/kvm_mmio.h>
Christoffer Dalld5d81842013-01-20 18:28:07 -050029#include <asm/kvm_asm.h>
Christoffer Dall94f8e642013-01-20 18:28:12 -050030#include <asm/kvm_emulate.h>
Christoffer Dalld5d81842013-01-20 18:28:07 -050031
32#include "trace.h"
Christoffer Dall342cd0a2013-01-20 18:28:06 -050033
34extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
35
Marc Zyngier5a677ce2013-04-12 19:12:06 +010036static pgd_t *boot_hyp_pgd;
Marc Zyngier2fb41052013-04-12 19:12:03 +010037static pgd_t *hyp_pgd;
Christoffer Dall342cd0a2013-01-20 18:28:06 -050038static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
39
Marc Zyngier5a677ce2013-04-12 19:12:06 +010040static void *init_bounce_page;
41static unsigned long hyp_idmap_start;
42static unsigned long hyp_idmap_end;
43static phys_addr_t hyp_idmap_vector;
44
Christoffer Dall9b5fdb92013-10-02 15:32:01 -070045#define kvm_pmd_huge(_x) (pmd_huge(_x) || pmd_trans_huge(_x))
Christoffer Dallad361f02012-11-01 17:14:45 +010046
Marc Zyngier48762762013-01-28 15:27:00 +000047static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
Christoffer Dalld5d81842013-01-20 18:28:07 -050048{
Marc Zyngierd4cb9df52013-05-14 12:11:34 +010049 /*
50 * This function also gets called when dealing with HYP page
51 * tables. As HYP doesn't have an associated struct kvm (and
52 * the HYP page tables are fairly static), we don't do
53 * anything there.
54 */
55 if (kvm)
56 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
Christoffer Dalld5d81842013-01-20 18:28:07 -050057}
58
Christoffer Dalld5d81842013-01-20 18:28:07 -050059static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
60 int min, int max)
61{
62 void *page;
63
64 BUG_ON(max > KVM_NR_MEM_OBJS);
65 if (cache->nobjs >= min)
66 return 0;
67 while (cache->nobjs < max) {
68 page = (void *)__get_free_page(PGALLOC_GFP);
69 if (!page)
70 return -ENOMEM;
71 cache->objects[cache->nobjs++] = page;
72 }
73 return 0;
74}
75
76static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
77{
78 while (mc->nobjs)
79 free_page((unsigned long)mc->objects[--mc->nobjs]);
80}
81
82static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
83{
84 void *p;
85
86 BUG_ON(!mc || !mc->nobjs);
87 p = mc->objects[--mc->nobjs];
88 return p;
89}
90
Marc Zyngier979acd52013-08-06 13:05:48 +010091static bool page_empty(void *ptr)
92{
93 struct page *ptr_page = virt_to_page(ptr);
94 return page_count(ptr_page) == 1;
95}
96
Marc Zyngierd4cb9df52013-05-14 12:11:34 +010097static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
Christoffer Dall342cd0a2013-01-20 18:28:06 -050098{
Christoffer Dallad361f02012-11-01 17:14:45 +010099 if (pud_huge(*pud)) {
100 pud_clear(pud);
101 kvm_tlb_flush_vmid_ipa(kvm, addr);
102 } else {
103 pmd_t *pmd_table = pmd_offset(pud, 0);
104 pud_clear(pud);
105 kvm_tlb_flush_vmid_ipa(kvm, addr);
106 pmd_free(NULL, pmd_table);
107 }
Marc Zyngier4f728272013-04-12 19:12:05 +0100108 put_page(virt_to_page(pud));
109}
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500110
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100111static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
Marc Zyngier4f728272013-04-12 19:12:05 +0100112{
Christoffer Dallad361f02012-11-01 17:14:45 +0100113 if (kvm_pmd_huge(*pmd)) {
114 pmd_clear(pmd);
115 kvm_tlb_flush_vmid_ipa(kvm, addr);
116 } else {
117 pte_t *pte_table = pte_offset_kernel(pmd, 0);
118 pmd_clear(pmd);
119 kvm_tlb_flush_vmid_ipa(kvm, addr);
120 pte_free_kernel(NULL, pte_table);
121 }
Marc Zyngier4f728272013-04-12 19:12:05 +0100122 put_page(virt_to_page(pmd));
123}
124
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100125static void clear_pte_entry(struct kvm *kvm, pte_t *pte, phys_addr_t addr)
Marc Zyngier4f728272013-04-12 19:12:05 +0100126{
127 if (pte_present(*pte)) {
128 kvm_set_pte(pte, __pte(0));
129 put_page(virt_to_page(pte));
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100130 kvm_tlb_flush_vmid_ipa(kvm, addr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500131 }
132}
133
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100134static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
135 unsigned long long start, u64 size)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500136{
137 pgd_t *pgd;
138 pud_t *pud;
139 pmd_t *pmd;
Marc Zyngier4f728272013-04-12 19:12:05 +0100140 pte_t *pte;
141 unsigned long long addr = start, end = start + size;
Christoffer Dalld3840b22013-08-06 13:50:54 -0700142 u64 next;
Marc Zyngier000d3992013-03-05 02:43:17 +0000143
Marc Zyngier4f728272013-04-12 19:12:05 +0100144 while (addr < end) {
145 pgd = pgdp + pgd_index(addr);
146 pud = pud_offset(pgd, addr);
147 if (pud_none(*pud)) {
Christoffer Dalld3840b22013-08-06 13:50:54 -0700148 addr = pud_addr_end(addr, end);
Marc Zyngier4f728272013-04-12 19:12:05 +0100149 continue;
150 }
Marc Zyngier000d3992013-03-05 02:43:17 +0000151
Christoffer Dallad361f02012-11-01 17:14:45 +0100152 if (pud_huge(*pud)) {
153 /*
154 * If we are dealing with a huge pud, just clear it and
155 * move on.
156 */
157 clear_pud_entry(kvm, pud, addr);
158 addr = pud_addr_end(addr, end);
159 continue;
160 }
161
Marc Zyngier4f728272013-04-12 19:12:05 +0100162 pmd = pmd_offset(pud, addr);
163 if (pmd_none(*pmd)) {
Christoffer Dalld3840b22013-08-06 13:50:54 -0700164 addr = pmd_addr_end(addr, end);
Marc Zyngier4f728272013-04-12 19:12:05 +0100165 continue;
166 }
Marc Zyngier000d3992013-03-05 02:43:17 +0000167
Christoffer Dallad361f02012-11-01 17:14:45 +0100168 if (!kvm_pmd_huge(*pmd)) {
169 pte = pte_offset_kernel(pmd, addr);
170 clear_pte_entry(kvm, pte, addr);
171 next = addr + PAGE_SIZE;
172 }
Marc Zyngier4f728272013-04-12 19:12:05 +0100173
Christoffer Dallad361f02012-11-01 17:14:45 +0100174 /*
175 * If the pmd entry is to be cleared, walk back up the ladder
176 */
177 if (kvm_pmd_huge(*pmd) || page_empty(pte)) {
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100178 clear_pmd_entry(kvm, pmd, addr);
Christoffer Dalld3840b22013-08-06 13:50:54 -0700179 next = pmd_addr_end(addr, end);
Marc Zyngier979acd52013-08-06 13:05:48 +0100180 if (page_empty(pmd) && !page_empty(pud)) {
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100181 clear_pud_entry(kvm, pud, addr);
Christoffer Dalld3840b22013-08-06 13:50:54 -0700182 next = pud_addr_end(addr, end);
Marc Zyngier4f728272013-04-12 19:12:05 +0100183 }
184 }
185
Christoffer Dalld3840b22013-08-06 13:50:54 -0700186 addr = next;
Marc Zyngier4f728272013-04-12 19:12:05 +0100187 }
Marc Zyngier000d3992013-03-05 02:43:17 +0000188}
189
190/**
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100191 * free_boot_hyp_pgd - free HYP boot page tables
192 *
193 * Free the HYP boot page tables. The bounce page is also freed.
194 */
195void free_boot_hyp_pgd(void)
196{
197 mutex_lock(&kvm_hyp_pgd_mutex);
198
199 if (boot_hyp_pgd) {
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100200 unmap_range(NULL, boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
201 unmap_range(NULL, boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100202 kfree(boot_hyp_pgd);
203 boot_hyp_pgd = NULL;
204 }
205
206 if (hyp_pgd)
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100207 unmap_range(NULL, hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100208
209 kfree(init_bounce_page);
210 init_bounce_page = NULL;
211
212 mutex_unlock(&kvm_hyp_pgd_mutex);
213}
214
215/**
Marc Zyngier4f728272013-04-12 19:12:05 +0100216 * free_hyp_pgds - free Hyp-mode page tables
Marc Zyngier000d3992013-03-05 02:43:17 +0000217 *
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100218 * Assumes hyp_pgd is a page table used strictly in Hyp-mode and
219 * therefore contains either mappings in the kernel memory area (above
220 * PAGE_OFFSET), or device mappings in the vmalloc range (from
221 * VMALLOC_START to VMALLOC_END).
222 *
223 * boot_hyp_pgd should only map two pages for the init code.
Marc Zyngier000d3992013-03-05 02:43:17 +0000224 */
Marc Zyngier4f728272013-04-12 19:12:05 +0100225void free_hyp_pgds(void)
Marc Zyngier000d3992013-03-05 02:43:17 +0000226{
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500227 unsigned long addr;
228
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100229 free_boot_hyp_pgd();
Marc Zyngier4f728272013-04-12 19:12:05 +0100230
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100231 mutex_lock(&kvm_hyp_pgd_mutex);
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100232
Marc Zyngier4f728272013-04-12 19:12:05 +0100233 if (hyp_pgd) {
234 for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100235 unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
Marc Zyngier4f728272013-04-12 19:12:05 +0100236 for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100237 unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
238
Marc Zyngier4f728272013-04-12 19:12:05 +0100239 kfree(hyp_pgd);
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100240 hyp_pgd = NULL;
Marc Zyngier4f728272013-04-12 19:12:05 +0100241 }
242
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500243 mutex_unlock(&kvm_hyp_pgd_mutex);
244}
245
246static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
Marc Zyngier6060df82013-04-12 19:12:01 +0100247 unsigned long end, unsigned long pfn,
248 pgprot_t prot)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500249{
250 pte_t *pte;
251 unsigned long addr;
252
Marc Zyngier3562c762013-04-12 19:12:02 +0100253 addr = start;
254 do {
Marc Zyngier6060df82013-04-12 19:12:01 +0100255 pte = pte_offset_kernel(pmd, addr);
256 kvm_set_pte(pte, pfn_pte(pfn, prot));
Marc Zyngier4f728272013-04-12 19:12:05 +0100257 get_page(virt_to_page(pte));
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100258 kvm_flush_dcache_to_poc(pte, sizeof(*pte));
Marc Zyngier6060df82013-04-12 19:12:01 +0100259 pfn++;
Marc Zyngier3562c762013-04-12 19:12:02 +0100260 } while (addr += PAGE_SIZE, addr != end);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500261}
262
263static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
Marc Zyngier6060df82013-04-12 19:12:01 +0100264 unsigned long end, unsigned long pfn,
265 pgprot_t prot)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500266{
267 pmd_t *pmd;
268 pte_t *pte;
269 unsigned long addr, next;
270
Marc Zyngier3562c762013-04-12 19:12:02 +0100271 addr = start;
272 do {
Marc Zyngier6060df82013-04-12 19:12:01 +0100273 pmd = pmd_offset(pud, addr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500274
275 BUG_ON(pmd_sect(*pmd));
276
277 if (pmd_none(*pmd)) {
Marc Zyngier6060df82013-04-12 19:12:01 +0100278 pte = pte_alloc_one_kernel(NULL, addr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500279 if (!pte) {
280 kvm_err("Cannot allocate Hyp pte\n");
281 return -ENOMEM;
282 }
283 pmd_populate_kernel(NULL, pmd, pte);
Marc Zyngier4f728272013-04-12 19:12:05 +0100284 get_page(virt_to_page(pmd));
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100285 kvm_flush_dcache_to_poc(pmd, sizeof(*pmd));
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500286 }
287
288 next = pmd_addr_end(addr, end);
289
Marc Zyngier6060df82013-04-12 19:12:01 +0100290 create_hyp_pte_mappings(pmd, addr, next, pfn, prot);
291 pfn += (next - addr) >> PAGE_SHIFT;
Marc Zyngier3562c762013-04-12 19:12:02 +0100292 } while (addr = next, addr != end);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500293
294 return 0;
295}
296
Marc Zyngier6060df82013-04-12 19:12:01 +0100297static int __create_hyp_mappings(pgd_t *pgdp,
298 unsigned long start, unsigned long end,
299 unsigned long pfn, pgprot_t prot)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500300{
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500301 pgd_t *pgd;
302 pud_t *pud;
303 pmd_t *pmd;
304 unsigned long addr, next;
305 int err = 0;
306
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500307 mutex_lock(&kvm_hyp_pgd_mutex);
Marc Zyngier3562c762013-04-12 19:12:02 +0100308 addr = start & PAGE_MASK;
309 end = PAGE_ALIGN(end);
310 do {
Marc Zyngier6060df82013-04-12 19:12:01 +0100311 pgd = pgdp + pgd_index(addr);
312 pud = pud_offset(pgd, addr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500313
314 if (pud_none_or_clear_bad(pud)) {
Marc Zyngier6060df82013-04-12 19:12:01 +0100315 pmd = pmd_alloc_one(NULL, addr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500316 if (!pmd) {
317 kvm_err("Cannot allocate Hyp pmd\n");
318 err = -ENOMEM;
319 goto out;
320 }
321 pud_populate(NULL, pud, pmd);
Marc Zyngier4f728272013-04-12 19:12:05 +0100322 get_page(virt_to_page(pud));
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100323 kvm_flush_dcache_to_poc(pud, sizeof(*pud));
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500324 }
325
326 next = pgd_addr_end(addr, end);
Marc Zyngier6060df82013-04-12 19:12:01 +0100327 err = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500328 if (err)
329 goto out;
Marc Zyngier6060df82013-04-12 19:12:01 +0100330 pfn += (next - addr) >> PAGE_SHIFT;
Marc Zyngier3562c762013-04-12 19:12:02 +0100331 } while (addr = next, addr != end);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500332out:
333 mutex_unlock(&kvm_hyp_pgd_mutex);
334 return err;
335}
336
Christoffer Dall40c27292013-11-15 13:14:12 -0800337static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
338{
339 if (!is_vmalloc_addr(kaddr)) {
340 BUG_ON(!virt_addr_valid(kaddr));
341 return __pa(kaddr);
342 } else {
343 return page_to_phys(vmalloc_to_page(kaddr)) +
344 offset_in_page(kaddr);
345 }
346}
347
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500348/**
Marc Zyngier06e8c3b2012-10-28 01:09:14 +0100349 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500350 * @from: The virtual kernel start address of the range
351 * @to: The virtual kernel end address of the range (exclusive)
352 *
Marc Zyngier06e8c3b2012-10-28 01:09:14 +0100353 * The same virtual address as the kernel virtual address is also used
354 * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
355 * physical pages.
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500356 */
357int create_hyp_mappings(void *from, void *to)
358{
Christoffer Dall40c27292013-11-15 13:14:12 -0800359 phys_addr_t phys_addr;
360 unsigned long virt_addr;
Marc Zyngier6060df82013-04-12 19:12:01 +0100361 unsigned long start = KERN_TO_HYP((unsigned long)from);
362 unsigned long end = KERN_TO_HYP((unsigned long)to);
363
Christoffer Dall40c27292013-11-15 13:14:12 -0800364 start = start & PAGE_MASK;
365 end = PAGE_ALIGN(end);
Marc Zyngier6060df82013-04-12 19:12:01 +0100366
Christoffer Dall40c27292013-11-15 13:14:12 -0800367 for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
368 int err;
369
370 phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
371 err = __create_hyp_mappings(hyp_pgd, virt_addr,
372 virt_addr + PAGE_SIZE,
373 __phys_to_pfn(phys_addr),
374 PAGE_HYP);
375 if (err)
376 return err;
377 }
378
379 return 0;
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500380}
381
382/**
Marc Zyngier06e8c3b2012-10-28 01:09:14 +0100383 * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode
384 * @from: The kernel start VA of the range
385 * @to: The kernel end VA of the range (exclusive)
Marc Zyngier6060df82013-04-12 19:12:01 +0100386 * @phys_addr: The physical start address which gets mapped
Marc Zyngier06e8c3b2012-10-28 01:09:14 +0100387 *
388 * The resulting HYP VA is the same as the kernel VA, modulo
389 * HYP_PAGE_OFFSET.
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500390 */
Marc Zyngier6060df82013-04-12 19:12:01 +0100391int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500392{
Marc Zyngier6060df82013-04-12 19:12:01 +0100393 unsigned long start = KERN_TO_HYP((unsigned long)from);
394 unsigned long end = KERN_TO_HYP((unsigned long)to);
395
396 /* Check for a valid kernel IO mapping */
397 if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1))
398 return -EINVAL;
399
400 return __create_hyp_mappings(hyp_pgd, start, end,
401 __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500402}
403
Christoffer Dalld5d81842013-01-20 18:28:07 -0500404/**
405 * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
406 * @kvm: The KVM struct pointer for the VM.
407 *
408 * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can
409 * support either full 40-bit input addresses or limited to 32-bit input
410 * addresses). Clears the allocated pages.
411 *
412 * Note we don't need locking here as this is only called when the VM is
413 * created, which can only be done once.
414 */
415int kvm_alloc_stage2_pgd(struct kvm *kvm)
416{
417 pgd_t *pgd;
418
419 if (kvm->arch.pgd != NULL) {
420 kvm_err("kvm_arch already initialized?\n");
421 return -EINVAL;
422 }
423
424 pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, S2_PGD_ORDER);
425 if (!pgd)
426 return -ENOMEM;
427
Christoffer Dalld5d81842013-01-20 18:28:07 -0500428 memset(pgd, 0, PTRS_PER_S2_PGD * sizeof(pgd_t));
Marc Zyngierc62ee2b2012-10-15 11:27:37 +0100429 kvm_clean_pgd(pgd);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500430 kvm->arch.pgd = pgd;
431
432 return 0;
433}
434
Christoffer Dalld5d81842013-01-20 18:28:07 -0500435/**
436 * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
437 * @kvm: The VM pointer
438 * @start: The intermediate physical base address of the range to unmap
439 * @size: The size of the area to unmap
440 *
441 * Clear a range of stage-2 mappings, lowering the various ref-counts. Must
442 * be called while holding mmu_lock (unless for freeing the stage2 pgd before
443 * destroying the VM), otherwise another faulting VCPU may come in and mess
444 * with things behind our backs.
445 */
446static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
447{
Marc Zyngierd4cb9df52013-05-14 12:11:34 +0100448 unmap_range(kvm, kvm->arch.pgd, start, size);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500449}
450
451/**
452 * kvm_free_stage2_pgd - free all stage-2 tables
453 * @kvm: The KVM struct pointer for the VM.
454 *
455 * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
456 * underlying level-2 and level-3 tables before freeing the actual level-1 table
457 * and setting the struct pointer to NULL.
458 *
459 * Note we don't need locking here as this is only called when the VM is
460 * destroyed, which can only be done once.
461 */
462void kvm_free_stage2_pgd(struct kvm *kvm)
463{
464 if (kvm->arch.pgd == NULL)
465 return;
466
467 unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
468 free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER);
469 kvm->arch.pgd = NULL;
470}
471
Christoffer Dallad361f02012-11-01 17:14:45 +0100472static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
473 phys_addr_t addr)
Christoffer Dalld5d81842013-01-20 18:28:07 -0500474{
475 pgd_t *pgd;
476 pud_t *pud;
477 pmd_t *pmd;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500478
Christoffer Dalld5d81842013-01-20 18:28:07 -0500479 pgd = kvm->arch.pgd + pgd_index(addr);
480 pud = pud_offset(pgd, addr);
481 if (pud_none(*pud)) {
482 if (!cache)
Christoffer Dallad361f02012-11-01 17:14:45 +0100483 return NULL;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500484 pmd = mmu_memory_cache_alloc(cache);
485 pud_populate(NULL, pud, pmd);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500486 get_page(virt_to_page(pud));
Marc Zyngierc62ee2b2012-10-15 11:27:37 +0100487 }
488
Christoffer Dallad361f02012-11-01 17:14:45 +0100489 return pmd_offset(pud, addr);
490}
Christoffer Dalld5d81842013-01-20 18:28:07 -0500491
Christoffer Dallad361f02012-11-01 17:14:45 +0100492static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
493 *cache, phys_addr_t addr, const pmd_t *new_pmd)
494{
495 pmd_t *pmd, old_pmd;
496
497 pmd = stage2_get_pmd(kvm, cache, addr);
498 VM_BUG_ON(!pmd);
499
500 /*
501 * Mapping in huge pages should only happen through a fault. If a
502 * page is merged into a transparent huge page, the individual
503 * subpages of that huge page should be unmapped through MMU
504 * notifiers before we get here.
505 *
506 * Merging of CompoundPages is not supported; they should become
507 * splitting first, unmapped, merged, and mapped back in on-demand.
508 */
509 VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd));
510
511 old_pmd = *pmd;
512 kvm_set_pmd(pmd, *new_pmd);
513 if (pmd_present(old_pmd))
514 kvm_tlb_flush_vmid_ipa(kvm, addr);
515 else
516 get_page(virt_to_page(pmd));
517 return 0;
518}
519
520static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
521 phys_addr_t addr, const pte_t *new_pte, bool iomap)
522{
523 pmd_t *pmd;
524 pte_t *pte, old_pte;
525
526 /* Create stage-2 page table mapping - Level 1 */
527 pmd = stage2_get_pmd(kvm, cache, addr);
528 if (!pmd) {
529 /*
530 * Ignore calls from kvm_set_spte_hva for unallocated
531 * address ranges.
532 */
533 return 0;
534 }
535
536 /* Create stage-2 page mappings - Level 2 */
Christoffer Dalld5d81842013-01-20 18:28:07 -0500537 if (pmd_none(*pmd)) {
538 if (!cache)
539 return 0; /* ignore calls from kvm_set_spte_hva */
540 pte = mmu_memory_cache_alloc(cache);
Marc Zyngierc62ee2b2012-10-15 11:27:37 +0100541 kvm_clean_pte(pte);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500542 pmd_populate_kernel(NULL, pmd, pte);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500543 get_page(virt_to_page(pmd));
Marc Zyngierc62ee2b2012-10-15 11:27:37 +0100544 }
545
546 pte = pte_offset_kernel(pmd, addr);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500547
548 if (iomap && pte_present(*pte))
549 return -EFAULT;
550
551 /* Create 2nd stage page table mapping - Level 3 */
552 old_pte = *pte;
553 kvm_set_pte(pte, *new_pte);
554 if (pte_present(old_pte))
Marc Zyngier48762762013-01-28 15:27:00 +0000555 kvm_tlb_flush_vmid_ipa(kvm, addr);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500556 else
557 get_page(virt_to_page(pte));
558
559 return 0;
560}
561
562/**
563 * kvm_phys_addr_ioremap - map a device range to guest IPA
564 *
565 * @kvm: The KVM pointer
566 * @guest_ipa: The IPA at which to insert the mapping
567 * @pa: The physical address of the device
568 * @size: The size of the mapping
569 */
570int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
571 phys_addr_t pa, unsigned long size)
572{
573 phys_addr_t addr, end;
574 int ret = 0;
575 unsigned long pfn;
576 struct kvm_mmu_memory_cache cache = { 0, };
577
578 end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
579 pfn = __phys_to_pfn(pa);
580
581 for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
Marc Zyngierc62ee2b2012-10-15 11:27:37 +0100582 pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500583
584 ret = mmu_topup_memory_cache(&cache, 2, 2);
585 if (ret)
586 goto out;
587 spin_lock(&kvm->mmu_lock);
588 ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
589 spin_unlock(&kvm->mmu_lock);
590 if (ret)
591 goto out;
592
593 pfn++;
594 }
595
596out:
597 mmu_free_memory_cache(&cache);
598 return ret;
599}
600
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700601static bool transparent_hugepage_adjust(pfn_t *pfnp, phys_addr_t *ipap)
602{
603 pfn_t pfn = *pfnp;
604 gfn_t gfn = *ipap >> PAGE_SHIFT;
605
606 if (PageTransCompound(pfn_to_page(pfn))) {
607 unsigned long mask;
608 /*
609 * The address we faulted on is backed by a transparent huge
610 * page. However, because we map the compound huge page and
611 * not the individual tail page, we need to transfer the
612 * refcount to the head page. We have to be careful that the
613 * THP doesn't start to split while we are adjusting the
614 * refcounts.
615 *
616 * We are sure this doesn't happen, because mmu_notifier_retry
617 * was successful and we are holding the mmu_lock, so if this
618 * THP is trying to split, it will be blocked in the mmu
619 * notifier before touching any of the pages, specifically
620 * before being able to call __split_huge_page_refcount().
621 *
622 * We can therefore safely transfer the refcount from PG_tail
623 * to PG_head and switch the pfn from a tail page to the head
624 * page accordingly.
625 */
626 mask = PTRS_PER_PMD - 1;
627 VM_BUG_ON((gfn & mask) != (pfn & mask));
628 if (pfn & mask) {
629 *ipap &= PMD_MASK;
630 kvm_release_pfn_clean(pfn);
631 pfn &= ~mask;
632 kvm_get_pfn(pfn);
633 *pfnp = pfn;
634 }
635
636 return true;
637 }
638
639 return false;
640}
641
Christoffer Dall94f8e642013-01-20 18:28:12 -0500642static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
Christoffer Dallad361f02012-11-01 17:14:45 +0100643 struct kvm_memory_slot *memslot,
Christoffer Dall94f8e642013-01-20 18:28:12 -0500644 unsigned long fault_status)
645{
Christoffer Dall94f8e642013-01-20 18:28:12 -0500646 int ret;
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700647 bool write_fault, writable, hugetlb = false, force_pte = false;
Christoffer Dall94f8e642013-01-20 18:28:12 -0500648 unsigned long mmu_seq;
Christoffer Dallad361f02012-11-01 17:14:45 +0100649 gfn_t gfn = fault_ipa >> PAGE_SHIFT;
650 unsigned long hva = gfn_to_hva(vcpu->kvm, gfn);
651 struct kvm *kvm = vcpu->kvm;
Christoffer Dall94f8e642013-01-20 18:28:12 -0500652 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
Christoffer Dallad361f02012-11-01 17:14:45 +0100653 struct vm_area_struct *vma;
654 pfn_t pfn;
Christoffer Dall94f8e642013-01-20 18:28:12 -0500655
Marc Zyngier7393b592012-09-17 19:27:09 +0100656 write_fault = kvm_is_write_fault(kvm_vcpu_get_hsr(vcpu));
Christoffer Dall94f8e642013-01-20 18:28:12 -0500657 if (fault_status == FSC_PERM && !write_fault) {
658 kvm_err("Unexpected L2 read permission error\n");
659 return -EFAULT;
660 }
661
Christoffer Dallad361f02012-11-01 17:14:45 +0100662 /* Let's check if we will get back a huge page backed by hugetlbfs */
663 down_read(&current->mm->mmap_sem);
664 vma = find_vma_intersection(current->mm, hva, hva + 1);
665 if (is_vm_hugetlb_page(vma)) {
666 hugetlb = true;
667 gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700668 } else {
669 /*
Marc Zyngier136d7372013-12-13 16:56:06 +0000670 * Pages belonging to memslots that don't have the same
671 * alignment for userspace and IPA cannot be mapped using
672 * block descriptors even if the pages belong to a THP for
673 * the process, because the stage-2 block descriptor will
674 * cover more than a single THP and we loose atomicity for
675 * unmapping, updates, and splits of the THP or other pages
676 * in the stage-2 block range.
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700677 */
Marc Zyngier136d7372013-12-13 16:56:06 +0000678 if ((memslot->userspace_addr & ~PMD_MASK) !=
679 ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK))
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700680 force_pte = true;
Christoffer Dallad361f02012-11-01 17:14:45 +0100681 }
682 up_read(&current->mm->mmap_sem);
683
Christoffer Dall94f8e642013-01-20 18:28:12 -0500684 /* We need minimum second+third level pages */
685 ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS);
686 if (ret)
687 return ret;
688
689 mmu_seq = vcpu->kvm->mmu_notifier_seq;
690 /*
691 * Ensure the read of mmu_notifier_seq happens before we call
692 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
693 * the page we just got a reference to gets unmapped before we have a
694 * chance to grab the mmu_lock, which ensure that if the page gets
695 * unmapped afterwards, the call to kvm_unmap_hva will take it away
696 * from us again properly. This smp_rmb() interacts with the smp_wmb()
697 * in kvm_mmu_notifier_invalidate_<page|range_end>.
698 */
699 smp_rmb();
700
Christoffer Dallad361f02012-11-01 17:14:45 +0100701 pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500702 if (is_error_pfn(pfn))
703 return -EFAULT;
704
Christoffer Dallad361f02012-11-01 17:14:45 +0100705 spin_lock(&kvm->mmu_lock);
706 if (mmu_notifier_retry(kvm, mmu_seq))
Christoffer Dall94f8e642013-01-20 18:28:12 -0500707 goto out_unlock;
Christoffer Dall9b5fdb92013-10-02 15:32:01 -0700708 if (!hugetlb && !force_pte)
709 hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
Christoffer Dallad361f02012-11-01 17:14:45 +0100710
711 if (hugetlb) {
712 pmd_t new_pmd = pfn_pmd(pfn, PAGE_S2);
713 new_pmd = pmd_mkhuge(new_pmd);
714 if (writable) {
715 kvm_set_s2pmd_writable(&new_pmd);
716 kvm_set_pfn_dirty(pfn);
717 }
718 coherent_icache_guest_page(kvm, hva & PMD_MASK, PMD_SIZE);
719 ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
720 } else {
721 pte_t new_pte = pfn_pte(pfn, PAGE_S2);
722 if (writable) {
723 kvm_set_s2pte_writable(&new_pte);
724 kvm_set_pfn_dirty(pfn);
725 }
726 coherent_icache_guest_page(kvm, hva, PAGE_SIZE);
727 ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, false);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500728 }
Christoffer Dallad361f02012-11-01 17:14:45 +0100729
Christoffer Dall94f8e642013-01-20 18:28:12 -0500730
731out_unlock:
Christoffer Dallad361f02012-11-01 17:14:45 +0100732 spin_unlock(&kvm->mmu_lock);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500733 kvm_release_pfn_clean(pfn);
Christoffer Dallad361f02012-11-01 17:14:45 +0100734 return ret;
Christoffer Dall94f8e642013-01-20 18:28:12 -0500735}
736
737/**
738 * kvm_handle_guest_abort - handles all 2nd stage aborts
739 * @vcpu: the VCPU pointer
740 * @run: the kvm_run structure
741 *
742 * Any abort that gets to the host is almost guaranteed to be caused by a
743 * missing second stage translation table entry, which can mean that either the
744 * guest simply needs more memory and we must allocate an appropriate page or it
745 * can mean that the guest tried to access I/O memory, which is emulated by user
746 * space. The distinction is based on the IPA causing the fault and whether this
747 * memory region has been registered as standard RAM by user space.
748 */
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500749int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
750{
Christoffer Dall94f8e642013-01-20 18:28:12 -0500751 unsigned long fault_status;
752 phys_addr_t fault_ipa;
753 struct kvm_memory_slot *memslot;
754 bool is_iabt;
755 gfn_t gfn;
756 int ret, idx;
757
Marc Zyngier52d1dba2012-10-15 10:33:38 +0100758 is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
Marc Zyngier7393b592012-09-17 19:27:09 +0100759 fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500760
Marc Zyngier7393b592012-09-17 19:27:09 +0100761 trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
762 kvm_vcpu_get_hfar(vcpu), fault_ipa);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500763
764 /* Check the stage-2 fault is trans. fault or write fault */
Marc Zyngier1cc287d2012-09-18 14:14:35 +0100765 fault_status = kvm_vcpu_trap_get_fault(vcpu);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500766 if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
Marc Zyngier52d1dba2012-10-15 10:33:38 +0100767 kvm_err("Unsupported fault status: EC=%#x DFCS=%#lx\n",
768 kvm_vcpu_trap_get_class(vcpu), fault_status);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500769 return -EFAULT;
770 }
771
772 idx = srcu_read_lock(&vcpu->kvm->srcu);
773
774 gfn = fault_ipa >> PAGE_SHIFT;
775 if (!kvm_is_visible_gfn(vcpu->kvm, gfn)) {
776 if (is_iabt) {
777 /* Prefetch Abort on I/O address */
Marc Zyngier7393b592012-09-17 19:27:09 +0100778 kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
Christoffer Dall94f8e642013-01-20 18:28:12 -0500779 ret = 1;
780 goto out_unlock;
781 }
782
783 if (fault_status != FSC_FAULT) {
784 kvm_err("Unsupported fault status on io memory: %#lx\n",
785 fault_status);
786 ret = -EFAULT;
787 goto out_unlock;
788 }
789
Marc Zyngiercfe39502012-12-12 14:42:09 +0000790 /*
791 * The IPA is reported as [MAX:12], so we need to
792 * complement it with the bottom 12 bits from the
793 * faulting VA. This is always 12 bits, irrespective
794 * of the page size.
795 */
796 fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
Christoffer Dall45e96ea2013-01-20 18:43:58 -0500797 ret = io_mem_abort(vcpu, run, fault_ipa);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500798 goto out_unlock;
799 }
800
801 memslot = gfn_to_memslot(vcpu->kvm, gfn);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500802
Christoffer Dallad361f02012-11-01 17:14:45 +0100803 ret = user_mem_abort(vcpu, fault_ipa, memslot, fault_status);
Christoffer Dall94f8e642013-01-20 18:28:12 -0500804 if (ret == 0)
805 ret = 1;
806out_unlock:
807 srcu_read_unlock(&vcpu->kvm->srcu, idx);
808 return ret;
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500809}
810
Christoffer Dalld5d81842013-01-20 18:28:07 -0500811static void handle_hva_to_gpa(struct kvm *kvm,
812 unsigned long start,
813 unsigned long end,
814 void (*handler)(struct kvm *kvm,
815 gpa_t gpa, void *data),
816 void *data)
817{
818 struct kvm_memslots *slots;
819 struct kvm_memory_slot *memslot;
820
821 slots = kvm_memslots(kvm);
822
823 /* we only care about the pages that the guest sees */
824 kvm_for_each_memslot(memslot, slots) {
825 unsigned long hva_start, hva_end;
826 gfn_t gfn, gfn_end;
827
828 hva_start = max(start, memslot->userspace_addr);
829 hva_end = min(end, memslot->userspace_addr +
830 (memslot->npages << PAGE_SHIFT));
831 if (hva_start >= hva_end)
832 continue;
833
834 /*
835 * {gfn(page) | page intersects with [hva_start, hva_end)} =
836 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
837 */
838 gfn = hva_to_gfn_memslot(hva_start, memslot);
839 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
840
841 for (; gfn < gfn_end; ++gfn) {
842 gpa_t gpa = gfn << PAGE_SHIFT;
843 handler(kvm, gpa, data);
844 }
845 }
846}
847
848static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
849{
850 unmap_stage2_range(kvm, gpa, PAGE_SIZE);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500851}
852
853int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
854{
855 unsigned long end = hva + PAGE_SIZE;
856
857 if (!kvm->arch.pgd)
858 return 0;
859
860 trace_kvm_unmap_hva(hva);
861 handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
862 return 0;
863}
864
865int kvm_unmap_hva_range(struct kvm *kvm,
866 unsigned long start, unsigned long end)
867{
868 if (!kvm->arch.pgd)
869 return 0;
870
871 trace_kvm_unmap_hva_range(start, end);
872 handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
873 return 0;
874}
875
876static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
877{
878 pte_t *pte = (pte_t *)data;
879
880 stage2_set_pte(kvm, NULL, gpa, pte, false);
881}
882
883
884void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
885{
886 unsigned long end = hva + PAGE_SIZE;
887 pte_t stage2_pte;
888
889 if (!kvm->arch.pgd)
890 return;
891
892 trace_kvm_set_spte_hva(hva);
893 stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
894 handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
895}
896
897void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
898{
899 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
900}
901
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500902phys_addr_t kvm_mmu_get_httbr(void)
903{
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500904 return virt_to_phys(hyp_pgd);
905}
906
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100907phys_addr_t kvm_mmu_get_boot_httbr(void)
908{
909 return virt_to_phys(boot_hyp_pgd);
910}
911
912phys_addr_t kvm_get_idmap_vector(void)
913{
914 return hyp_idmap_vector;
915}
916
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500917int kvm_mmu_init(void)
918{
Marc Zyngier2fb41052013-04-12 19:12:03 +0100919 int err;
920
Santosh Shilimkar4fda3422013-11-19 14:59:12 -0500921 hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start);
922 hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end);
923 hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init);
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100924
925 if ((hyp_idmap_start ^ hyp_idmap_end) & PAGE_MASK) {
926 /*
927 * Our init code is crossing a page boundary. Allocate
928 * a bounce page, copy the code over and use that.
929 */
930 size_t len = __hyp_idmap_text_end - __hyp_idmap_text_start;
931 phys_addr_t phys_base;
932
933 init_bounce_page = kmalloc(PAGE_SIZE, GFP_KERNEL);
934 if (!init_bounce_page) {
935 kvm_err("Couldn't allocate HYP init bounce page\n");
936 err = -ENOMEM;
937 goto out;
938 }
939
940 memcpy(init_bounce_page, __hyp_idmap_text_start, len);
941 /*
942 * Warning: the code we just copied to the bounce page
943 * must be flushed to the point of coherency.
944 * Otherwise, the data may be sitting in L2, and HYP
945 * mode won't be able to observe it as it runs with
946 * caches off at that point.
947 */
948 kvm_flush_dcache_to_poc(init_bounce_page, len);
949
Santosh Shilimkar4fda3422013-11-19 14:59:12 -0500950 phys_base = kvm_virt_to_phys(init_bounce_page);
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100951 hyp_idmap_vector += phys_base - hyp_idmap_start;
952 hyp_idmap_start = phys_base;
953 hyp_idmap_end = phys_base + len;
954
955 kvm_info("Using HYP init bounce page @%lx\n",
956 (unsigned long)phys_base);
957 }
958
Marc Zyngier2fb41052013-04-12 19:12:03 +0100959 hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100960 boot_hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
961 if (!hyp_pgd || !boot_hyp_pgd) {
Christoffer Dalld5d81842013-01-20 18:28:07 -0500962 kvm_err("Hyp mode PGD not allocated\n");
Marc Zyngier2fb41052013-04-12 19:12:03 +0100963 err = -ENOMEM;
964 goto out;
965 }
966
967 /* Create the idmap in the boot page tables */
968 err = __create_hyp_mappings(boot_hyp_pgd,
969 hyp_idmap_start, hyp_idmap_end,
970 __phys_to_pfn(hyp_idmap_start),
971 PAGE_HYP);
972
973 if (err) {
974 kvm_err("Failed to idmap %lx-%lx\n",
975 hyp_idmap_start, hyp_idmap_end);
976 goto out;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500977 }
978
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100979 /* Map the very same page at the trampoline VA */
980 err = __create_hyp_mappings(boot_hyp_pgd,
981 TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
982 __phys_to_pfn(hyp_idmap_start),
983 PAGE_HYP);
984 if (err) {
985 kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n",
986 TRAMPOLINE_VA);
987 goto out;
988 }
989
990 /* Map the same page again into the runtime page tables */
991 err = __create_hyp_mappings(hyp_pgd,
992 TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
993 __phys_to_pfn(hyp_idmap_start),
994 PAGE_HYP);
995 if (err) {
996 kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n",
997 TRAMPOLINE_VA);
998 goto out;
999 }
1000
Christoffer Dalld5d81842013-01-20 18:28:07 -05001001 return 0;
Marc Zyngier2fb41052013-04-12 19:12:03 +01001002out:
Marc Zyngier4f728272013-04-12 19:12:05 +01001003 free_hyp_pgds();
Marc Zyngier2fb41052013-04-12 19:12:03 +01001004 return err;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001005}