blob: ca8f174289bb43a148bdd9159f71ca645e0c416a [file] [log] [blame]
Alexander Graf07372792010-04-16 00:11:35 +02001/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 *
15 * Copyright SUSE Linux Products GmbH 2010
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20/* Real mode helpers */
21
22#if defined(CONFIG_PPC_BOOK3S_64)
23
24#define GET_SHADOW_VCPU(reg) \
Paul Mackerras3c42bf82011-06-29 00:20:58 +000025 mr reg, r13
Alexander Graf07372792010-04-16 00:11:35 +020026
27#elif defined(CONFIG_PPC_BOOK3S_32)
28
29#define GET_SHADOW_VCPU(reg) \
30 tophys(reg, r2); \
31 lwz reg, (THREAD + THREAD_KVM_SVCPU)(reg); \
32 tophys(reg, reg)
33
34#endif
35
36/* Disable for nested KVM */
37#define USE_QUICK_LAST_INST
38
39
40/* Get helper functions for subarch specific functionality */
41
42#if defined(CONFIG_PPC_BOOK3S_64)
43#include "book3s_64_slb.S"
44#elif defined(CONFIG_PPC_BOOK3S_32)
45#include "book3s_32_sr.S"
46#endif
47
48/******************************************************************************
49 * *
50 * Entry code *
51 * *
52 *****************************************************************************/
53
54.global kvmppc_handler_trampoline_enter
55kvmppc_handler_trampoline_enter:
56
57 /* Required state:
58 *
59 * MSR = ~IR|DR
Alexander Graf07372792010-04-16 00:11:35 +020060 * R1 = host R1
61 * R2 = host R2
Paul Mackerras02143942011-07-23 17:41:44 +100062 * R4 = guest shadow MSR
63 * R5 = normal host MSR
64 * R6 = current host MSR (EE, IR, DR off)
65 * LR = highmem guest exit code
Alexander Graf07372792010-04-16 00:11:35 +020066 * all other volatile GPRS = free
67 * SVCPU[CR] = guest CR
68 * SVCPU[XER] = guest XER
69 * SVCPU[CTR] = guest CTR
70 * SVCPU[LR] = guest LR
71 */
72
73 /* r3 = shadow vcpu */
74 GET_SHADOW_VCPU(r3)
75
Paul Mackerras02143942011-07-23 17:41:44 +100076 /* Save guest exit handler address and MSR */
77 mflr r0
78 PPC_STL r0, HSTATE_VMHANDLER(r3)
79 PPC_STL r5, HSTATE_HOST_MSR(r3)
80
Paul Mackerras3c42bf82011-06-29 00:20:58 +000081 /* Save R1/R2 in the PACA (64-bit) or shadow_vcpu (32-bit) */
82 PPC_STL r1, HSTATE_HOST_R1(r3)
83 PPC_STL r2, HSTATE_HOST_R2(r3)
84
Alexander Graf07372792010-04-16 00:11:35 +020085 /* Activate guest mode, so faults get handled by KVM */
86 li r11, KVM_GUEST_MODE_GUEST
Paul Mackerras3c42bf82011-06-29 00:20:58 +000087 stb r11, HSTATE_IN_GUEST(r3)
Alexander Graf07372792010-04-16 00:11:35 +020088
89 /* Switch to guest segment. This is subarch specific. */
90 LOAD_GUEST_SEGMENTS
91
Paul Mackerras02143942011-07-23 17:41:44 +100092#ifdef CONFIG_PPC_BOOK3S_64
Alexander Graf616dff82014-04-29 16:48:44 +020093BEGIN_FTR_SECTION
94 /* Save host FSCR */
95 mfspr r8, SPRN_FSCR
96 std r8, HSTATE_HOST_FSCR(r13)
97 /* Set FSCR during guest execution */
98 ld r9, SVCPU_SHADOW_FSCR(r13)
99 mtspr SPRN_FSCR, r9
100END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
101
Paul Mackerras02143942011-07-23 17:41:44 +1000102 /* Some guests may need to have dcbz set to 32 byte length.
103 *
104 * Usually we ensure that by patching the guest's instructions
105 * to trap on dcbz and emulate it in the hypervisor.
106 *
107 * If we can, we should tell the CPU to use 32 byte dcbz though,
108 * because that's a lot faster.
109 */
110 lbz r0, HSTATE_RESTORE_HID5(r3)
111 cmpwi r0, 0
112 beq no_dcbz32_on
113
114 mfspr r0,SPRN_HID5
115 ori r0, r0, 0x80 /* XXX HID5_dcbz32 = 0x80 */
116 mtspr SPRN_HID5,r0
117no_dcbz32_on:
118
119#endif /* CONFIG_PPC_BOOK3S_64 */
120
Alexander Graf07372792010-04-16 00:11:35 +0200121 /* Enter guest */
122
Paul Mackerras02143942011-07-23 17:41:44 +1000123 PPC_LL r8, SVCPU_CTR(r3)
124 PPC_LL r9, SVCPU_LR(r3)
125 lwz r10, SVCPU_CR(r3)
Sam bobroffc63517c2015-05-27 09:56:57 +1000126 PPC_LL r11, SVCPU_XER(r3)
Alexander Graf07372792010-04-16 00:11:35 +0200127
Paul Mackerras02143942011-07-23 17:41:44 +1000128 mtctr r8
129 mtlr r9
130 mtcr r10
131 mtxer r11
132
133 /* Move SRR0 and SRR1 into the respective regs */
134 PPC_LL r9, SVCPU_PC(r3)
135 /* First clear RI in our current MSR value */
136 li r0, MSR_RI
137 andc r6, r6, r0
Alexander Graf07372792010-04-16 00:11:35 +0200138
Paul Mackerrasde56a942011-06-29 00:21:34 +0000139 PPC_LL r0, SVCPU_R0(r3)
140 PPC_LL r1, SVCPU_R1(r3)
141 PPC_LL r2, SVCPU_R2(r3)
Paul Mackerrasde56a942011-06-29 00:21:34 +0000142 PPC_LL r5, SVCPU_R5(r3)
Paul Mackerrasde56a942011-06-29 00:21:34 +0000143 PPC_LL r7, SVCPU_R7(r3)
144 PPC_LL r8, SVCPU_R8(r3)
Paul Mackerrasde56a942011-06-29 00:21:34 +0000145 PPC_LL r10, SVCPU_R10(r3)
146 PPC_LL r11, SVCPU_R11(r3)
147 PPC_LL r12, SVCPU_R12(r3)
148 PPC_LL r13, SVCPU_R13(r3)
Alexander Graf07372792010-04-16 00:11:35 +0200149
Alexander Graf8c2d0be2012-04-25 14:28:23 +0200150 MTMSR_EERI(r6)
151 mtsrr0 r9
152 mtsrr1 r4
153
154 PPC_LL r4, SVCPU_R4(r3)
155 PPC_LL r6, SVCPU_R6(r3)
156 PPC_LL r9, SVCPU_R9(r3)
Alexander Graf07372792010-04-16 00:11:35 +0200157 PPC_LL r3, (SVCPU_R3)(r3)
158
159 RFI
160kvmppc_handler_trampoline_enter_end:
161
162
163
164/******************************************************************************
165 * *
166 * Exit code *
167 * *
168 *****************************************************************************/
169
170.global kvmppc_handler_trampoline_exit
171kvmppc_handler_trampoline_exit:
172
Aneesh Kumar K.Vdd96b2c2013-10-07 22:17:55 +0530173.global kvmppc_interrupt_pr
174kvmppc_interrupt_pr:
Paul Mackerrasb01c8b52011-06-29 00:18:26 +0000175
Alexander Graf07372792010-04-16 00:11:35 +0200176 /* Register usage at this point:
177 *
178 * SPRG_SCRATCH0 = guest R13
179 * R12 = exit handler id
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000180 * R13 = shadow vcpu (32-bit) or PACA (64-bit)
181 * HSTATE.SCRATCH0 = guest R12
182 * HSTATE.SCRATCH1 = guest CR
Alexander Graf07372792010-04-16 00:11:35 +0200183 *
184 */
185
186 /* Save registers */
187
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000188 PPC_STL r0, SVCPU_R0(r13)
189 PPC_STL r1, SVCPU_R1(r13)
190 PPC_STL r2, SVCPU_R2(r13)
191 PPC_STL r3, SVCPU_R3(r13)
192 PPC_STL r4, SVCPU_R4(r13)
193 PPC_STL r5, SVCPU_R5(r13)
194 PPC_STL r6, SVCPU_R6(r13)
195 PPC_STL r7, SVCPU_R7(r13)
196 PPC_STL r8, SVCPU_R8(r13)
197 PPC_STL r9, SVCPU_R9(r13)
198 PPC_STL r10, SVCPU_R10(r13)
199 PPC_STL r11, SVCPU_R11(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200200
201 /* Restore R1/R2 so we can handle faults */
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000202 PPC_LL r1, HSTATE_HOST_R1(r13)
203 PPC_LL r2, HSTATE_HOST_R2(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200204
205 /* Save guest PC and MSR */
Paul Mackerrasb01c8b52011-06-29 00:18:26 +0000206#ifdef CONFIG_PPC64
207BEGIN_FTR_SECTION
Alexander Graf32c7dbf2012-05-10 03:58:50 +0200208 andi. r0, r12, 0x2
209 cmpwi cr1, r0, 0
Benjamin Herrenschmidta5d4f3a2011-04-05 14:20:31 +1000210 beq 1f
211 mfspr r3,SPRN_HSRR0
212 mfspr r4,SPRN_HSRR1
213 andi. r12,r12,0x3ffd
214 b 2f
Paul Mackerras969391c2011-06-29 00:26:11 +0000215END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
Paul Mackerrasb01c8b52011-06-29 00:18:26 +0000216#endif
Benjamin Herrenschmidta5d4f3a2011-04-05 14:20:31 +10002171: mfsrr0 r3
Alexander Graf07372792010-04-16 00:11:35 +0200218 mfsrr1 r4
Benjamin Herrenschmidta5d4f3a2011-04-05 14:20:31 +10002192:
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000220 PPC_STL r3, SVCPU_PC(r13)
221 PPC_STL r4, SVCPU_SHADOW_SRR1(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200222
223 /* Get scratch'ed off registers */
Paul Mackerras673b1892011-04-05 13:59:58 +1000224 GET_SCRATCH0(r9)
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000225 PPC_LL r8, HSTATE_SCRATCH0(r13)
226 lwz r7, HSTATE_SCRATCH1(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200227
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000228 PPC_STL r9, SVCPU_R13(r13)
229 PPC_STL r8, SVCPU_R12(r13)
230 stw r7, SVCPU_CR(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200231
232 /* Save more register state */
233
234 mfxer r5
235 mfdar r6
236 mfdsisr r7
237 mfctr r8
238 mflr r9
239
Sam bobroffc63517c2015-05-27 09:56:57 +1000240 PPC_STL r5, SVCPU_XER(r13)
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000241 PPC_STL r6, SVCPU_FAULT_DAR(r13)
242 stw r7, SVCPU_FAULT_DSISR(r13)
243 PPC_STL r8, SVCPU_CTR(r13)
244 PPC_STL r9, SVCPU_LR(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200245
246 /*
247 * In order for us to easily get the last instruction,
248 * we got the #vmexit at, we exploit the fact that the
249 * virtual layout is still the same here, so we can just
250 * ld from the guest's PC address
251 */
252
253 /* We only load the last instruction when it's safe */
254 cmpwi r12, BOOK3S_INTERRUPT_DATA_STORAGE
255 beq ld_last_inst
256 cmpwi r12, BOOK3S_INTERRUPT_PROGRAM
257 beq ld_last_inst
Alexander Graf77e675a2011-08-08 16:11:36 +0200258 cmpwi r12, BOOK3S_INTERRUPT_SYSCALL
259 beq ld_last_prev_inst
Alexander Graf6fc55822010-04-20 02:49:49 +0200260 cmpwi r12, BOOK3S_INTERRUPT_ALIGNMENT
261 beq- ld_last_inst
Alexander Graf7ef4e982012-05-10 03:54:58 +0200262#ifdef CONFIG_PPC64
263BEGIN_FTR_SECTION
264 cmpwi r12, BOOK3S_INTERRUPT_H_EMUL_ASSIST
265 beq- ld_last_inst
266END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
Alexander Graf616dff82014-04-29 16:48:44 +0200267BEGIN_FTR_SECTION
268 cmpwi r12, BOOK3S_INTERRUPT_FAC_UNAVAIL
269 beq- ld_last_inst
270END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
Alexander Graf7ef4e982012-05-10 03:54:58 +0200271#endif
Alexander Graf07372792010-04-16 00:11:35 +0200272
273 b no_ld_last_inst
274
Alexander Graf77e675a2011-08-08 16:11:36 +0200275ld_last_prev_inst:
276 addi r3, r3, -4
277
Alexander Graf07372792010-04-16 00:11:35 +0200278ld_last_inst:
279 /* Save off the guest instruction we're at */
280
281 /* In case lwz faults */
282 li r0, KVM_INST_FETCH_FAILED
283
284#ifdef USE_QUICK_LAST_INST
285
286 /* Set guest mode to 'jump over instruction' so if lwz faults
287 * we'll just continue at the next IP. */
288 li r9, KVM_GUEST_MODE_SKIP
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000289 stb r9, HSTATE_IN_GUEST(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200290
291 /* 1) enable paging for data */
292 mfmsr r9
293 ori r11, r9, MSR_DR /* Enable paging for data */
294 mtmsr r11
295 sync
296 /* 2) fetch the instruction */
297 lwz r0, 0(r3)
298 /* 3) disable paging again */
299 mtmsr r9
300 sync
301
302#endif
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000303 stw r0, SVCPU_LAST_INST(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200304
305no_ld_last_inst:
306
307 /* Unset guest mode */
308 li r9, KVM_GUEST_MODE_NONE
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000309 stb r9, HSTATE_IN_GUEST(r13)
Alexander Graf07372792010-04-16 00:11:35 +0200310
311 /* Switch back to host MMU */
312 LOAD_HOST_SEGMENTS
313
Paul Mackerras02143942011-07-23 17:41:44 +1000314#ifdef CONFIG_PPC_BOOK3S_64
315
316 lbz r5, HSTATE_RESTORE_HID5(r13)
317 cmpwi r5, 0
318 beq no_dcbz32_off
319
320 li r4, 0
321 mfspr r5,SPRN_HID5
322 rldimi r5,r4,6,56
323 mtspr SPRN_HID5,r5
324
325no_dcbz32_off:
326
Alexander Graf616dff82014-04-29 16:48:44 +0200327BEGIN_FTR_SECTION
328 /* Save guest FSCR on a FAC_UNAVAIL interrupt */
329 cmpwi r12, BOOK3S_INTERRUPT_FAC_UNAVAIL
330 bne+ no_fscr_save
331 mfspr r7, SPRN_FSCR
332 std r7, SVCPU_SHADOW_FSCR(r13)
333no_fscr_save:
334 /* Restore host FSCR */
335 ld r8, HSTATE_HOST_FSCR(r13)
336 mtspr SPRN_FSCR, r8
337END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
338
Paul Mackerras02143942011-07-23 17:41:44 +1000339#endif /* CONFIG_PPC_BOOK3S_64 */
340
341 /*
342 * For some interrupts, we need to call the real Linux
343 * handler, so it can do work for us. This has to happen
344 * as if the interrupt arrived from the kernel though,
345 * so let's fake it here where most state is restored.
346 *
347 * Having set up SRR0/1 with the address where we want
348 * to continue with relocation on (potentially in module
349 * space), we either just go straight there with rfi[d],
Alexander Graf56e13db2012-04-27 16:33:35 +0200350 * or we jump to an interrupt handler if there is an
351 * interrupt to be handled first. In the latter case,
352 * the rfi[d] at the end of the interrupt handler will
353 * get us back to where we want to continue.
Paul Mackerras02143942011-07-23 17:41:44 +1000354 */
355
Alexander Graf07372792010-04-16 00:11:35 +0200356 /* Register usage at this point:
357 *
358 * R1 = host R1
359 * R2 = host R2
Alexander Graf56e13db2012-04-27 16:33:35 +0200360 * R10 = raw exit handler id
Alexander Graf07372792010-04-16 00:11:35 +0200361 * R12 = exit handler id
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000362 * R13 = shadow vcpu (32-bit) or PACA (64-bit)
Alexander Graf07372792010-04-16 00:11:35 +0200363 * SVCPU.* = guest *
364 *
365 */
366
Paul Mackerras02143942011-07-23 17:41:44 +1000367 PPC_LL r6, HSTATE_HOST_MSR(r13)
Paul Mackerras3c42bf82011-06-29 00:20:58 +0000368 PPC_LL r8, HSTATE_VMHANDLER(r13)
Paul Mackerras02143942011-07-23 17:41:44 +1000369
Alexander Graf56e13db2012-04-27 16:33:35 +0200370#ifdef CONFIG_PPC64
371BEGIN_FTR_SECTION
Alexander Graf32c7dbf2012-05-10 03:58:50 +0200372 beq cr1, 1f
Alexander Graf56e13db2012-04-27 16:33:35 +0200373 mtspr SPRN_HSRR1, r6
374 mtspr SPRN_HSRR0, r8
375END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
376#endif
3771: /* Restore host msr -> SRR1 */
Paul Mackerras02143942011-07-23 17:41:44 +1000378 mtsrr1 r6
379 /* Load highmem handler address */
Alexander Graf07372792010-04-16 00:11:35 +0200380 mtsrr0 r8
381
Paul Mackerras02143942011-07-23 17:41:44 +1000382 /* RFI into the highmem handler, or jump to interrupt handler */
Alexander Graf56e13db2012-04-27 16:33:35 +0200383 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
384 beqa BOOK3S_INTERRUPT_EXTERNAL
385 cmpwi r12, BOOK3S_INTERRUPT_DECREMENTER
386 beqa BOOK3S_INTERRUPT_DECREMENTER
387 cmpwi r12, BOOK3S_INTERRUPT_PERFMON
388 beqa BOOK3S_INTERRUPT_PERFMON
Paul Mackerras40688902014-01-08 21:25:36 +1100389 cmpwi r12, BOOK3S_INTERRUPT_DOORBELL
390 beqa BOOK3S_INTERRUPT_DOORBELL
Alexander Graf56e13db2012-04-27 16:33:35 +0200391
Alexander Graf07372792010-04-16 00:11:35 +0200392 RFI
393kvmppc_handler_trampoline_exit_end: