blob: feb6d40a0860f450d5e2c29f2920a139c98794f8 [file] [log] [blame]
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -07001/*
Tejun Heo130ace12009-02-06 00:57:48 +09002 * Asm versions of Xen pv-ops, suitable for either direct use or
3 * inlining. The inline versions are the same as the direct-use
4 * versions, with the pre- and post-amble chopped off.
5 *
6 * This code is encoded for size rather than absolute efficiency, with
7 * a view to being able to inline as much as possible.
8 *
9 * We only bother with direct forms (ie, vcpu in pda) of the
10 * operations here; the indirect forms are better handled in C, since
11 * they're generally too large to inline anyway.
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070012 */
13
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070014#include <asm/thread_info.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070015#include <asm/processor-flags.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070016#include <asm/segment.h>
H. Peter Anvin8f6380b2012-04-20 12:19:52 -070017#include <asm/asm.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070018
19#include <xen/interface/xen.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070020
Jeremy Fitzhardinge53937442009-02-02 13:55:42 -080021#include "xen-asm.h"
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070022
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070023/*
Tejun Heo130ace12009-02-06 00:57:48 +090024 * Force an event check by making a hypercall, but preserve regs
25 * before making the call.
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070026 */
Jeremy Fitzhardinge53937442009-02-02 13:55:42 -080027check_events:
28 push %eax
29 push %ecx
30 push %edx
31 call xen_force_evtchn_callback
32 pop %edx
33 pop %ecx
34 pop %eax
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070035 ret
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070036
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070037/*
Tejun Heo130ace12009-02-06 00:57:48 +090038 * This is run where a normal iret would be run, with the same stack setup:
39 * 8: eflags
40 * 4: cs
41 * esp-> 0: eip
42 *
43 * This attempts to make sure that any pending events are dealt with
44 * on return to usermode, but there is a small window in which an
45 * event can happen just before entering usermode. If the nested
46 * interrupt ends up setting one of the TIF_WORK_MASK pending work
47 * flags, they will not be tested again before returning to
48 * usermode. This means that a process can end up with pending work,
49 * which will be unprocessed until the process enters and leaves the
50 * kernel again, which could be an unbounded amount of time. This
51 * means that a pending signal or reschedule event could be
52 * indefinitely delayed.
53 *
54 * The fix is to notice a nested interrupt in the critical window, and
55 * if one occurs, then fold the nested interrupt into the current
56 * interrupt stack frame, and re-process it iteratively rather than
57 * recursively. This means that it will exit via the normal path, and
58 * all pending work will be dealt with appropriately.
59 *
60 * Because the nested interrupt handler needs to deal with the current
61 * stack state in whatever form its in, we keep things simple by only
62 * using a single register which is pushed/popped on the stack.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070063 */
Boris Ostrovsky4461bbc2014-04-10 12:17:09 -040064
65.macro POP_FS
661:
67 popw %fs
68.pushsection .fixup, "ax"
692: movw $0, (%esp)
70 jmp 1b
71.popsection
72 _ASM_EXTABLE(1b,2b)
73.endm
74
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +020075ENTRY(xen_iret)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070076 /* test eflags for special cases */
77 testl $(X86_EFLAGS_VM | XEN_EFLAGS_NMI), 8(%esp)
78 jnz hyper_iret
79
80 push %eax
81 ESP_OFFSET=4 # bytes pushed onto stack
82
Boris Ostrovsky4461bbc2014-04-10 12:17:09 -040083 /* Store vcpu_info pointer for easy access */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070084#ifdef CONFIG_SMP
Boris Ostrovsky4461bbc2014-04-10 12:17:09 -040085 pushw %fs
86 movl $(__KERNEL_PERCPU), %eax
87 movl %eax, %fs
88 movl %fs:xen_vcpu, %eax
89 POP_FS
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070090#else
Jan Beulich13d2b4d2013-01-24 13:11:10 +000091 movl %ss:xen_vcpu, %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070092#endif
93
94 /* check IF state we're restoring */
95 testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp)
96
Tejun Heo130ace12009-02-06 00:57:48 +090097 /*
98 * Maybe enable events. Once this happens we could get a
99 * recursive event, so the critical region starts immediately
100 * afterwards. However, if that happens we don't end up
101 * resuming the code, so we don't have to be worried about
102 * being preempted to another CPU.
103 */
Jan Beulich13d2b4d2013-01-24 13:11:10 +0000104 setz %ss:XEN_vcpu_info_mask(%eax)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700105xen_iret_start_crit:
106
107 /* check for unmasked and pending */
Jan Beulich13d2b4d2013-01-24 13:11:10 +0000108 cmpw $0x0001, %ss:XEN_vcpu_info_pending(%eax)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700109
Tejun Heo130ace12009-02-06 00:57:48 +0900110 /*
111 * If there's something pending, mask events again so we can
Igor Mammedovd198d492011-09-01 13:46:55 +0200112 * jump back into xen_hypervisor_callback. Otherwise do not
113 * touch XEN_vcpu_info_mask.
Tejun Heo130ace12009-02-06 00:57:48 +0900114 */
Igor Mammedovd198d492011-09-01 13:46:55 +0200115 jne 1f
Jan Beulich13d2b4d2013-01-24 13:11:10 +0000116 movb $1, %ss:XEN_vcpu_info_mask(%eax)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700117
Igor Mammedovd198d492011-09-01 13:46:55 +02001181: popl %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700119
Tejun Heo130ace12009-02-06 00:57:48 +0900120 /*
121 * From this point on the registers are restored and the stack
122 * updated, so we don't need to worry about it if we're
123 * preempted
124 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700125iret_restore_end:
126
Tejun Heo130ace12009-02-06 00:57:48 +0900127 /*
128 * Jump to hypervisor_callback after fixing up the stack.
129 * Events are masked, so jumping out of the critical region is
130 * OK.
131 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700132 je xen_hypervisor_callback
133
Jeremy Fitzhardinge90e9f532008-03-17 16:37:12 -07001341: iret
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700135xen_iret_end_crit:
H. Peter Anvin8f6380b2012-04-20 12:19:52 -0700136 _ASM_EXTABLE(1b, iret_exc)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700137
138hyper_iret:
139 /* put this out of line since its very rarely used */
140 jmp hypercall_page + __HYPERVISOR_iret * 32
141
142 .globl xen_iret_start_crit, xen_iret_end_crit
143
144/*
Tejun Heo130ace12009-02-06 00:57:48 +0900145 * This is called by xen_hypervisor_callback in entry.S when it sees
146 * that the EIP at the time of interrupt was between
147 * xen_iret_start_crit and xen_iret_end_crit. We're passed the EIP in
148 * %eax so we can do a more refined determination of what to do.
149 *
150 * The stack format at this point is:
151 * ----------------
152 * ss : (ss/esp may be present if we came from usermode)
153 * esp :
154 * eflags } outer exception info
155 * cs }
156 * eip }
157 * ---------------- <- edi (copy dest)
158 * eax : outer eax if it hasn't been restored
159 * ----------------
160 * eflags } nested exception info
161 * cs } (no ss/esp because we're nested
162 * eip } from the same ring)
163 * orig_eax }<- esi (copy src)
164 * - - - - - - - -
165 * fs }
166 * es }
167 * ds } SAVE_ALL state
168 * eax }
169 * : :
170 * ebx }<- esp
171 * ----------------
172 *
173 * In order to deliver the nested exception properly, we need to shift
174 * everything from the return addr up to the error code so it sits
175 * just under the outer exception info. This means that when we
176 * handle the exception, we do it in the context of the outer
177 * exception rather than starting a new one.
178 *
179 * The only caveat is that if the outer eax hasn't been restored yet
180 * (ie, it's still on stack), we need to insert its value into the
181 * SAVE_ALL state before going on, since it's usermode state which we
182 * eventually need to restore.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700183 */
184ENTRY(xen_iret_crit_fixup)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700185 /*
Tejun Heo130ace12009-02-06 00:57:48 +0900186 * Paranoia: Make sure we're really coming from kernel space.
187 * One could imagine a case where userspace jumps into the
188 * critical range address, but just before the CPU delivers a
189 * GP, it decides to deliver an interrupt instead. Unlikely?
190 * Definitely. Easy to avoid? Yes. The Intel documents
191 * explicitly say that the reported EIP for a bad jump is the
192 * jump instruction itself, not the destination, but some
193 * virtual environments get this wrong.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700194 */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700195 movl PT_CS(%esp), %ecx
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700196 andl $SEGMENT_RPL_MASK, %ecx
197 cmpl $USER_RPL, %ecx
198 je 2f
199
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700200 lea PT_ORIG_EAX(%esp), %esi
201 lea PT_EFLAGS(%esp), %edi
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700202
Tejun Heo130ace12009-02-06 00:57:48 +0900203 /*
204 * If eip is before iret_restore_end then stack
205 * hasn't been restored yet.
206 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700207 cmp $iret_restore_end, %eax
208 jae 1f
209
Tejun Heo130ace12009-02-06 00:57:48 +0900210 movl 0+4(%edi), %eax /* copy EAX (just above top of frame) */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700211 movl %eax, PT_EAX(%esp)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700212
Tejun Heo130ace12009-02-06 00:57:48 +0900213 lea ESP_OFFSET(%edi), %edi /* move dest up over saved regs */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700214
215 /* set up the copy */
2161: std
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700217 mov $PT_EIP / 4, %ecx /* saved regs up to orig_eax */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700218 rep movsl
219 cld
220
Tejun Heo130ace12009-02-06 00:57:48 +0900221 lea 4(%edi), %esp /* point esp to new frame */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -07002222: jmp xen_do_upcall
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700223