blob: f9643fc50de571636347a1e0510d45f916728967 [file] [log] [blame]
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -07001/*
Tejun Heo130ace12009-02-06 00:57:48 +09002 * Asm versions of Xen pv-ops, suitable for either direct use or
3 * inlining. The inline versions are the same as the direct-use
4 * versions, with the pre- and post-amble chopped off.
5 *
6 * This code is encoded for size rather than absolute efficiency, with
7 * a view to being able to inline as much as possible.
8 *
9 * We only bother with direct forms (ie, vcpu in pda) of the
10 * operations here; the indirect forms are better handled in C, since
11 * they're generally too large to inline anyway.
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070012 */
13
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070014#include <asm/thread_info.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070015#include <asm/processor-flags.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070016#include <asm/segment.h>
H. Peter Anvin8f6380b2012-04-20 12:19:52 -070017#include <asm/asm.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070018
19#include <xen/interface/xen.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070020
Jeremy Fitzhardinge53937442009-02-02 13:55:42 -080021#include "xen-asm.h"
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070022
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070023/*
Tejun Heo130ace12009-02-06 00:57:48 +090024 * Force an event check by making a hypercall, but preserve regs
25 * before making the call.
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070026 */
Jeremy Fitzhardinge53937442009-02-02 13:55:42 -080027check_events:
28 push %eax
29 push %ecx
30 push %edx
31 call xen_force_evtchn_callback
32 pop %edx
33 pop %ecx
34 pop %eax
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070035 ret
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070036
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070037/*
Tejun Heo130ace12009-02-06 00:57:48 +090038 * We can't use sysexit directly, because we're not running in ring0.
39 * But we can easily fake it up using iret. Assuming xen_sysexit is
40 * jumped to with a standard stack frame, we can just strip it back to
41 * a standard iret frame and use iret.
Jeremy Fitzhardingeb77797f2008-04-02 10:54:11 -070042 */
43ENTRY(xen_sysexit)
44 movl PT_EAX(%esp), %eax /* Shouldn't be necessary? */
45 orl $X86_EFLAGS_IF, PT_EFLAGS(%esp)
46 lea PT_EIP(%esp), %esp
47
48 jmp xen_iret
49ENDPROC(xen_sysexit)
50
51/*
Tejun Heo130ace12009-02-06 00:57:48 +090052 * This is run where a normal iret would be run, with the same stack setup:
53 * 8: eflags
54 * 4: cs
55 * esp-> 0: eip
56 *
57 * This attempts to make sure that any pending events are dealt with
58 * on return to usermode, but there is a small window in which an
59 * event can happen just before entering usermode. If the nested
60 * interrupt ends up setting one of the TIF_WORK_MASK pending work
61 * flags, they will not be tested again before returning to
62 * usermode. This means that a process can end up with pending work,
63 * which will be unprocessed until the process enters and leaves the
64 * kernel again, which could be an unbounded amount of time. This
65 * means that a pending signal or reschedule event could be
66 * indefinitely delayed.
67 *
68 * The fix is to notice a nested interrupt in the critical window, and
69 * if one occurs, then fold the nested interrupt into the current
70 * interrupt stack frame, and re-process it iteratively rather than
71 * recursively. This means that it will exit via the normal path, and
72 * all pending work will be dealt with appropriately.
73 *
74 * Because the nested interrupt handler needs to deal with the current
75 * stack state in whatever form its in, we keep things simple by only
76 * using a single register which is pushed/popped on the stack.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070077 */
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +020078ENTRY(xen_iret)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070079 /* test eflags for special cases */
80 testl $(X86_EFLAGS_VM | XEN_EFLAGS_NMI), 8(%esp)
81 jnz hyper_iret
82
83 push %eax
84 ESP_OFFSET=4 # bytes pushed onto stack
85
Tejun Heo130ace12009-02-06 00:57:48 +090086 /*
87 * Store vcpu_info pointer for easy access. Do it this way to
88 * avoid having to reload %fs
89 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070090#ifdef CONFIG_SMP
91 GET_THREAD_INFO(%eax)
Tejun Heo130ace12009-02-06 00:57:48 +090092 movl TI_cpu(%eax), %eax
93 movl __per_cpu_offset(,%eax,4), %eax
Rusty Russelldd17c8f2009-10-29 22:34:15 +090094 mov xen_vcpu(%eax), %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070095#else
Rusty Russelldd17c8f2009-10-29 22:34:15 +090096 movl xen_vcpu, %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070097#endif
98
99 /* check IF state we're restoring */
100 testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp)
101
Tejun Heo130ace12009-02-06 00:57:48 +0900102 /*
103 * Maybe enable events. Once this happens we could get a
104 * recursive event, so the critical region starts immediately
105 * afterwards. However, if that happens we don't end up
106 * resuming the code, so we don't have to be worried about
107 * being preempted to another CPU.
108 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700109 setz XEN_vcpu_info_mask(%eax)
110xen_iret_start_crit:
111
112 /* check for unmasked and pending */
113 cmpw $0x0001, XEN_vcpu_info_pending(%eax)
114
Tejun Heo130ace12009-02-06 00:57:48 +0900115 /*
116 * If there's something pending, mask events again so we can
Igor Mammedovd198d492011-09-01 13:46:55 +0200117 * jump back into xen_hypervisor_callback. Otherwise do not
118 * touch XEN_vcpu_info_mask.
Tejun Heo130ace12009-02-06 00:57:48 +0900119 */
Igor Mammedovd198d492011-09-01 13:46:55 +0200120 jne 1f
121 movb $1, XEN_vcpu_info_mask(%eax)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700122
Igor Mammedovd198d492011-09-01 13:46:55 +02001231: popl %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700124
Tejun Heo130ace12009-02-06 00:57:48 +0900125 /*
126 * From this point on the registers are restored and the stack
127 * updated, so we don't need to worry about it if we're
128 * preempted
129 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700130iret_restore_end:
131
Tejun Heo130ace12009-02-06 00:57:48 +0900132 /*
133 * Jump to hypervisor_callback after fixing up the stack.
134 * Events are masked, so jumping out of the critical region is
135 * OK.
136 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700137 je xen_hypervisor_callback
138
Jeremy Fitzhardinge90e9f532008-03-17 16:37:12 -07001391: iret
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700140xen_iret_end_crit:
H. Peter Anvin8f6380b2012-04-20 12:19:52 -0700141 _ASM_EXTABLE(1b, iret_exc)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700142
143hyper_iret:
144 /* put this out of line since its very rarely used */
145 jmp hypercall_page + __HYPERVISOR_iret * 32
146
147 .globl xen_iret_start_crit, xen_iret_end_crit
148
149/*
Tejun Heo130ace12009-02-06 00:57:48 +0900150 * This is called by xen_hypervisor_callback in entry.S when it sees
151 * that the EIP at the time of interrupt was between
152 * xen_iret_start_crit and xen_iret_end_crit. We're passed the EIP in
153 * %eax so we can do a more refined determination of what to do.
154 *
155 * The stack format at this point is:
156 * ----------------
157 * ss : (ss/esp may be present if we came from usermode)
158 * esp :
159 * eflags } outer exception info
160 * cs }
161 * eip }
162 * ---------------- <- edi (copy dest)
163 * eax : outer eax if it hasn't been restored
164 * ----------------
165 * eflags } nested exception info
166 * cs } (no ss/esp because we're nested
167 * eip } from the same ring)
168 * orig_eax }<- esi (copy src)
169 * - - - - - - - -
170 * fs }
171 * es }
172 * ds } SAVE_ALL state
173 * eax }
174 * : :
175 * ebx }<- esp
176 * ----------------
177 *
178 * In order to deliver the nested exception properly, we need to shift
179 * everything from the return addr up to the error code so it sits
180 * just under the outer exception info. This means that when we
181 * handle the exception, we do it in the context of the outer
182 * exception rather than starting a new one.
183 *
184 * The only caveat is that if the outer eax hasn't been restored yet
185 * (ie, it's still on stack), we need to insert its value into the
186 * SAVE_ALL state before going on, since it's usermode state which we
187 * eventually need to restore.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700188 */
189ENTRY(xen_iret_crit_fixup)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700190 /*
Tejun Heo130ace12009-02-06 00:57:48 +0900191 * Paranoia: Make sure we're really coming from kernel space.
192 * One could imagine a case where userspace jumps into the
193 * critical range address, but just before the CPU delivers a
194 * GP, it decides to deliver an interrupt instead. Unlikely?
195 * Definitely. Easy to avoid? Yes. The Intel documents
196 * explicitly say that the reported EIP for a bad jump is the
197 * jump instruction itself, not the destination, but some
198 * virtual environments get this wrong.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700199 */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700200 movl PT_CS(%esp), %ecx
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700201 andl $SEGMENT_RPL_MASK, %ecx
202 cmpl $USER_RPL, %ecx
203 je 2f
204
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700205 lea PT_ORIG_EAX(%esp), %esi
206 lea PT_EFLAGS(%esp), %edi
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700207
Tejun Heo130ace12009-02-06 00:57:48 +0900208 /*
209 * If eip is before iret_restore_end then stack
210 * hasn't been restored yet.
211 */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700212 cmp $iret_restore_end, %eax
213 jae 1f
214
Tejun Heo130ace12009-02-06 00:57:48 +0900215 movl 0+4(%edi), %eax /* copy EAX (just above top of frame) */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700216 movl %eax, PT_EAX(%esp)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700217
Tejun Heo130ace12009-02-06 00:57:48 +0900218 lea ESP_OFFSET(%edi), %edi /* move dest up over saved regs */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700219
220 /* set up the copy */
2211: std
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700222 mov $PT_EIP / 4, %ecx /* saved regs up to orig_eax */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700223 rep movsl
224 cld
225
Tejun Heo130ace12009-02-06 00:57:48 +0900226 lea 4(%edi), %esp /* point esp to new frame */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -07002272: jmp xen_do_upcall
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700228