blob: 2497a30f41de0331b93c5f36df5804013a68ee9b [file] [log] [blame]
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -07001/*
2 Asm versions of Xen pv-ops, suitable for either direct use or inlining.
3 The inline versions are the same as the direct-use versions, with the
4 pre- and post-amble chopped off.
5
6 This code is encoded for size rather than absolute efficiency,
7 with a view to being able to inline as much as possible.
8
9 We only bother with direct forms (ie, vcpu in pda) of the operations
10 here; the indirect forms are better handled in C, since they're
11 generally too large to inline anyway.
12 */
13
14#include <linux/linkage.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070015
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070016#include <asm/asm-offsets.h>
17#include <asm/thread_info.h>
18#include <asm/percpu.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070019#include <asm/processor-flags.h>
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070020#include <asm/segment.h>
21
22#include <xen/interface/xen.h>
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070023
24#define RELOC(x, v) .globl x##_reloc; x##_reloc=v
25#define ENDPATCH(x) .globl x##_end; x##_end=.
26
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070027/* Pseudo-flag used for virtual NMI, which we don't implement yet */
28#define XEN_EFLAGS_NMI 0x80000000
29
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070030/*
31 Enable events. This clears the event mask and tests the pending
32 event status with one and operation. If there are pending
33 events, then enter the hypervisor to get them handled.
34 */
35ENTRY(xen_irq_enable_direct)
Jeremy Fitzhardinge04c44a02008-03-17 16:36:52 -070036 /* Unmask events */
37 movb $0, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
38
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070039 /* Preempt here doesn't matter because that will deal with
40 any pending interrupts. The pending check may end up being
41 run on the wrong CPU, but that doesn't hurt. */
Jeremy Fitzhardinge04c44a02008-03-17 16:36:52 -070042
43 /* Test for pending */
44 testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070045 jz 1f
Jeremy Fitzhardinge04c44a02008-03-17 16:36:52 -070046
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700472: call check_events
481:
49ENDPATCH(xen_irq_enable_direct)
50 ret
51 ENDPROC(xen_irq_enable_direct)
52 RELOC(xen_irq_enable_direct, 2b+1)
53
54
55/*
56 Disabling events is simply a matter of making the event mask
57 non-zero.
58 */
59ENTRY(xen_irq_disable_direct)
60 movb $1, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
61ENDPATCH(xen_irq_disable_direct)
62 ret
63 ENDPROC(xen_irq_disable_direct)
64 RELOC(xen_irq_disable_direct, 0)
65
66/*
67 (xen_)save_fl is used to get the current interrupt enable status.
68 Callers expect the status to be in X86_EFLAGS_IF, and other bits
69 may be set in the return value. We take advantage of this by
70 making sure that X86_EFLAGS_IF has the right value (and other bits
71 in that byte are 0), but other bits in the return value are
72 undefined. We need to toggle the state of the bit, because
73 Xen and x86 use opposite senses (mask vs enable).
74 */
75ENTRY(xen_save_fl_direct)
76 testb $0xff, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
77 setz %ah
78 addb %ah,%ah
79ENDPATCH(xen_save_fl_direct)
80 ret
81 ENDPROC(xen_save_fl_direct)
82 RELOC(xen_save_fl_direct, 0)
83
84
85/*
86 In principle the caller should be passing us a value return
87 from xen_save_fl_direct, but for robustness sake we test only
88 the X86_EFLAGS_IF flag rather than the whole byte. After
89 setting the interrupt mask state, it checks for unmasked
90 pending events and enters the hypervisor to get them delivered
91 if so.
92 */
93ENTRY(xen_restore_fl_direct)
94 testb $X86_EFLAGS_IF>>8, %ah
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -070095 setz PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_mask
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -070096 /* Preempt here doesn't matter because that will deal with
97 any pending interrupts. The pending check may end up being
98 run on the wrong CPU, but that doesn't hurt. */
99
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700100 /* check for unmasked and pending */
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700101 cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info)+XEN_vcpu_info_pending
102 jz 1f
1032: call check_events
1041:
105ENDPATCH(xen_restore_fl_direct)
106 ret
107 ENDPROC(xen_restore_fl_direct)
108 RELOC(xen_restore_fl_direct, 2b+1)
109
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700110/*
Jeremy Fitzhardingeb77797f2008-04-02 10:54:11 -0700111 We can't use sysexit directly, because we're not running in ring0.
112 But we can easily fake it up using iret. Assuming xen_sysexit
113 is jumped to with a standard stack frame, we can just strip it
114 back to a standard iret frame and use iret.
115 */
116ENTRY(xen_sysexit)
117 movl PT_EAX(%esp), %eax /* Shouldn't be necessary? */
118 orl $X86_EFLAGS_IF, PT_EFLAGS(%esp)
119 lea PT_EIP(%esp), %esp
120
121 jmp xen_iret
122ENDPROC(xen_sysexit)
123
124/*
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700125 This is run where a normal iret would be run, with the same stack setup:
126 8: eflags
127 4: cs
128 esp-> 0: eip
129
130 This attempts to make sure that any pending events are dealt
131 with on return to usermode, but there is a small window in
132 which an event can happen just before entering usermode. If
133 the nested interrupt ends up setting one of the TIF_WORK_MASK
134 pending work flags, they will not be tested again before
135 returning to usermode. This means that a process can end up
136 with pending work, which will be unprocessed until the process
137 enters and leaves the kernel again, which could be an
138 unbounded amount of time. This means that a pending signal or
139 reschedule event could be indefinitely delayed.
140
141 The fix is to notice a nested interrupt in the critical
142 window, and if one occurs, then fold the nested interrupt into
143 the current interrupt stack frame, and re-process it
144 iteratively rather than recursively. This means that it will
145 exit via the normal path, and all pending work will be dealt
146 with appropriately.
147
148 Because the nested interrupt handler needs to deal with the
149 current stack state in whatever form its in, we keep things
150 simple by only using a single register which is pushed/popped
151 on the stack.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700152 */
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +0200153ENTRY(xen_iret)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700154 /* test eflags for special cases */
155 testl $(X86_EFLAGS_VM | XEN_EFLAGS_NMI), 8(%esp)
156 jnz hyper_iret
157
158 push %eax
159 ESP_OFFSET=4 # bytes pushed onto stack
160
161 /* Store vcpu_info pointer for easy access. Do it this
162 way to avoid having to reload %fs */
163#ifdef CONFIG_SMP
164 GET_THREAD_INFO(%eax)
165 movl TI_cpu(%eax),%eax
166 movl __per_cpu_offset(,%eax,4),%eax
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +0200167 mov per_cpu__xen_vcpu(%eax),%eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700168#else
Jeremy Fitzhardinge81e103f2008-04-17 17:40:51 +0200169 movl per_cpu__xen_vcpu, %eax
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700170#endif
171
172 /* check IF state we're restoring */
173 testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp)
174
175 /* Maybe enable events. Once this happens we could get a
176 recursive event, so the critical region starts immediately
177 afterwards. However, if that happens we don't end up
178 resuming the code, so we don't have to be worried about
179 being preempted to another CPU. */
180 setz XEN_vcpu_info_mask(%eax)
181xen_iret_start_crit:
182
183 /* check for unmasked and pending */
184 cmpw $0x0001, XEN_vcpu_info_pending(%eax)
185
186 /* If there's something pending, mask events again so we
187 can jump back into xen_hypervisor_callback */
188 sete XEN_vcpu_info_mask(%eax)
189
190 popl %eax
191
192 /* From this point on the registers are restored and the stack
193 updated, so we don't need to worry about it if we're preempted */
194iret_restore_end:
195
196 /* Jump to hypervisor_callback after fixing up the stack.
197 Events are masked, so jumping out of the critical
198 region is OK. */
199 je xen_hypervisor_callback
200
Jeremy Fitzhardinge90e9f532008-03-17 16:37:12 -07002011: iret
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700202xen_iret_end_crit:
Jeremy Fitzhardinge90e9f532008-03-17 16:37:12 -0700203.section __ex_table,"a"
204 .align 4
205 .long 1b,iret_exc
206.previous
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700207
208hyper_iret:
209 /* put this out of line since its very rarely used */
210 jmp hypercall_page + __HYPERVISOR_iret * 32
211
212 .globl xen_iret_start_crit, xen_iret_end_crit
213
214/*
215 This is called by xen_hypervisor_callback in entry.S when it sees
216 that the EIP at the time of interrupt was between xen_iret_start_crit
217 and xen_iret_end_crit. We're passed the EIP in %eax so we can do
218 a more refined determination of what to do.
219
220 The stack format at this point is:
221 ----------------
222 ss : (ss/esp may be present if we came from usermode)
223 esp :
224 eflags } outer exception info
225 cs }
226 eip }
227 ---------------- <- edi (copy dest)
228 eax : outer eax if it hasn't been restored
229 ----------------
230 eflags } nested exception info
231 cs } (no ss/esp because we're nested
232 eip } from the same ring)
233 orig_eax }<- esi (copy src)
234 - - - - - - - -
235 fs }
236 es }
237 ds } SAVE_ALL state
238 eax }
239 : :
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700240 ebx }<- esp
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700241 ----------------
242
243 In order to deliver the nested exception properly, we need to shift
244 everything from the return addr up to the error code so it
245 sits just under the outer exception info. This means that when we
246 handle the exception, we do it in the context of the outer exception
247 rather than starting a new one.
248
249 The only caveat is that if the outer eax hasn't been
250 restored yet (ie, it's still on stack), we need to insert
251 its value into the SAVE_ALL state before going on, since
252 it's usermode state which we eventually need to restore.
253 */
254ENTRY(xen_iret_crit_fixup)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700255 /*
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700256 Paranoia: Make sure we're really coming from kernel space.
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700257 One could imagine a case where userspace jumps into the
258 critical range address, but just before the CPU delivers a GP,
259 it decides to deliver an interrupt instead. Unlikely?
260 Definitely. Easy to avoid? Yes. The Intel documents
261 explicitly say that the reported EIP for a bad jump is the
262 jump instruction itself, not the destination, but some virtual
263 environments get this wrong.
264 */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700265 movl PT_CS(%esp), %ecx
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700266 andl $SEGMENT_RPL_MASK, %ecx
267 cmpl $USER_RPL, %ecx
268 je 2f
269
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700270 lea PT_ORIG_EAX(%esp), %esi
271 lea PT_EFLAGS(%esp), %edi
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700272
273 /* If eip is before iret_restore_end then stack
274 hasn't been restored yet. */
275 cmp $iret_restore_end, %eax
276 jae 1f
277
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700278 movl 0+4(%edi),%eax /* copy EAX (just above top of frame) */
279 movl %eax, PT_EAX(%esp)
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700280
281 lea ESP_OFFSET(%edi),%edi /* move dest up over saved regs */
282
283 /* set up the copy */
2841: std
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -0700285 mov $PT_EIP / 4, %ecx /* saved regs up to orig_eax */
Jeremy Fitzhardinge9ec2b802007-07-17 18:37:07 -0700286 rep movsl
287 cld
288
289 lea 4(%edi),%esp /* point esp to new frame */
Jeremy Fitzhardinge0f2c8762008-03-17 16:37:22 -07002902: jmp xen_do_upcall
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700291
292
Jeremy Fitzhardinge64876732007-07-17 18:37:07 -0700293/*
294 Force an event check by making a hypercall,
295 but preserve regs before making the call.
296 */
297check_events:
298 push %eax
299 push %ecx
300 push %edx
301 call force_evtchn_callback
302 pop %edx
303 pop %ecx
304 pop %eax
305 ret