[PATCH] i386: PARAVIRT: Use patch site IDs computed from offset in paravirt_ops structure
Use patch type identifiers derived from the offset of the operation in
the paravirt_ops structure. This avoids having to maintain a separate
enum for patch site types.
Also, since the identifier is derived from the offset into
paravirt_ops, the offset can be derived from the identifier. This is
used to remove replicated information in the various callsite macros,
which has been a source of bugs in the past.
This patch also drops the fused save_fl+cli operation, which doesn't
really add much and makes things more complex - specifically because
it breaks the 1:1 relationship between identifiers and offsets. If
this operation turns out to be particularly beneficial, then the right
answer is to define a new entrypoint for it.
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Zachary Amsden <zach@vmware.com>
diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h
index b4cc2fc..1dbc01f 100644
--- a/include/asm-i386/paravirt.h
+++ b/include/asm-i386/paravirt.h
@@ -4,19 +4,8 @@
* para-virtualization: those hooks are defined here. */
#ifdef CONFIG_PARAVIRT
-#include <linux/stringify.h>
#include <asm/page.h>
-/* These are the most performance critical ops, so we want to be able to patch
- * callers */
-#define PARAVIRT_IRQ_DISABLE 0
-#define PARAVIRT_IRQ_ENABLE 1
-#define PARAVIRT_RESTORE_FLAGS 2
-#define PARAVIRT_SAVE_FLAGS 3
-#define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
-#define PARAVIRT_INTERRUPT_RETURN 5
-#define PARAVIRT_STI_SYSEXIT 6
-
/* Bitmask of what can be clobbered: usually at least eax. */
#define CLBR_NONE 0x0
#define CLBR_EAX 0x1
@@ -191,6 +180,28 @@
extern struct paravirt_ops paravirt_ops;
+#define PARAVIRT_PATCH(x) \
+ (offsetof(struct paravirt_ops, x) / sizeof(void *))
+
+#define paravirt_type(type) \
+ [paravirt_typenum] "i" (PARAVIRT_PATCH(type))
+#define paravirt_clobber(clobber) \
+ [paravirt_clobber] "i" (clobber)
+
+#define PARAVIRT_CALL "call *paravirt_ops+%c[paravirt_typenum]*4;"
+
+#define _paravirt_alt(insn_string, type, clobber) \
+ "771:\n\t" insn_string "\n" "772:\n" \
+ ".pushsection .parainstructions,\"a\"\n" \
+ " .long 771b\n" \
+ " .byte " type "\n" \
+ " .byte 772b-771b\n" \
+ " .short " clobber "\n" \
+ ".popsection\n"
+
+#define paravirt_alt(insn_string) \
+ _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
+
#define paravirt_enabled() (paravirt_ops.paravirt_enabled)
static inline void load_esp0(struct tss_struct *tss,
@@ -515,93 +526,89 @@
extern struct paravirt_patch_site __parainstructions[],
__parainstructions_end[];
-#define paravirt_alt(insn_string, typenum, clobber) \
- "771:\n\t" insn_string "\n" "772:\n" \
- ".pushsection .parainstructions,\"a\"\n" \
- " .long 771b\n" \
- " .byte " __stringify(typenum) "\n" \
- " .byte 772b-771b\n" \
- " .short " __stringify(clobber) "\n" \
- ".popsection"
-
static inline unsigned long __raw_local_save_flags(void)
{
unsigned long f;
- __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
- "call *%1;"
- "popl %%edx; popl %%ecx",
- PARAVIRT_SAVE_FLAGS, CLBR_NONE)
- : "=a"(f): "m"(paravirt_ops.save_fl)
- : "memory", "cc");
+ asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
+ PARAVIRT_CALL
+ "popl %%edx; popl %%ecx")
+ : "=a"(f)
+ : paravirt_type(save_fl),
+ paravirt_clobber(CLBR_NONE)
+ : "memory", "cc");
return f;
}
static inline void raw_local_irq_restore(unsigned long f)
{
- __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
- "call *%1;"
- "popl %%edx; popl %%ecx",
- PARAVIRT_RESTORE_FLAGS, CLBR_EAX)
- : "=a"(f) : "m" (paravirt_ops.restore_fl), "0"(f)
- : "memory", "cc");
+ asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
+ PARAVIRT_CALL
+ "popl %%edx; popl %%ecx")
+ : "=a"(f)
+ : "0"(f),
+ paravirt_type(restore_fl),
+ paravirt_clobber(CLBR_EAX)
+ : "memory", "cc");
}
static inline void raw_local_irq_disable(void)
{
- __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
- "call *%0;"
- "popl %%edx; popl %%ecx",
- PARAVIRT_IRQ_DISABLE, CLBR_EAX)
- : : "m" (paravirt_ops.irq_disable)
- : "memory", "eax", "cc");
+ asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
+ PARAVIRT_CALL
+ "popl %%edx; popl %%ecx")
+ :
+ : paravirt_type(irq_disable),
+ paravirt_clobber(CLBR_EAX)
+ : "memory", "eax", "cc");
}
static inline void raw_local_irq_enable(void)
{
- __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
- "call *%0;"
- "popl %%edx; popl %%ecx",
- PARAVIRT_IRQ_ENABLE, CLBR_EAX)
- : : "m" (paravirt_ops.irq_enable)
- : "memory", "eax", "cc");
+ asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
+ PARAVIRT_CALL
+ "popl %%edx; popl %%ecx")
+ :
+ : paravirt_type(irq_enable),
+ paravirt_clobber(CLBR_EAX)
+ : "memory", "eax", "cc");
}
static inline unsigned long __raw_local_irq_save(void)
{
unsigned long f;
- __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
- "call *%1; pushl %%eax;"
- "call *%2; popl %%eax;"
- "popl %%edx; popl %%ecx",
- PARAVIRT_SAVE_FLAGS_IRQ_DISABLE,
- CLBR_NONE)
- : "=a"(f)
- : "m" (paravirt_ops.save_fl),
- "m" (paravirt_ops.irq_disable)
- : "memory", "cc");
+ f = __raw_local_save_flags();
+ raw_local_irq_disable();
return f;
}
-#define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
- "call *paravirt_ops+%c[irq_disable];" \
- "popl %%edx; popl %%ecx", \
- PARAVIRT_IRQ_DISABLE, CLBR_EAX)
+#define CLI_STRING \
+ _paravirt_alt("pushl %%ecx; pushl %%edx;" \
+ "call *paravirt_ops+%c[paravirt_cli_type]*4;" \
+ "popl %%edx; popl %%ecx", \
+ "%c[paravirt_cli_type]", "%c[paravirt_clobber]")
-#define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
- "call *paravirt_ops+%c[irq_enable];" \
- "popl %%edx; popl %%ecx", \
- PARAVIRT_IRQ_ENABLE, CLBR_EAX)
+#define STI_STRING \
+ _paravirt_alt("pushl %%ecx; pushl %%edx;" \
+ "call *paravirt_ops+%c[paravirt_sti_type]*4;" \
+ "popl %%edx; popl %%ecx", \
+ "%c[paravirt_sti_type]", "%c[paravirt_clobber]")
+
#define CLI_STI_CLOBBERS , "%eax"
-#define CLI_STI_INPUT_ARGS \
+#define CLI_STI_INPUT_ARGS \
, \
- [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \
- [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable))
+ [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \
+ [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \
+ paravirt_clobber(CLBR_EAX)
+
+#undef PARAVIRT_CALL
#else /* __ASSEMBLY__ */
-#define PARA_PATCH(ptype, clobbers, ops) \
+#define PARA_PATCH(off) ((off) / 4)
+
+#define PARA_SITE(ptype, clobbers, ops) \
771:; \
ops; \
772:; \
@@ -612,25 +619,25 @@
.short clobbers; \
.popsection
-#define INTERRUPT_RETURN \
- PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \
- jmp *%cs:paravirt_ops+PARAVIRT_iret)
+#define INTERRUPT_RETURN \
+ PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_ANY, \
+ jmp *%cs:paravirt_ops+PARAVIRT_iret)
-#define DISABLE_INTERRUPTS(clobbers) \
- PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \
- pushl %ecx; pushl %edx; \
- call *paravirt_ops+PARAVIRT_irq_disable; \
- popl %edx; popl %ecx) \
+#define DISABLE_INTERRUPTS(clobbers) \
+ PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \
+ pushl %ecx; pushl %edx; \
+ call *%cs:paravirt_ops+PARAVIRT_irq_disable; \
+ popl %edx; popl %ecx) \
-#define ENABLE_INTERRUPTS(clobbers) \
- PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \
- pushl %ecx; pushl %edx; \
- call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
- popl %edx; popl %ecx)
+#define ENABLE_INTERRUPTS(clobbers) \
+ PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \
+ pushl %ecx; pushl %edx; \
+ call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
+ popl %edx; popl %ecx)
-#define ENABLE_INTERRUPTS_SYSEXIT \
- PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \
- jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
+#define ENABLE_INTERRUPTS_SYSEXIT \
+ PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_ANY, \
+ jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
#define GET_CR0_INTO_EAX \
call *paravirt_ops+PARAVIRT_read_cr0