x86: Add new static_cpu_has() function using alternatives
For CPU-feature-specific code that touches performance-critical paths,
introduce a static patching version of [boot_]cpu_has(). This is run
at alternatives time and is therefore not appropriate for most
initialization code, but on the other hand initialization code is
generally not performance critical.
On gcc 4.5+ this uses the new "asm goto" feature.
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Cc: Avi Kivity <avi@redhat.com>
Cc: Suresh Siddha <suresh.b.siddha@intel.com>
LKML-Reference: <1273135546-29690-2-git-send-email-avi@redhat.com>
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 0cd82d0..9b11a5c 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -175,6 +175,7 @@
#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
+#include <asm/asm.h>
#include <linux/bitops.h>
extern const char * const x86_cap_flags[NCAPINTS*32];
@@ -283,6 +284,62 @@
#endif /* CONFIG_X86_64 */
+/*
+ * Static testing of CPU features. Used the same as boot_cpu_has().
+ * These are only valid after alternatives have run, but will statically
+ * patch the target code for additional performance.
+ *
+ */
+static __always_inline __pure bool __static_cpu_has(u8 bit)
+{
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)
+ asm goto("1: jmp %l[t_no]\n"
+ "2:\n"
+ ".section .altinstructions,\"a\"\n"
+ _ASM_ALIGN "\n"
+ _ASM_PTR "1b\n"
+ _ASM_PTR "0\n" /* no replacement */
+ " .byte %P0\n" /* feature bit */
+ " .byte 2b - 1b\n" /* source len */
+ " .byte 0\n" /* replacement len */
+ " .byte 0xff + 0 - (2b-1b)\n" /* padding */
+ ".previous\n"
+ : : "i" (bit) : : t_no);
+ return true;
+ t_no:
+ return false;
+#else
+ u8 flag;
+ /* Open-coded due to __stringify() in ALTERNATIVE() */
+ asm volatile("1: movb $0,%0\n"
+ "2:\n"
+ ".section .altinstructions,\"a\"\n"
+ _ASM_ALIGN "\n"
+ _ASM_PTR "1b\n"
+ _ASM_PTR "3f\n"
+ " .byte %P1\n" /* feature bit */
+ " .byte 2b - 1b\n" /* source len */
+ " .byte 4f - 3f\n" /* replacement len */
+ " .byte 0xff + (4f-3f) - (2b-1b)\n" /* padding */
+ ".previous\n"
+ ".section .altinstr_replacement,\"ax\"\n"
+ "3: movb $1,%0\n"
+ "4:\n"
+ ".previous\n"
+ : "=qm" (flag) : "i" (bit));
+ return flag;
+#endif
+}
+
+#define static_cpu_has(bit) \
+( \
+ __builtin_constant_p(boot_cpu_has(bit)) ? \
+ boot_cpu_has(bit) : \
+ (__builtin_constant_p(bit) && !((bit) & ~0xff)) ? \
+ __static_cpu_has(bit) : \
+ boot_cpu_has(bit) \
+)
+
#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
#endif /* _ASM_X86_CPUFEATURE_H */