add support for clang Control Flow Integrity (CFI)
This change adds the CONFIG_CFI_CLANG option, CFI error handling,
and a faster look-up table for cross module CFI checks.
Bug: 67506682
Change-Id: Ic009f0a629b552a0eb16e6d89808c7029e91447d
Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
diff --git a/Makefile b/Makefile
index 24b1f92..dd65b60 100644
--- a/Makefile
+++ b/Makefile
@@ -676,6 +676,33 @@
export LDFINAL_vmlinux LDFLAGS_FINAL_vmlinux
endif
+ifdef CONFIG_CFI_CLANG
+cfi-clang-flags += -fsanitize=cfi
+DISABLE_CFI_CLANG := -fno-sanitize=cfi
+ifdef CONFIG_MODULES
+cfi-clang-flags += -fsanitize-cfi-cross-dso
+DISABLE_CFI_CLANG += -fno-sanitize-cfi-cross-dso
+endif
+ifdef CONFIG_CFI_PERMISSIVE
+cfi-clang-flags += -fsanitize-recover=cfi -fno-sanitize-trap=cfi
+endif
+
+# also disable CFI when LTO is disabled
+DISABLE_LTO_CLANG += $(DISABLE_CFI_CLANG)
+# allow disabling only clang CFI where needed
+export DISABLE_CFI_CLANG
+endif
+
+ifdef CONFIG_CFI
+# cfi-flags are re-tested in prepare-compiler-check
+cfi-flags := $(cfi-clang-flags)
+KBUILD_CFLAGS += $(cfi-flags)
+
+DISABLE_CFI := $(DISABLE_CFI_CLANG)
+DISABLE_LTO += $(DISABLE_CFI)
+export DISABLE_CFI
+endif
+
ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE
KBUILD_CFLAGS += $(call cc-option,-Oz,-Os)
KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,)
@@ -1148,6 +1175,11 @@
$(stackp-flag) available but compiler is broken >&2 && exit 1
endif
endif
+ifdef cfi-flags
+ ifeq ($(call cc-option, $(cfi-flags)),)
+ @echo Cannot use CONFIG_CFI: $(cfi-flags) not supported by compiler >&2 && exit 1
+ endif
+endif
@:
# Generate some files
diff --git a/arch/Kconfig b/arch/Kconfig
index 194d179..c0efc0b 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -518,6 +518,33 @@
5.0 (make CC=clang) and GNU gold from binutils >= 2.27, and have the
LLVMgold plug-in in LD_LIBRARY_PATH.
+config CFI
+ bool
+
+config CFI_PERMISSIVE
+ bool "Use CFI in permissive mode"
+ depends on CFI
+ help
+ When selected, Control Flow Integrity (CFI) violations result in a
+ warning instead of a kernel panic. This option is useful for finding
+ CFI violations in drivers during development.
+
+config CFI_CLANG
+ bool "Use clang Control Flow Integrity (CFI) (EXPERIMENTAL)"
+ depends on LTO_CLANG
+ depends on KALLSYMS
+ select CFI
+ help
+ This option enables clang Control Flow Integrity (CFI), which adds
+ runtime checking for indirect function calls.
+
+config CFI_CLANG_SHADOW
+ bool "Use CFI shadow to speed up cross-module checks"
+ default y
+ depends on CFI_CLANG
+ help
+ If you select this option, the kernel builds a fast look-up table of
+ CFI check functions in loaded modules to reduce overhead.
config HAVE_ARCH_WITHIN_STACK_FRAMES
bool
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index f2d0a7f..f8f0e48 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -67,10 +67,12 @@
*/
#ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION
#define TEXT_MAIN .text .text.[0-9a-zA-Z_]*
+#define TEXT_CFI_MAIN .text.cfi .text.[0-9a-zA-Z_]*.cfi
#define DATA_MAIN .data .data.[0-9a-zA-Z_]*
#define BSS_MAIN .bss .bss.[0-9a-zA-Z_]*
#else
#define TEXT_MAIN .text
+#define TEXT_CFI_MAIN .text.cfi
#define DATA_MAIN .data
#define BSS_MAIN .bss
#endif
@@ -461,6 +463,7 @@
ALIGN_FUNCTION(); \
*(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \
*(.text..ftrace) \
+ *(TEXT_CFI_MAIN) \
*(.ref.text) \
MEM_KEEP(init.text) \
MEM_KEEP(exit.text) \
diff --git a/include/linux/cfi.h b/include/linux/cfi.h
new file mode 100644
index 0000000..e27033d
--- /dev/null
+++ b/include/linux/cfi.h
@@ -0,0 +1,38 @@
+#ifndef _LINUX_CFI_H
+#define _LINUX_CFI_H
+
+#include <linux/stringify.h>
+
+#ifdef CONFIG_CFI_CLANG
+#ifdef CONFIG_MODULES
+
+typedef void (*cfi_check_fn)(uint64_t, void *, void *);
+
+/* Compiler-generated function in each module, and the kernel */
+#define CFI_CHECK_FN __cfi_check
+#define CFI_CHECK_FN_NAME __stringify(CFI_CHECK_FN)
+
+extern void CFI_CHECK_FN(uint64_t, void *, void *);
+
+#ifdef CONFIG_CFI_CLANG_SHADOW
+extern void cfi_module_add(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr);
+
+extern void cfi_module_remove(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr);
+#else
+static inline void cfi_module_add(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr)
+{
+}
+
+static inline void cfi_module_remove(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr)
+{
+}
+#endif /* CONFIG_CFI_CLANG_SHADOW */
+
+#endif /* CONFIG_MODULES */
+#endif /* CONFIG_CFI_CLANG */
+
+#endif /* _LINUX_CFI_H */
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index fd29945..eeb2331 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -21,4 +21,6 @@
#define __norecordmcount \
__attribute__((__section__(".text..ftrace")))
#endif
+
+#define __nocfi __attribute__((no_sanitize("cfi")))
#endif
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 40445d0..81bcdca 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -455,6 +455,10 @@
#define __norecordmcount
#endif
+#ifndef __nocfi
+#define __nocfi
+#endif
+
/*
* Assume alignment of return value.
*/
diff --git a/include/linux/init.h b/include/linux/init.h
index cd365db..2a10aca 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -46,7 +46,7 @@
/* These are for everybody (although not all archs will actually
discard it in modules) */
-#define __init __section(.init.text) __cold notrace __latent_entropy __noretpoline
+#define __init __section(.init.text) __cold notrace __latent_entropy __noretpoline __nocfi
#define __initdata __section(.init.data)
#define __initconst __section(.init.rodata)
#define __exitdata __section(.exit.data)
diff --git a/include/linux/module.h b/include/linux/module.h
index fd9e121..9d6fd1d 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -20,6 +20,7 @@
#include <linux/export.h>
#include <linux/extable.h> /* only as arch move module.h -> extable.h */
#include <linux/rbtree_latch.h>
+#include <linux/cfi.h>
#include <linux/percpu.h>
#include <asm/module.h>
@@ -349,6 +350,10 @@
const unsigned long *crcs;
unsigned int num_syms;
+#ifdef CONFIG_CFI_CLANG
+ cfi_check_fn cfi_check;
+#endif
+
/* Kernel parameters. */
#ifdef CONFIG_SYSFS
struct mutex param_lock;
diff --git a/init/Kconfig b/init/Kconfig
index 544d791..ead5f45 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -2245,7 +2245,7 @@
config MODULES_TREE_LOOKUP
def_bool y
- depends on PERF_EVENTS || TRACING
+ depends on PERF_EVENTS || TRACING || CFI_CLANG
config INIT_ALL_POSSIBLE
bool
diff --git a/kernel/Makefile b/kernel/Makefile
index 314e7d6..108f5be 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -32,6 +32,9 @@
# cond_syscall is currently not LTO compatible
CFLAGS_sys_ni.o = $(DISABLE_LTO)
+# Don't instrument error handlers
+CFLAGS_cfi.o = $(DISABLE_CFI_CLANG)
+
obj-y += sched/
obj-y += locking/
obj-y += power/
@@ -101,6 +104,7 @@
obj-$(CONFIG_IRQ_WORK) += irq_work.o
obj-$(CONFIG_CPU_PM) += cpu_pm.o
obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_CFI_CLANG) += cfi.o
obj-$(CONFIG_PERF_EVENTS) += events/
diff --git a/kernel/cfi.c b/kernel/cfi.c
new file mode 100644
index 0000000..87053e2
--- /dev/null
+++ b/kernel/cfi.c
@@ -0,0 +1,299 @@
+/*
+ * CFI (Control Flow Integrity) error and slowpath handling
+ *
+ * Copyright (C) 2017 Google, Inc.
+ */
+
+#include <linux/gfp.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/rcupdate.h>
+#include <linux/spinlock.h>
+#include <asm/bug.h>
+#include <asm/cacheflush.h>
+#include <asm/memory.h>
+
+/* Compiler-defined handler names */
+#ifdef CONFIG_CFI_PERMISSIVE
+#define cfi_failure_handler __ubsan_handle_cfi_check_fail
+#define cfi_slowpath_handler __cfi_slowpath_diag
+#else /* enforcing */
+#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
+#define cfi_slowpath_handler __cfi_slowpath
+#endif /* CONFIG_CFI_PERMISSIVE */
+
+static inline void handle_cfi_failure()
+{
+#ifdef CONFIG_CFI_PERMISSIVE
+ WARN_RATELIMIT(1, "CFI failure:\n");
+#else
+ pr_err("CFI failure:\n");
+ BUG();
+#endif
+}
+
+#ifdef CONFIG_MODULES
+#ifdef CONFIG_CFI_CLANG_SHADOW
+struct shadow_range {
+ /* Module address range */
+ unsigned long mod_min_addr;
+ unsigned long mod_max_addr;
+ /* Module page range */
+ unsigned long min_page;
+ unsigned long max_page;
+};
+
+#define SHADOW_ORDER 1
+#define SHADOW_PAGES (1 << SHADOW_ORDER)
+#define SHADOW_SIZE \
+ ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16))
+#define SHADOW_INVALID 0xFFFF
+
+struct cfi_shadow {
+ /* Page range covered by the shadow */
+ struct shadow_range r;
+ /* Page offsets to __cfi_check functions in modules */
+ u16 shadow[SHADOW_SIZE];
+};
+
+static DEFINE_SPINLOCK(shadow_update_lock);
+static struct cfi_shadow __rcu *cfi_shadow __read_mostly = NULL;
+
+static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
+{
+ unsigned long index;
+ unsigned long page = ptr >> PAGE_SHIFT;
+
+ if (unlikely(page < s->r.min_page))
+ return -1; /* Outside of module area */
+
+ index = page - s->r.min_page;
+
+ if (index >= SHADOW_SIZE)
+ return -1; /* Cannot be addressed with shadow */
+
+ return (int)index;
+}
+
+static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
+ int index)
+{
+ BUG_ON(index < 0 || index >= SHADOW_SIZE);
+
+ if (unlikely(s->shadow[index] == SHADOW_INVALID))
+ return 0;
+
+ return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT;
+}
+
+static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
+ struct cfi_shadow *next)
+{
+ int i, index, check;
+
+ /* Mark everything invalid */
+ memset(next->shadow, 0xFF, sizeof(next->shadow));
+
+ if (!prev)
+ return; /* No previous shadow */
+
+ /* If the base address didn't change, update is not needed */
+ if (prev->r.min_page == next->r.min_page) {
+ memcpy(next->shadow, prev->shadow, sizeof(next->shadow));
+ return;
+ }
+
+ /* Convert the previous shadow to the new address range */
+ for (i = 0; i < SHADOW_SIZE; ++i) {
+ if (prev->shadow[i] == SHADOW_INVALID)
+ continue;
+
+ index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
+ if (index < 0)
+ continue;
+
+ check = ptr_to_shadow(next,
+ shadow_to_ptr(prev, prev->shadow[i]));
+ if (check < 0)
+ continue;
+
+ next->shadow[index] = (u16)check;
+ }
+}
+
+static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod)
+{
+ unsigned long ptr;
+ unsigned long min_page_addr;
+ unsigned long max_page_addr;
+ unsigned long check = (unsigned long)mod->cfi_check;
+ int check_index = ptr_to_shadow(s, check);
+
+ BUG_ON((check & PAGE_MASK) != check); /* Must be page aligned */
+
+ if (check_index < 0)
+ return; /* Module not addressable with shadow */
+
+ min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
+ max_page_addr = (unsigned long)mod->core_layout.base +
+ mod->core_layout.text_size;
+ max_page_addr &= PAGE_MASK;
+
+ /* For each page, store the check function index in the shadow */
+ for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
+ int index = ptr_to_shadow(s, ptr);
+ if (index >= 0) {
+ /* Assume a page only contains code for one module */
+ BUG_ON(s->shadow[index] != SHADOW_INVALID);
+ s->shadow[index] = (u16)check_index;
+ }
+ }
+}
+
+static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod)
+{
+ unsigned long ptr;
+ unsigned long min_page_addr;
+ unsigned long max_page_addr;
+
+ min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK;
+ max_page_addr = (unsigned long)mod->core_layout.base +
+ mod->core_layout.text_size;
+ max_page_addr &= PAGE_MASK;
+
+ for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) {
+ int index = ptr_to_shadow(s, ptr);
+ if (index >= 0)
+ s->shadow[index] = SHADOW_INVALID;
+ }
+}
+
+typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *);
+
+static void update_shadow(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr, update_shadow_fn fn)
+{
+ struct cfi_shadow *prev;
+ struct cfi_shadow *next = (struct cfi_shadow *)
+ __get_free_pages(GFP_KERNEL, SHADOW_ORDER);
+
+ BUG_ON(!next);
+
+ next->r.mod_min_addr = min_addr;
+ next->r.mod_max_addr = max_addr;
+ next->r.min_page = min_addr >> PAGE_SHIFT;
+ next->r.max_page = max_addr >> PAGE_SHIFT;
+
+ spin_lock(&shadow_update_lock);
+ prev = rcu_dereference_protected(cfi_shadow, 1);
+ prepare_next_shadow(prev, next);
+
+ fn(next, mod);
+ set_memory_ro((unsigned long)next, SHADOW_PAGES);
+ rcu_assign_pointer(cfi_shadow, next);
+
+ spin_unlock(&shadow_update_lock);
+ synchronize_rcu();
+
+ if (prev) {
+ set_memory_rw((unsigned long)prev, SHADOW_PAGES);
+ free_pages((unsigned long)prev, SHADOW_ORDER);
+ }
+}
+
+void cfi_module_add(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr)
+{
+ update_shadow(mod, min_addr, max_addr, add_module_to_shadow);
+}
+EXPORT_SYMBOL(cfi_module_add);
+
+void cfi_module_remove(struct module *mod, unsigned long min_addr,
+ unsigned long max_addr)
+{
+ update_shadow(mod, min_addr, max_addr, remove_module_from_shadow);
+}
+EXPORT_SYMBOL(cfi_module_remove);
+
+static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
+ unsigned long ptr)
+{
+ int index;
+ unsigned long check;
+
+ if (unlikely(!s))
+ return NULL; /* No shadow available */
+
+ if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr)
+ return NULL; /* Not in a mapped module */
+
+ index = ptr_to_shadow(s, ptr);
+ if (index < 0)
+ return NULL; /* Cannot be addressed with shadow */
+
+ return (cfi_check_fn)shadow_to_ptr(s, index);
+}
+#endif /* CONFIG_CFI_CLANG_SHADOW */
+
+static inline cfi_check_fn find_module_cfi_check(void *ptr)
+{
+ struct module *mod;
+
+ preempt_disable();
+ mod = __module_address((unsigned long)ptr);
+ preempt_enable();
+
+ if (mod)
+ return mod->cfi_check;
+
+ return CFI_CHECK_FN;
+}
+
+static inline cfi_check_fn find_cfi_check(void *ptr)
+{
+#ifdef CONFIG_CFI_CLANG_SHADOW
+ cfi_check_fn f;
+
+ if (!rcu_access_pointer(cfi_shadow))
+ return CFI_CHECK_FN; /* No loaded modules */
+
+ /* Look up the __cfi_check function to use */
+ rcu_read_lock();
+ f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr);
+ rcu_read_unlock();
+
+ if (f)
+ return f;
+
+ /*
+ * Fall back to find_module_cfi_check, which works also for a larger
+ * module address space, but is slower.
+ */
+#endif /* CONFIG_CFI_CLANG_SHADOW */
+
+ return find_module_cfi_check(ptr);
+}
+
+void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag)
+{
+ cfi_check_fn check = find_cfi_check(ptr);
+
+ if (likely(check))
+ check(id, ptr, diag);
+ else /* Don't allow unchecked modules */
+ handle_cfi_failure();
+}
+EXPORT_SYMBOL(cfi_slowpath_handler);
+#endif /* CONFIG_MODULES */
+
+void cfi_failure_handler(void *data, void *value, void *vtable)
+{
+ handle_cfi_failure();
+}
+EXPORT_SYMBOL(cfi_failure_handler);
+
+void __cfi_check_fail(void *data, void *value)
+{
+ handle_cfi_failure();
+}
diff --git a/kernel/module.c b/kernel/module.c
index 07bfb99..dec20c7 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -2085,6 +2085,8 @@
{
}
+static void cfi_cleanup(struct module *mod);
+
/* Free a module, remove from lists, etc. */
static void free_module(struct module *mod)
{
@@ -2126,6 +2128,10 @@
/* This may be empty, but that's OK */
disable_ro_nx(&mod->init_layout);
+
+ /* Clean up CFI for the module. */
+ cfi_cleanup(mod);
+
module_arch_freeing_init(mod);
module_memfree(mod->init_layout.base);
kfree(mod->args);
@@ -3307,6 +3313,8 @@
return 0;
}
+static void cfi_init(struct module *mod);
+
static int post_relocation(struct module *mod, const struct load_info *info)
{
/* Sort exception table now relocations are done. */
@@ -3319,6 +3327,9 @@
/* Setup kallsyms-specific fields. */
add_kallsyms(mod, info);
+ /* Setup CFI for the module. */
+ cfi_init(mod);
+
/* Arch-specific module finalizing. */
return module_finalize(info->hdr, info->sechdrs, mod);
}
@@ -4053,6 +4064,22 @@
}
#endif /* CONFIG_KALLSYMS */
+static void cfi_init(struct module *mod)
+{
+#ifdef CONFIG_CFI_CLANG
+ mod->cfi_check =
+ (cfi_check_fn)mod_find_symname(mod, CFI_CHECK_FN_NAME);
+ cfi_module_add(mod, module_addr_min, module_addr_max);
+#endif
+}
+
+static void cfi_cleanup(struct module *mod)
+{
+#ifdef CONFIG_CFI_CLANG
+ cfi_module_remove(mod, module_addr_min, module_addr_max);
+#endif
+}
+
static char *module_flags(struct module *mod, char *buf)
{
int bx = 0;
diff --git a/net/Kconfig b/net/Kconfig
index 28071fc..3775760 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -294,6 +294,7 @@
bool "enable BPF Just In Time compiler"
depends on HAVE_CBPF_JIT || HAVE_EBPF_JIT
depends on MODULES
+ depends on !CFI
---help---
Berkeley Packet Filter filtering capabilities are normally handled
by an interpreter. This option allows kernel to generate a native