blob: c513a40510f534f3136b015f8fafa2d597b18099 [file] [log] [blame]
Jason Baronbf5438fc2010-09-17 11:09:00 -04001#ifndef _LINUX_JUMP_LABEL_H
2#define _LINUX_JUMP_LABEL_H
3
Peter Zijlstraefb30402012-01-26 13:32:15 +01004/*
5 * Jump label support
6 *
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 *
10 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support the result
Ingo Molnarc5905af2012-02-24 08:31:31 +010012 * of a "if (static_key_false(&key))" statement is a unconditional branch (which
Peter Zijlstraefb30402012-01-26 13:32:15 +010013 * defaults to false - and the true block is placed out of line).
14 *
Ingo Molnarc5905af2012-02-24 08:31:31 +010015 * However at runtime we can change the branch target using
16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
Peter Zijlstraefb30402012-01-26 13:32:15 +010017 * object and for as long as there are references all branches referring to
18 * that particular key will point to the (out of line) true block.
19 *
Ingo Molnarc5905af2012-02-24 08:31:31 +010020 * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
Peter Zijlstraefb30402012-01-26 13:32:15 +010021 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total
24 * effect is a single NOP of appropriate size. The on case will patch in a jump
25 * to the out-of-line block.
26 *
27 * When the control is directly exposed to userspace it is prudent to delay the
28 * decrement to avoid high frequency code modifications which can (and do)
Ingo Molnarc5905af2012-02-24 08:31:31 +010029 * cause significant performance degradation. Struct static_key_deferred and
30 * static_key_slow_dec_deferred() provide for this.
Peter Zijlstraefb30402012-01-26 13:32:15 +010031 *
32 * Lacking toolchain and or architecture support, it falls back to a simple
33 * conditional branch.
Ingo Molnarc5905af2012-02-24 08:31:31 +010034 *
35 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 *
37 * if (static_key_true(&my_key)) {
38 * }
39 *
40 * will result in the true case being in-line and starts the key with a single
41 * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 * allowed.
43 *
44 * Not initializing the key (static data is initialized to 0s anyway) is the
45 * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
46 * equivalent with static_branch().
47 *
48*/
Peter Zijlstraefb30402012-01-26 13:32:15 +010049
Jason Barond430d3d2011-03-16 17:29:47 -040050#include <linux/types.h>
51#include <linux/compiler.h>
Gleb Natapovb2029522011-11-27 17:59:09 +020052#include <linux/workqueue.h>
Jason Barond430d3d2011-03-16 17:29:47 -040053
Steven Rostedt45f81b12010-10-29 12:33:43 -040054#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
Jason Barond430d3d2011-03-16 17:29:47 -040055
Ingo Molnarc5905af2012-02-24 08:31:31 +010056struct static_key {
Jason Barond430d3d2011-03-16 17:29:47 -040057 atomic_t enabled;
Ingo Molnarc5905af2012-02-24 08:31:31 +010058/* Set lsb bit to 1 if branch is default true, 0 ot */
Jason Barond430d3d2011-03-16 17:29:47 -040059 struct jump_entry *entries;
60#ifdef CONFIG_MODULES
Ingo Molnarc5905af2012-02-24 08:31:31 +010061 struct static_key_mod *next;
Jason Barond430d3d2011-03-16 17:29:47 -040062#endif
63};
64
Ingo Molnarc5905af2012-02-24 08:31:31 +010065struct static_key_deferred {
66 struct static_key key;
Gleb Natapovb2029522011-11-27 17:59:09 +020067 unsigned long timeout;
68 struct delayed_work work;
69};
70
Jason Baronbf5438fc2010-09-17 11:09:00 -040071# include <asm/jump_label.h>
72# define HAVE_JUMP_LABEL
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -070073#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
Jason Baronbf5438fc2010-09-17 11:09:00 -040074
75enum jump_label_type {
Jason Barond430d3d2011-03-16 17:29:47 -040076 JUMP_LABEL_DISABLE = 0,
Jason Baronbf5438fc2010-09-17 11:09:00 -040077 JUMP_LABEL_ENABLE,
Jason Baronbf5438fc2010-09-17 11:09:00 -040078};
79
80struct module;
81
82#ifdef HAVE_JUMP_LABEL
83
Ingo Molnarc5905af2012-02-24 08:31:31 +010084#define JUMP_LABEL_TRUE_BRANCH 1UL
Jason Barond430d3d2011-03-16 17:29:47 -040085
Ingo Molnarc5905af2012-02-24 08:31:31 +010086static
87inline struct jump_entry *jump_label_get_entries(struct static_key *key)
88{
89 return (struct jump_entry *)((unsigned long)key->entries
90 & ~JUMP_LABEL_TRUE_BRANCH);
91}
92
93static inline bool jump_label_get_branch_default(struct static_key *key)
94{
95 if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
96 return true;
97 return false;
98}
99
100static __always_inline bool static_key_false(struct static_key *key)
101{
102 return arch_static_branch(key);
103}
104
105static __always_inline bool static_key_true(struct static_key *key)
106{
107 return !static_key_false(key);
108}
109
110/* Deprecated. Please use 'static_key_false() instead. */
111static __always_inline bool static_branch(struct static_key *key)
Jason Barond430d3d2011-03-16 17:29:47 -0400112{
113 return arch_static_branch(key);
114}
115
Jason Baronbf5438fc2010-09-17 11:09:00 -0400116extern struct jump_entry __start___jump_table[];
117extern struct jump_entry __stop___jump_table[];
118
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700119extern void jump_label_init(void);
Jason Baron91bad2f82010-10-01 17:23:48 -0400120extern void jump_label_lock(void);
121extern void jump_label_unlock(void);
Jason Baronbf5438fc2010-09-17 11:09:00 -0400122extern void arch_jump_label_transform(struct jump_entry *entry,
Jeremy Fitzhardinge37348802011-09-29 11:10:05 -0700123 enum jump_label_type type);
Jeremy Fitzhardinge20284aa2011-10-03 11:01:46 -0700124extern void arch_jump_label_transform_static(struct jump_entry *entry,
125 enum jump_label_type type);
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400126extern int jump_label_text_reserved(void *start, void *end);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100127extern void static_key_slow_inc(struct static_key *key);
128extern void static_key_slow_dec(struct static_key *key);
129extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
Jason Barond430d3d2011-03-16 17:29:47 -0400130extern void jump_label_apply_nops(struct module *mod);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100131extern void
132jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
133
134#define STATIC_KEY_INIT_TRUE ((struct static_key) \
135 { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
136#define STATIC_KEY_INIT_FALSE ((struct static_key) \
137 { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
Jason Baronbf5438fc2010-09-17 11:09:00 -0400138
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700139#else /* !HAVE_JUMP_LABEL */
Jason Baronbf5438fc2010-09-17 11:09:00 -0400140
Arun Sharma60063492011-07-26 16:09:06 -0700141#include <linux/atomic.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -0400142
Ingo Molnarc5905af2012-02-24 08:31:31 +0100143struct static_key {
Jason Barond430d3d2011-03-16 17:29:47 -0400144 atomic_t enabled;
145};
Jason Baronbf5438fc2010-09-17 11:09:00 -0400146
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700147static __always_inline void jump_label_init(void)
148{
149}
150
Ingo Molnarc5905af2012-02-24 08:31:31 +0100151struct static_key_deferred {
152 struct static_key key;
Gleb Natapovb2029522011-11-27 17:59:09 +0200153};
154
Ingo Molnarc5905af2012-02-24 08:31:31 +0100155static __always_inline bool static_key_false(struct static_key *key)
Jason Baronbf5438fc2010-09-17 11:09:00 -0400156{
Ingo Molnarc5905af2012-02-24 08:31:31 +0100157 if (unlikely(atomic_read(&key->enabled)) > 0)
Jason Barond430d3d2011-03-16 17:29:47 -0400158 return true;
159 return false;
160}
161
Ingo Molnarc5905af2012-02-24 08:31:31 +0100162static __always_inline bool static_key_true(struct static_key *key)
163{
164 if (likely(atomic_read(&key->enabled)) > 0)
165 return true;
166 return false;
167}
168
169/* Deprecated. Please use 'static_key_false() instead. */
170static __always_inline bool static_branch(struct static_key *key)
171{
172 if (unlikely(atomic_read(&key->enabled)) > 0)
173 return true;
174 return false;
175}
176
177static inline void static_key_slow_inc(struct static_key *key)
Jason Barond430d3d2011-03-16 17:29:47 -0400178{
179 atomic_inc(&key->enabled);
180}
181
Ingo Molnarc5905af2012-02-24 08:31:31 +0100182static inline void static_key_slow_dec(struct static_key *key)
Jason Barond430d3d2011-03-16 17:29:47 -0400183{
184 atomic_dec(&key->enabled);
Jason Baronbf5438fc2010-09-17 11:09:00 -0400185}
186
Ingo Molnarc5905af2012-02-24 08:31:31 +0100187static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
Gleb Natapovb2029522011-11-27 17:59:09 +0200188{
Ingo Molnarc5905af2012-02-24 08:31:31 +0100189 static_key_slow_dec(&key->key);
Gleb Natapovb2029522011-11-27 17:59:09 +0200190}
191
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400192static inline int jump_label_text_reserved(void *start, void *end)
193{
194 return 0;
195}
196
Jason Baron91bad2f82010-10-01 17:23:48 -0400197static inline void jump_label_lock(void) {}
198static inline void jump_label_unlock(void) {}
199
Jason Barond430d3d2011-03-16 17:29:47 -0400200static inline int jump_label_apply_nops(struct module *mod)
201{
202 return 0;
203}
Gleb Natapovb2029522011-11-27 17:59:09 +0200204
Ingo Molnarc5905af2012-02-24 08:31:31 +0100205static inline void
206jump_label_rate_limit(struct static_key_deferred *key,
Gleb Natapovb2029522011-11-27 17:59:09 +0200207 unsigned long rl)
208{
209}
Ingo Molnarc5905af2012-02-24 08:31:31 +0100210
211#define STATIC_KEY_INIT_TRUE ((struct static_key) \
212 { .enabled = ATOMIC_INIT(1) })
213#define STATIC_KEY_INIT_FALSE ((struct static_key) \
214 { .enabled = ATOMIC_INIT(0) })
215
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700216#endif /* HAVE_JUMP_LABEL */
Jason Barond430d3d2011-03-16 17:29:47 -0400217
Ingo Molnarc5905af2012-02-24 08:31:31 +0100218#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
219#define jump_label_enabled static_key_enabled
Peter Zijlstraac99b862011-07-06 14:20:14 +0200220
Jason Baron8eedce92012-02-28 13:49:01 -0500221static inline bool static_key_enabled(struct static_key *key)
222{
223 return (atomic_read(&key->enabled) > 0);
224}
225
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700226#endif /* _LINUX_JUMP_LABEL_H */