1 #ifndef _LINUX_JUMP_LABEL_H
2 #define _LINUX_JUMP_LABEL_H
3 
4 /*
5  * Jump label support
6  *
7  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8  * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9  *
10  * Jump labels provide an interface to generate dynamic branches using
11  * self-modifying code. Assuming toolchain and architecture support the result
12  * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13  * defaults to false - and the true block is placed out of line).
14  *
15  * However at runtime we can change the branch target using
16  * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17  * object and for as long as there are references all branches referring to
18  * that particular key will point to the (out of line) true block.
19  *
20  * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21  * must be considered absolute slow paths (machine wide synchronization etc.).
22  * OTOH, since the affected branches are unconditional their runtime overhead
23  * will be absolutely minimal, esp. in the default (off) case where the total
24  * effect is a single NOP of appropriate size. The on case will patch in a jump
25  * to the out-of-line block.
26  *
27  * When the control is directly exposed to userspace it is prudent to delay the
28  * decrement to avoid high frequency code modifications which can (and do)
29  * cause significant performance degradation. Struct static_key_deferred and
30  * static_key_slow_dec_deferred() provide for this.
31  *
32  * Lacking toolchain and or architecture support, it falls back to a simple
33  * conditional branch.
34  *
35  * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36  *
37  *   if (static_key_true(&my_key)) {
38  *   }
39  *
40  * will result in the true case being in-line and starts the key with a single
41  * reference. Mixing static_key_true() and static_key_false() on the same key is not
42  * allowed.
43  *
44  * Not initializing the key (static data is initialized to 0s anyway) is the
45  * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
46  * equivalent with static_branch().
47  *
48 */
49 
50 #include <linux/types.h>
51 #include <linux/compiler.h>
52 #include <linux/workqueue.h>
53 
54 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
55 
56 struct static_key {
57 	atomic_t enabled;
58 /* Set lsb bit to 1 if branch is default true, 0 ot */
59 	struct jump_entry *entries;
60 #ifdef CONFIG_MODULES
61 	struct static_key_mod *next;
62 #endif
63 };
64 
65 struct static_key_deferred {
66 	struct static_key key;
67 	unsigned long timeout;
68 	struct delayed_work work;
69 };
70 
71 # include <asm/jump_label.h>
72 # define HAVE_JUMP_LABEL
73 #endif	/* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
74 
75 enum jump_label_type {
76 	JUMP_LABEL_DISABLE = 0,
77 	JUMP_LABEL_ENABLE,
78 };
79 
80 struct module;
81 
82 #ifdef HAVE_JUMP_LABEL
83 
84 #define JUMP_LABEL_TRUE_BRANCH 1UL
85 
86 static
jump_label_get_entries(struct static_key * key)87 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
88 {
89 	return (struct jump_entry *)((unsigned long)key->entries
90 						& ~JUMP_LABEL_TRUE_BRANCH);
91 }
92 
jump_label_get_branch_default(struct static_key * key)93 static inline bool jump_label_get_branch_default(struct static_key *key)
94 {
95 	if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
96 		return true;
97 	return false;
98 }
99 
static_key_false(struct static_key * key)100 static __always_inline bool static_key_false(struct static_key *key)
101 {
102 	return arch_static_branch(key);
103 }
104 
static_key_true(struct static_key * key)105 static __always_inline bool static_key_true(struct static_key *key)
106 {
107 	return !static_key_false(key);
108 }
109 
110 /* Deprecated. Please use 'static_key_false() instead. */
static_branch(struct static_key * key)111 static __always_inline bool static_branch(struct static_key *key)
112 {
113 	return arch_static_branch(key);
114 }
115 
116 extern struct jump_entry __start___jump_table[];
117 extern struct jump_entry __stop___jump_table[];
118 
119 extern void jump_label_init(void);
120 extern void jump_label_lock(void);
121 extern void jump_label_unlock(void);
122 extern void arch_jump_label_transform(struct jump_entry *entry,
123 				      enum jump_label_type type);
124 extern void arch_jump_label_transform_static(struct jump_entry *entry,
125 					     enum jump_label_type type);
126 extern int jump_label_text_reserved(void *start, void *end);
127 extern void static_key_slow_inc(struct static_key *key);
128 extern void static_key_slow_dec(struct static_key *key);
129 extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
130 extern void jump_label_apply_nops(struct module *mod);
131 extern void
132 jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
133 
134 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
135 	{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
136 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
137 	{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
138 
139 #else  /* !HAVE_JUMP_LABEL */
140 
141 #include <linux/atomic.h>
142 
143 struct static_key {
144 	atomic_t enabled;
145 };
146 
jump_label_init(void)147 static __always_inline void jump_label_init(void)
148 {
149 }
150 
151 struct static_key_deferred {
152 	struct static_key  key;
153 };
154 
static_key_false(struct static_key * key)155 static __always_inline bool static_key_false(struct static_key *key)
156 {
157 	if (unlikely(atomic_read(&key->enabled)) > 0)
158 		return true;
159 	return false;
160 }
161 
static_key_true(struct static_key * key)162 static __always_inline bool static_key_true(struct static_key *key)
163 {
164 	if (likely(atomic_read(&key->enabled)) > 0)
165 		return true;
166 	return false;
167 }
168 
169 /* Deprecated. Please use 'static_key_false() instead. */
static_branch(struct static_key * key)170 static __always_inline bool static_branch(struct static_key *key)
171 {
172 	if (unlikely(atomic_read(&key->enabled)) > 0)
173 		return true;
174 	return false;
175 }
176 
static_key_slow_inc(struct static_key * key)177 static inline void static_key_slow_inc(struct static_key *key)
178 {
179 	atomic_inc(&key->enabled);
180 }
181 
static_key_slow_dec(struct static_key * key)182 static inline void static_key_slow_dec(struct static_key *key)
183 {
184 	atomic_dec(&key->enabled);
185 }
186 
static_key_slow_dec_deferred(struct static_key_deferred * key)187 static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
188 {
189 	static_key_slow_dec(&key->key);
190 }
191 
jump_label_text_reserved(void * start,void * end)192 static inline int jump_label_text_reserved(void *start, void *end)
193 {
194 	return 0;
195 }
196 
jump_label_lock(void)197 static inline void jump_label_lock(void) {}
jump_label_unlock(void)198 static inline void jump_label_unlock(void) {}
199 
jump_label_apply_nops(struct module * mod)200 static inline int jump_label_apply_nops(struct module *mod)
201 {
202 	return 0;
203 }
204 
205 static inline void
jump_label_rate_limit(struct static_key_deferred * key,unsigned long rl)206 jump_label_rate_limit(struct static_key_deferred *key,
207 		unsigned long rl)
208 {
209 }
210 
211 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
212 		{ .enabled = ATOMIC_INIT(1) })
213 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
214 		{ .enabled = ATOMIC_INIT(0) })
215 
216 #endif	/* HAVE_JUMP_LABEL */
217 
218 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
219 #define jump_label_enabled static_key_enabled
220 
static_key_enabled(struct static_key * key)221 static inline bool static_key_enabled(struct static_key *key)
222 {
223 	return (atomic_read(&key->enabled) > 0);
224 }
225 
226 #endif	/* _LINUX_JUMP_LABEL_H */
227