jump_label.h 9.5 KB
Newer Older
1 2 3
#ifndef _LINUX_JUMP_LABEL_H
#define _LINUX_JUMP_LABEL_H

4 5 6 7 8 9 10
/*
 * Jump label support
 *
 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
 *
 * Jump labels provide an interface to generate dynamic branches using
11 12
 * self-modifying code. Assuming toolchain and architecture support, the result
 * of a "if (static_key_false(&key))" statement is an unconditional branch (which
13 14
 * defaults to false - and the true block is placed out of line).
 *
15 16
 * However at runtime we can change the branch target using
 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17
 * object, and for as long as there are references all branches referring to
18 19
 * that particular key will point to the (out of line) true block.
 *
20
 * Since this relies on modifying code, the static_key_slow_{inc,dec}() functions
21
 * must be considered absolute slow paths (machine wide synchronization etc.).
22
 * OTOH, since the affected branches are unconditional, their runtime overhead
23 24 25 26
 * will be absolutely minimal, esp. in the default (off) case where the total
 * effect is a single NOP of appropriate size. The on case will patch in a jump
 * to the out-of-line block.
 *
27
 * When the control is directly exposed to userspace, it is prudent to delay the
28
 * decrement to avoid high frequency code modifications which can (and do)
29 30
 * cause significant performance degradation. Struct static_key_deferred and
 * static_key_slow_dec_deferred() provide for this.
31
 *
32
 * Lacking toolchain and or architecture support, jump labels fall back to a simple
33
 * conditional branch.
34 35 36 37 38 39 40 41 42 43 44
 *
 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
 *
 *   if (static_key_true(&my_key)) {
 *   }
 *
 * will result in the true case being in-line and starts the key with a single
 * reference. Mixing static_key_true() and static_key_false() on the same key is not
 * allowed.
 *
 * Not initializing the key (static data is initialized to 0s anyway) is the
J
Jason Baron 已提交
45
 * same as using STATIC_KEY_INIT_FALSE.
46
 */
47

48 49 50 51 52 53
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
# define HAVE_JUMP_LABEL
#endif

#ifndef __ASSEMBLY__

54 55
#include <linux/types.h>
#include <linux/compiler.h>
56 57 58 59 60 61 62
#include <linux/bug.h>

extern bool static_key_initialized;

#define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized,		      \
				    "%s used before call to jump_label_init", \
				    __func__)
63

64
#ifdef HAVE_JUMP_LABEL
65

66
struct static_key {
67
	atomic_t enabled;
68
/* Set lsb bit to 1 if branch is default true, 0 ot */
69 70
	struct jump_entry *entries;
#ifdef CONFIG_MODULES
71
	struct static_key_mod *next;
72 73 74
#endif
};

75 76 77 78
#else
struct static_key {
	atomic_t enabled;
};
79 80 81 82 83 84 85 86
#endif	/* HAVE_JUMP_LABEL */
#endif /* __ASSEMBLY__ */

#ifdef HAVE_JUMP_LABEL
#include <asm/jump_label.h>
#endif

#ifndef __ASSEMBLY__
87 88

enum jump_label_type {
89 90
	JUMP_LABEL_NOP = 0,
	JUMP_LABEL_JMP,
91 92 93 94
};

struct module;

95
#include <linux/atomic.h>
96 97 98 99 100 101

static inline int static_key_count(struct static_key *key)
{
	return atomic_read(&key->enabled);
}

102 103
#ifdef HAVE_JUMP_LABEL

104 105 106
#define JUMP_TYPE_FALSE	0UL
#define JUMP_TYPE_TRUE	1UL
#define JUMP_TYPE_MASK	1UL
107 108 109

static __always_inline bool static_key_false(struct static_key *key)
{
110
	return arch_static_branch(key, false);
111
}
112

113 114
static __always_inline bool static_key_true(struct static_key *key)
{
115
	return !arch_static_branch(key, true);
116 117
}

118 119 120
extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];

121
extern void jump_label_init(void);
122 123
extern void jump_label_lock(void);
extern void jump_label_unlock(void);
124
extern void arch_jump_label_transform(struct jump_entry *entry,
125
				      enum jump_label_type type);
126 127
extern void arch_jump_label_transform_static(struct jump_entry *entry,
					     enum jump_label_type type);
128
extern int jump_label_text_reserved(void *start, void *end);
129 130
extern void static_key_slow_inc(struct static_key *key);
extern void static_key_slow_dec(struct static_key *key);
131
extern void jump_label_apply_nops(struct module *mod);
132

133
#define STATIC_KEY_INIT_TRUE					\
134
	{ .enabled = ATOMIC_INIT(1),				\
135 136
	  .entries = (void *)JUMP_TYPE_TRUE }
#define STATIC_KEY_INIT_FALSE					\
137
	{ .enabled = ATOMIC_INIT(0),				\
138
	  .entries = (void *)JUMP_TYPE_FALSE }
139

140
#else  /* !HAVE_JUMP_LABEL */
141

142 143
static __always_inline void jump_label_init(void)
{
144
	static_key_initialized = true;
145 146
}

147 148
static __always_inline bool static_key_false(struct static_key *key)
{
149
	if (unlikely(static_key_count(key) > 0))
150 151 152 153 154
		return true;
	return false;
}

static __always_inline bool static_key_true(struct static_key *key)
155
{
156
	if (likely(static_key_count(key) > 0))
157 158 159
		return true;
	return false;
}
160

161
static inline void static_key_slow_inc(struct static_key *key)
162
{
163
	STATIC_KEY_CHECK_USE();
164 165
	atomic_inc(&key->enabled);
}
166

167
static inline void static_key_slow_dec(struct static_key *key)
168
{
169
	STATIC_KEY_CHECK_USE();
170
	atomic_dec(&key->enabled);
171 172
}

173 174 175 176 177
static inline int jump_label_text_reserved(void *start, void *end)
{
	return 0;
}

178 179 180
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}

181 182 183 184
static inline int jump_label_apply_nops(struct module *mod)
{
	return 0;
}
185

186 187
#define STATIC_KEY_INIT_TRUE	{ .enabled = ATOMIC_INIT(1) }
#define STATIC_KEY_INIT_FALSE	{ .enabled = ATOMIC_INIT(0) }
188

189
#endif	/* HAVE_JUMP_LABEL */
190

191 192
#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
#define jump_label_enabled static_key_enabled
193

194 195
static inline bool static_key_enabled(struct static_key *key)
{
196
	return static_key_count(key) > 0;
197 198
}

199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
static inline void static_key_enable(struct static_key *key)
{
	int count = static_key_count(key);

	WARN_ON_ONCE(count < 0 || count > 1);

	if (!count)
		static_key_slow_inc(key);
}

static inline void static_key_disable(struct static_key *key)
{
	int count = static_key_count(key);

	WARN_ON_ONCE(count < 0 || count > 1);

	if (count)
		static_key_slow_dec(key);
}

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
/* -------------------------------------------------------------------------- */

/*
 * Two type wrappers around static_key, such that we can use compile time
 * type differentiation to emit the right code.
 *
 * All the below code is macros in order to play type games.
 */

struct static_key_true {
	struct static_key key;
};

struct static_key_false {
	struct static_key key;
};

#define STATIC_KEY_TRUE_INIT  (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE,  }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }

#define DEFINE_STATIC_KEY_TRUE(name)	\
	struct static_key_true name = STATIC_KEY_TRUE_INIT

#define DEFINE_STATIC_KEY_FALSE(name)	\
	struct static_key_false name = STATIC_KEY_FALSE_INIT

#ifdef HAVE_JUMP_LABEL

/*
 * Combine the right initial value (type) with the right branch order
 * to generate the desired result.
 *
 *
 * type\branch|	likely (1)	      |	unlikely (0)
 * -----------+-----------------------+------------------
 *            |                       |
 *  true (1)  |	   ...		      |	   ...
 *            |    NOP		      |	   JMP L
 *            |    <br-stmts>	      |	1: ...
 *            |	L: ...		      |
 *            |			      |
 *            |			      |	L: <br-stmts>
 *            |			      |	   jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *            |                       |
 *  false (0) |	   ...		      |	   ...
 *            |    JMP L	      |	   NOP
 *            |    <br-stmts>	      |	1: ...
 *            |	L: ...		      |
 *            |			      |
 *            |			      |	L: <br-stmts>
 *            |			      |	   jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *
 * The initial value is encoded in the LSB of static_key::entries,
 * type: 0 = false, 1 = true.
 *
 * The branch type is encoded in the LSB of jump_entry::key,
 * branch: 0 = unlikely, 1 = likely.
 *
 * This gives the following logic table:
 *
 *	enabled	type	branch	  instuction
 * -----------------------------+-----------
 *	0	0	0	| NOP
 *	0	0	1	| JMP
 *	0	1	0	| NOP
 *	0	1	1	| JMP
 *
 *	1	0	0	| JMP
 *	1	0	1	| NOP
 *	1	1	0	| JMP
 *	1	1	1	| NOP
 *
 * Which gives the following functions:
 *
 *   dynamic: instruction = enabled ^ branch
 *   static:  instruction = type ^ branch
 *
 * See jump_label_type() / jump_label_init_type().
 */

extern bool ____wrong_branch_error(void);

#define static_branch_likely(x)							\
({										\
	bool branch;								\
	if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))	\
		branch = !arch_static_branch(&(x)->key, true);			\
	else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
		branch = !arch_static_branch_jump(&(x)->key, true);		\
	else									\
		branch = ____wrong_branch_error();				\
	branch;									\
})

#define static_branch_unlikely(x)						\
({										\
	bool branch;								\
	if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))	\
		branch = arch_static_branch_jump(&(x)->key, false);		\
	else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
		branch = arch_static_branch(&(x)->key, false);			\
	else									\
		branch = ____wrong_branch_error();				\
	branch;									\
})

#else /* !HAVE_JUMP_LABEL */

#define static_branch_likely(x)		likely(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x)	unlikely(static_key_enabled(&(x)->key))

#endif /* HAVE_JUMP_LABEL */

/*
 * Advanced usage; refcount, branch is enabled when: count != 0
 */

#define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
#define static_branch_dec(x)		static_key_slow_dec(&(x)->key)

/*
 * Normal usage; boolean enable/disable.
 */

#define static_branch_enable(x)		static_key_enable(&(x)->key)
#define static_branch_disable(x)	static_key_disable(&(x)->key)

350
#endif	/* _LINUX_JUMP_LABEL_H */
351 352

#endif /* __ASSEMBLY__ */