jump_label.h 13.3 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4
#ifndef _LINUX_JUMP_LABEL_H
#define _LINUX_JUMP_LABEL_H

5 6 7 8
/*
 * Jump label support
 *
 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9
 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24
 * DEPRECATED API:
 *
 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
 *
 * struct static_key false = STATIC_KEY_INIT_FALSE;
 * struct static_key true = STATIC_KEY_INIT_TRUE;
 * static_key_true()
 * static_key_false()
 *
 * The updated API replacements are:
 *
 * DEFINE_STATIC_KEY_TRUE(key);
 * DEFINE_STATIC_KEY_FALSE(key);
25 26
 * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
 * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
27 28
 * static_branch_likely()
 * static_branch_unlikely()
29
 *
30
 * Jump labels provide an interface to generate dynamic branches using
31 32 33 34 35 36 37 38 39 40
 * self-modifying code. Assuming toolchain and architecture support, if we
 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
 * (which defaults to false - and the true block is placed out of line).
 * Similarly, we can define an initially true key via
 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
 * "if (static_branch_unlikely(&key))", in which case we will generate an
 * unconditional branch to the out-of-line true branch. Keys that are
 * initially true or false can be using in both static_branch_unlikely()
 * and static_branch_likely() statements.
41
 *
42 43 44 45 46 47 48 49
 * At runtime we can change the branch target by setting the key
 * to true via a call to static_branch_enable(), or false using
 * static_branch_disable(). If the direction of the branch is switched by
 * these calls then we run-time modify the branch target via a
 * no-op -> jump or jump -> no-op conversion. For example, for an
 * initially false key that is used in an "if (static_branch_unlikely(&key))"
 * statement, setting the key to true requires us to patch in a jump
 * to the out-of-line of true branch.
50
 *
51
 * In addition to static_branch_{enable,disable}, we can also reference count
52 53
 * the key or branch direction via static_branch_{inc,dec}. Thus,
 * static_branch_inc() can be thought of as a 'make more true' and
54
 * static_branch_dec() as a 'make more false'.
55 56
 *
 * Since this relies on modifying code, the branch modifying functions
57
 * must be considered absolute slow paths (machine wide synchronization etc.).
58
 * OTOH, since the affected branches are unconditional, their runtime overhead
59 60 61 62
 * will be absolutely minimal, esp. in the default (off) case where the total
 * effect is a single NOP of appropriate size. The on case will patch in a jump
 * to the out-of-line block.
 *
63
 * When the control is directly exposed to userspace, it is prudent to delay the
64
 * decrement to avoid high frequency code modifications which can (and do)
65 66
 * cause significant performance degradation. Struct static_key_deferred and
 * static_key_slow_dec_deferred() provide for this.
67
 *
68 69
 * Lacking toolchain and or architecture support, static keys fall back to a
 * simple conditional branch.
70
 *
71
 * Additional babbling in: Documentation/static-keys.txt
72
 */
73

74 75 76 77 78 79
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
# define HAVE_JUMP_LABEL
#endif

#ifndef __ASSEMBLY__

80 81
#include <linux/types.h>
#include <linux/compiler.h>
82 83 84

extern bool static_key_initialized;

85 86 87
#define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized,		      \
				    "%s(): static key '%pS' used before call to jump_label_init()", \
				    __func__, (key))
88

89
#ifdef HAVE_JUMP_LABEL
90

91
struct static_key {
92
	atomic_t enabled;
93
/*
94 95 96 97 98 99 100
 * Note:
 *   To make anonymous unions work with old compilers, the static
 *   initialization of them requires brackets. This creates a dependency
 *   on the order of the struct with the initializers. If any fields
 *   are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
 *   to be modified.
 *
101 102 103 104 105 106 107 108 109 110
 * bit 0 => 1 if key is initially true
 *	    0 if initially false
 * bit 1 => 1 if points to struct static_key_mod
 *	    0 if points to struct jump_entry
 */
	union {
		unsigned long type;
		struct jump_entry *entries;
		struct static_key_mod *next;
	};
111 112
};

113 114 115 116
#else
struct static_key {
	atomic_t enabled;
};
117 118 119 120 121 122 123 124
#endif	/* HAVE_JUMP_LABEL */
#endif /* __ASSEMBLY__ */

#ifdef HAVE_JUMP_LABEL
#include <asm/jump_label.h>
#endif

#ifndef __ASSEMBLY__
125 126

enum jump_label_type {
127 128
	JUMP_LABEL_NOP = 0,
	JUMP_LABEL_JMP,
129 130 131 132
};

struct module;

133 134
#ifdef HAVE_JUMP_LABEL

135 136 137 138
#define JUMP_TYPE_FALSE		0UL
#define JUMP_TYPE_TRUE		1UL
#define JUMP_TYPE_LINKED	2UL
#define JUMP_TYPE_MASK		3UL
139 140 141

static __always_inline bool static_key_false(struct static_key *key)
{
142
	return arch_static_branch(key, false);
143
}
144

145 146
static __always_inline bool static_key_true(struct static_key *key)
{
147
	return !arch_static_branch(key, true);
148 149
}

150 151 152
extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];

153
extern void jump_label_init(void);
154 155
extern void jump_label_lock(void);
extern void jump_label_unlock(void);
156
extern void arch_jump_label_transform(struct jump_entry *entry,
157
				      enum jump_label_type type);
158 159
extern void arch_jump_label_transform_static(struct jump_entry *entry,
					     enum jump_label_type type);
160
extern int jump_label_text_reserved(void *start, void *end);
161 162
extern void static_key_slow_inc(struct static_key *key);
extern void static_key_slow_dec(struct static_key *key);
163 164
extern void static_key_slow_inc_cpuslocked(struct static_key *key);
extern void static_key_slow_dec_cpuslocked(struct static_key *key);
165
extern void jump_label_apply_nops(struct module *mod);
166 167 168
extern int static_key_count(struct static_key *key);
extern void static_key_enable(struct static_key *key);
extern void static_key_disable(struct static_key *key);
169 170
extern void static_key_enable_cpuslocked(struct static_key *key);
extern void static_key_disable_cpuslocked(struct static_key *key);
171

172 173 174 175 176 177 178
/*
 * We should be using ATOMIC_INIT() for initializing .enabled, but
 * the inclusion of atomic.h is problematic for inclusion of jump_label.h
 * in 'low-level' headers. Thus, we are initializing .enabled with a
 * raw value, but have added a BUILD_BUG_ON() to catch any issues in
 * jump_label_init() see: kernel/jump_label.c.
 */
179
#define STATIC_KEY_INIT_TRUE					\
180
	{ .enabled = { 1 },					\
181
	  { .entries = (void *)JUMP_TYPE_TRUE } }
182
#define STATIC_KEY_INIT_FALSE					\
183
	{ .enabled = { 0 },					\
184
	  { .entries = (void *)JUMP_TYPE_FALSE } }
185

186
#else  /* !HAVE_JUMP_LABEL */
187

188 189 190
#include <linux/atomic.h>
#include <linux/bug.h>

191 192 193 194 195
static inline int static_key_count(struct static_key *key)
{
	return atomic_read(&key->enabled);
}

196 197
static __always_inline void jump_label_init(void)
{
198
	static_key_initialized = true;
199 200
}

201 202
static __always_inline bool static_key_false(struct static_key *key)
{
203
	if (unlikely(static_key_count(key) > 0))
204 205 206 207 208
		return true;
	return false;
}

static __always_inline bool static_key_true(struct static_key *key)
209
{
210
	if (likely(static_key_count(key) > 0))
211 212 213
		return true;
	return false;
}
214

215
static inline void static_key_slow_inc(struct static_key *key)
216
{
217
	STATIC_KEY_CHECK_USE(key);
218 219
	atomic_inc(&key->enabled);
}
220

221
static inline void static_key_slow_dec(struct static_key *key)
222
{
223
	STATIC_KEY_CHECK_USE(key);
224
	atomic_dec(&key->enabled);
225 226
}

227 228 229
#define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
#define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)

230 231 232 233 234
static inline int jump_label_text_reserved(void *start, void *end)
{
	return 0;
}

235 236 237
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}

238 239 240 241
static inline int jump_label_apply_nops(struct module *mod)
{
	return 0;
}
242

243 244
static inline void static_key_enable(struct static_key *key)
{
245
	STATIC_KEY_CHECK_USE(key);
246

247 248 249 250 251
	if (atomic_read(&key->enabled) != 0) {
		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
		return;
	}
	atomic_set(&key->enabled, 1);
252 253 254 255
}

static inline void static_key_disable(struct static_key *key)
{
256
	STATIC_KEY_CHECK_USE(key);
257

258 259 260 261 262
	if (atomic_read(&key->enabled) != 1) {
		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
		return;
	}
	atomic_set(&key->enabled, 0);
263 264
}

265 266 267
#define static_key_enable_cpuslocked(k)		static_key_enable((k))
#define static_key_disable_cpuslocked(k)	static_key_disable((k))

268 269 270 271 272 273 274 275
#define STATIC_KEY_INIT_TRUE	{ .enabled = ATOMIC_INIT(1) }
#define STATIC_KEY_INIT_FALSE	{ .enabled = ATOMIC_INIT(0) }

#endif	/* HAVE_JUMP_LABEL */

#define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
#define jump_label_enabled static_key_enabled

276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
/* -------------------------------------------------------------------------- */

/*
 * Two type wrappers around static_key, such that we can use compile time
 * type differentiation to emit the right code.
 *
 * All the below code is macros in order to play type games.
 */

struct static_key_true {
	struct static_key key;
};

struct static_key_false {
	struct static_key key;
};

#define STATIC_KEY_TRUE_INIT  (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE,  }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }

#define DEFINE_STATIC_KEY_TRUE(name)	\
	struct static_key_true name = STATIC_KEY_TRUE_INIT

299 300 301
#define DECLARE_STATIC_KEY_TRUE(name)	\
	extern struct static_key_true name

302 303 304
#define DEFINE_STATIC_KEY_FALSE(name)	\
	struct static_key_false name = STATIC_KEY_FALSE_INIT

305 306 307
#define DECLARE_STATIC_KEY_FALSE(name)	\
	extern struct static_key_false name

308 309 310 311 312 313 314 315 316 317
#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)		\
	struct static_key_true name[count] = {			\
		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
	}

#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)		\
	struct static_key_false name[count] = {			\
		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
	}

318 319 320 321 322 323 324 325 326 327 328
extern bool ____wrong_branch_error(void);

#define static_key_enabled(x)							\
({										\
	if (!__builtin_types_compatible_p(typeof(*x), struct static_key) &&	\
	    !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
	    !__builtin_types_compatible_p(typeof(*x), struct static_key_false))	\
		____wrong_branch_error();					\
	static_key_count((struct static_key *)x) > 0;				\
})

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
#ifdef HAVE_JUMP_LABEL

/*
 * Combine the right initial value (type) with the right branch order
 * to generate the desired result.
 *
 *
 * type\branch|	likely (1)	      |	unlikely (0)
 * -----------+-----------------------+------------------
 *            |                       |
 *  true (1)  |	   ...		      |	   ...
 *            |    NOP		      |	   JMP L
 *            |    <br-stmts>	      |	1: ...
 *            |	L: ...		      |
 *            |			      |
 *            |			      |	L: <br-stmts>
 *            |			      |	   jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *            |                       |
 *  false (0) |	   ...		      |	   ...
 *            |    JMP L	      |	   NOP
 *            |    <br-stmts>	      |	1: ...
 *            |	L: ...		      |
 *            |			      |
 *            |			      |	L: <br-stmts>
 *            |			      |	   jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *
 * The initial value is encoded in the LSB of static_key::entries,
 * type: 0 = false, 1 = true.
 *
 * The branch type is encoded in the LSB of jump_entry::key,
 * branch: 0 = unlikely, 1 = likely.
 *
 * This gives the following logic table:
 *
 *	enabled	type	branch	  instuction
 * -----------------------------+-----------
 *	0	0	0	| NOP
 *	0	0	1	| JMP
 *	0	1	0	| NOP
 *	0	1	1	| JMP
 *
 *	1	0	0	| JMP
 *	1	0	1	| NOP
 *	1	1	0	| JMP
 *	1	1	1	| NOP
 *
 * Which gives the following functions:
 *
 *   dynamic: instruction = enabled ^ branch
 *   static:  instruction = type ^ branch
 *
 * See jump_label_type() / jump_label_init_type().
 */

#define static_branch_likely(x)							\
({										\
	bool branch;								\
	if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))	\
		branch = !arch_static_branch(&(x)->key, true);			\
	else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
		branch = !arch_static_branch_jump(&(x)->key, true);		\
	else									\
		branch = ____wrong_branch_error();				\
	branch;									\
})

#define static_branch_unlikely(x)						\
({										\
	bool branch;								\
	if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))	\
		branch = arch_static_branch_jump(&(x)->key, false);		\
	else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
		branch = arch_static_branch(&(x)->key, false);			\
	else									\
		branch = ____wrong_branch_error();				\
	branch;									\
})

#else /* !HAVE_JUMP_LABEL */

#define static_branch_likely(x)		likely(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x)	unlikely(static_key_enabled(&(x)->key))

#endif /* HAVE_JUMP_LABEL */

/*
 * Advanced usage; refcount, branch is enabled when: count != 0
 */

#define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
#define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
424 425
#define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
#define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
426 427 428 429 430

/*
 * Normal usage; boolean enable/disable.
 */

431 432 433 434
#define static_branch_enable(x)			static_key_enable(&(x)->key)
#define static_branch_disable(x)		static_key_disable(&(x)->key)
#define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
#define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)
435

436
#endif /* __ASSEMBLY__ */
437 438

#endif	/* _LINUX_JUMP_LABEL_H */