1 /* SPDX-License-Identifier: GPL-2.0 */
8  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
31  * self-modifying code. Assuming toolchain and architecture support, if we
34  * (which defaults to false - and the true block is placed out of line).
38  * unconditional branch to the out-of-line true branch. Keys that are
45  * these calls then we run-time modify the branch target via a
46  * no-op -> jump or jump -> no-op conversion. For example, for an
49  * to the out-of-line of true branch.
61  * to the out-of-line block.
71  * Additional babbling in: Documentation/staging/static-keys.rst
123 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
125 	return (unsigned long)&entry->code + entry->code;  in jump_entry_code()
128 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
130 	return (unsigned long)&entry->target + entry->target;  in jump_entry_target()
133 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
135 	long offset = entry->key & ~3L;  in jump_entry_key()
137 	return (struct static_key *)((unsigned long)&entry->key + offset);  in jump_entry_key()
142 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
144 	return entry->code;  in jump_entry_code()
147 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
149 	return entry->target;  in jump_entry_target()
152 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
154 	return (struct static_key *)((unsigned long)entry->key & ~3UL);  in jump_entry_key()
159 static inline bool jump_entry_is_branch(const struct jump_entry *entry)  in jump_entry_is_branch()  argument
161 	return (unsigned long)entry->key & 1UL;  in jump_entry_is_branch()
164 static inline bool jump_entry_is_init(const struct jump_entry *entry)  in jump_entry_is_init()  argument
166 	return (unsigned long)entry->key & 2UL;  in jump_entry_is_init()
169 static inline void jump_entry_set_init(struct jump_entry *entry, bool set)  in jump_entry_set_init()  argument
172 		entry->key |= 2;  in jump_entry_set_init()
174 		entry->key &= ~2;  in jump_entry_set_init()
177 static inline int jump_entry_size(struct jump_entry *entry)  in jump_entry_size()  argument
182 	return arch_jump_entry_size(entry);  in jump_entry_size()
221 extern void arch_jump_label_transform(struct jump_entry *entry,
223 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
236 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
241  * in 'low-level' headers. Thus, we are initializing .enabled with a
259 	return arch_atomic_read(&key->enabled);  in static_key_count()
284 	atomic_inc(&key->enabled);  in static_key_slow_inc()
290 	atomic_dec(&key->enabled);  in static_key_slow_dec()
308 	if (atomic_read(&key->enabled) != 0) {  in static_key_enable()
309 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable()
312 	atomic_set(&key->enabled, 1);  in static_key_enable()
319 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable()
320 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable()
323 	atomic_set(&key->enabled, 0);  in static_key_disable()
337 /* -------------------------------------------------------------------------- */
357 #define DEFINE_STATIC_KEY_TRUE(name)	\  argument
358 	struct static_key_true name = STATIC_KEY_TRUE_INIT
360 #define DEFINE_STATIC_KEY_TRUE_RO(name)	\  argument
361 	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
363 #define DECLARE_STATIC_KEY_TRUE(name)	\  argument
364 	extern struct static_key_true name
366 #define DEFINE_STATIC_KEY_FALSE(name)	\  argument
367 	struct static_key_false name = STATIC_KEY_FALSE_INIT
369 #define DEFINE_STATIC_KEY_FALSE_RO(name)	\  argument
370 	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
372 #define DECLARE_STATIC_KEY_FALSE(name)	\  argument
373 	extern struct static_key_false name
375 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)		\  argument
376 	struct static_key_true name[count] = {			\
377 		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
380 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)		\  argument
381 	struct static_key_false name[count] = {			\
382 		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
385 #define _DEFINE_STATIC_KEY_1(name)	DEFINE_STATIC_KEY_TRUE(name)  argument
386 #define _DEFINE_STATIC_KEY_0(name)	DEFINE_STATIC_KEY_FALSE(name)  argument
387 #define DEFINE_STATIC_KEY_MAYBE(cfg, name)			\  argument
388 	__PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
390 #define _DEFINE_STATIC_KEY_RO_1(name)	DEFINE_STATIC_KEY_TRUE_RO(name)  argument
391 #define _DEFINE_STATIC_KEY_RO_0(name)	DEFINE_STATIC_KEY_FALSE_RO(name)  argument
392 #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)			\  argument
393 	__PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
395 #define _DECLARE_STATIC_KEY_1(name)	DECLARE_STATIC_KEY_TRUE(name)  argument
396 #define _DECLARE_STATIC_KEY_0(name)	DECLARE_STATIC_KEY_FALSE(name)  argument
397 #define DECLARE_STATIC_KEY_MAYBE(cfg, name)			\  argument
398 	__PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
419  * -----------+-----------------------+------------------
423  *            |    <br-stmts>	      |	1: ...
426  *            |			      |	L: <br-stmts>
429  * -----------+-----------------------+------------------
433  *            |    <br-stmts>	      |	1: ...
436  *            |			      |	L: <br-stmts>
439  * -----------+-----------------------+------------------
450  * -----------------------------+-----------
473 		branch = !arch_static_branch(&(x)->key, true);			\
475 		branch = !arch_static_branch_jump(&(x)->key, true);		\
485 		branch = arch_static_branch_jump(&(x)->key, false);		\
487 		branch = arch_static_branch(&(x)->key, false);			\
495 #define static_branch_likely(x)		likely_notrace(static_key_enabled(&(x)->key))
496 #define static_branch_unlikely(x)	unlikely_notrace(static_key_enabled(&(x)->key))
508 #define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
509 #define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
510 #define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
511 #define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
517 #define static_branch_enable(x)			static_key_enable(&(x)->key)
518 #define static_branch_disable(x)		static_key_disable(&(x)->key)
519 #define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
520 #define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)