1 /* SPDX-License-Identifier: GPL-2.0 */
8  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
31  * self-modifying code. Assuming toolchain and architecture support, if we
34  * (which defaults to false - and the true block is placed out of line).
38  * unconditional branch to the out-of-line true branch. Keys that are
45  * these calls then we run-time modify the branch target via a
46  * no-op -> jump or jump -> no-op conversion. For example, for an
49  * to the out-of-line of true branch.
61  * to the out-of-line block.
71  * Additional babbling in: Documentation/staging/static-keys.rst
128 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
130 	return (unsigned long)&entry->code + entry->code;  in jump_entry_code()
133 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
135 	return (unsigned long)&entry->target + entry->target;  in jump_entry_target()
138 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
140 	long offset = entry->key & ~3L;  in jump_entry_key()
142 	return (struct static_key *)((unsigned long)&entry->key + offset);  in jump_entry_key()
147 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
149 	return entry->code;  in jump_entry_code()
152 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
154 	return entry->target;  in jump_entry_target()
157 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
159 	return (struct static_key *)((unsigned long)entry->key & ~3UL);  in jump_entry_key()
164 static inline bool jump_entry_is_branch(const struct jump_entry *entry)  in jump_entry_is_branch()  argument
166 	return (unsigned long)entry->key & 1UL;  in jump_entry_is_branch()
169 static inline bool jump_entry_is_init(const struct jump_entry *entry)  in jump_entry_is_init()  argument
171 	return (unsigned long)entry->key & 2UL;  in jump_entry_is_init()
174 static inline void jump_entry_set_init(struct jump_entry *entry, bool set)  in jump_entry_set_init()  argument
177 		entry->key |= 2;  in jump_entry_set_init()
179 		entry->key &= ~2;  in jump_entry_set_init()
182 static inline int jump_entry_size(struct jump_entry *entry)  in jump_entry_size()  argument
187 	return arch_jump_entry_size(entry);  in jump_entry_size()
226 extern void arch_jump_label_transform(struct jump_entry *entry,
228 extern void arch_jump_label_transform_static(struct jump_entry *entry,
230 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
248  * in 'low-level' headers. Thus, we are initializing .enabled with a
266 	return atomic_read(&key->enabled);  in static_key_count()
291 	atomic_inc(&key->enabled);  in static_key_slow_inc()
297 	atomic_dec(&key->enabled);  in static_key_slow_dec()
320 	if (atomic_read(&key->enabled) != 0) {  in static_key_enable()
321 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable()
324 	atomic_set(&key->enabled, 1);  in static_key_enable()
331 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable()
332 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable()
335 	atomic_set(&key->enabled, 0);  in static_key_disable()
349 /* -------------------------------------------------------------------------- */
369 #define DEFINE_STATIC_KEY_TRUE(name)	\  argument
370 	struct static_key_true name = STATIC_KEY_TRUE_INIT
372 #define DEFINE_STATIC_KEY_TRUE_RO(name)	\  argument
373 	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
375 #define DECLARE_STATIC_KEY_TRUE(name)	\  argument
376 	extern struct static_key_true name
378 #define DEFINE_STATIC_KEY_FALSE(name)	\  argument
379 	struct static_key_false name = STATIC_KEY_FALSE_INIT
381 #define DEFINE_STATIC_KEY_FALSE_RO(name)	\  argument
382 	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
384 #define DECLARE_STATIC_KEY_FALSE(name)	\  argument
385 	extern struct static_key_false name
387 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)		\  argument
388 	struct static_key_true name[count] = {			\
389 		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
392 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)		\  argument
393 	struct static_key_false name[count] = {			\
394 		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
397 #define _DEFINE_STATIC_KEY_1(name)	DEFINE_STATIC_KEY_TRUE(name)  argument
398 #define _DEFINE_STATIC_KEY_0(name)	DEFINE_STATIC_KEY_FALSE(name)  argument
399 #define DEFINE_STATIC_KEY_MAYBE(cfg, name)			\  argument
400 	__PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
402 #define _DEFINE_STATIC_KEY_RO_1(name)	DEFINE_STATIC_KEY_TRUE_RO(name)  argument
403 #define _DEFINE_STATIC_KEY_RO_0(name)	DEFINE_STATIC_KEY_FALSE_RO(name)  argument
404 #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)			\  argument
405 	__PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
407 #define _DECLARE_STATIC_KEY_1(name)	DECLARE_STATIC_KEY_TRUE(name)  argument
408 #define _DECLARE_STATIC_KEY_0(name)	DECLARE_STATIC_KEY_FALSE(name)  argument
409 #define DECLARE_STATIC_KEY_MAYBE(cfg, name)			\  argument
410 	__PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
431  * -----------+-----------------------+------------------
435  *            |    <br-stmts>	      |	1: ...
438  *            |			      |	L: <br-stmts>
441  * -----------+-----------------------+------------------
445  *            |    <br-stmts>	      |	1: ...
448  *            |			      |	L: <br-stmts>
451  * -----------+-----------------------+------------------
462  * -----------------------------+-----------
485 		branch = !arch_static_branch(&(x)->key, true);			\
487 		branch = !arch_static_branch_jump(&(x)->key, true);		\
497 		branch = arch_static_branch_jump(&(x)->key, false);		\
499 		branch = arch_static_branch(&(x)->key, false);			\
507 #define static_branch_likely(x)		likely_notrace(static_key_enabled(&(x)->key))
508 #define static_branch_unlikely(x)	unlikely_notrace(static_key_enabled(&(x)->key))
520 #define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
521 #define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
522 #define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
523 #define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
529 #define static_branch_enable(x)			static_key_enable(&(x)->key)
530 #define static_branch_disable(x)		static_key_disable(&(x)->key)
531 #define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
532 #define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)