41 	if (jea->key < jeb->key)  in jump_label_cmp()
44 	if (jea->key > jeb->key)  in jump_label_cmp()
60 static void jump_label_update(struct static_key *key);
71 int static_key_count(struct static_key *key)  in static_key_count()  argument
77 	int n = atomic_read(&key->enabled);  in static_key_count()
83 void static_key_slow_inc_cpuslocked(struct static_key *key)  in static_key_slow_inc_cpuslocked()  argument
87 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_inc_cpuslocked()
101 	for (v = atomic_read(&key->enabled); v > 0; v = v1) {  in static_key_slow_inc_cpuslocked()
102 		v1 = atomic_cmpxchg(&key->enabled, v, v + 1);  in static_key_slow_inc_cpuslocked()
108 	if (atomic_read(&key->enabled) == 0) {  in static_key_slow_inc_cpuslocked()
109 		atomic_set(&key->enabled, -1);  in static_key_slow_inc_cpuslocked()
110 		jump_label_update(key);  in static_key_slow_inc_cpuslocked()
115 		atomic_set_release(&key->enabled, 1);  in static_key_slow_inc_cpuslocked()
117 		atomic_inc(&key->enabled);  in static_key_slow_inc_cpuslocked()
122 void static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
125 	static_key_slow_inc_cpuslocked(key);  in static_key_slow_inc()
130 void static_key_enable_cpuslocked(struct static_key *key)  in static_key_enable_cpuslocked()  argument
132 	STATIC_KEY_CHECK_USE(key);  in static_key_enable_cpuslocked()
134 	if (atomic_read(&key->enabled) > 0) {  in static_key_enable_cpuslocked()
135 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable_cpuslocked()
140 	if (atomic_read(&key->enabled) == 0) {  in static_key_enable_cpuslocked()
141 		atomic_set(&key->enabled, -1);  in static_key_enable_cpuslocked()
142 		jump_label_update(key);  in static_key_enable_cpuslocked()
146 		atomic_set_release(&key->enabled, 1);  in static_key_enable_cpuslocked()
152 void static_key_enable(struct static_key *key)  in static_key_enable()  argument
155 	static_key_enable_cpuslocked(key);  in static_key_enable()
160 void static_key_disable_cpuslocked(struct static_key *key)  in static_key_disable_cpuslocked()  argument
162 	STATIC_KEY_CHECK_USE(key);  in static_key_disable_cpuslocked()
164 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable_cpuslocked()
165 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable_cpuslocked()
170 	if (atomic_cmpxchg(&key->enabled, 1, 0))  in static_key_disable_cpuslocked()
171 		jump_label_update(key);  in static_key_disable_cpuslocked()
176 void static_key_disable(struct static_key *key)  in static_key_disable()  argument
179 	static_key_disable_cpuslocked(key);  in static_key_disable()
184 static void __static_key_slow_dec_cpuslocked(struct static_key *key,  in __static_key_slow_dec_cpuslocked()  argument
195 	if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {  in __static_key_slow_dec_cpuslocked()
196 		WARN(atomic_read(&key->enabled) < 0,  in __static_key_slow_dec_cpuslocked()
202 		atomic_inc(&key->enabled);  in __static_key_slow_dec_cpuslocked()
205 		jump_label_update(key);  in __static_key_slow_dec_cpuslocked()
210 static void __static_key_slow_dec(struct static_key *key,  in __static_key_slow_dec()  argument
215 	__static_key_slow_dec_cpuslocked(key, rate_limit, work);  in __static_key_slow_dec()
221 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
223 	__static_key_slow_dec(&key->key, 0, NULL);  in jump_label_update_timeout()
226 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
228 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
229 	__static_key_slow_dec(key, 0, NULL);  in static_key_slow_dec()
233 void static_key_slow_dec_cpuslocked(struct static_key *key)  in static_key_slow_dec_cpuslocked()  argument
235 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_cpuslocked()
236 	__static_key_slow_dec_cpuslocked(key, 0, NULL);  in static_key_slow_dec_cpuslocked()
239 void static_key_slow_dec_deferred(struct static_key_deferred *key)  in static_key_slow_dec_deferred()  argument
241 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_deferred()
242 	__static_key_slow_dec(&key->key, key->timeout, &key->work);  in static_key_slow_dec_deferred()
246 void static_key_deferred_flush(struct static_key_deferred *key)  in static_key_deferred_flush()  argument
248 	STATIC_KEY_CHECK_USE(key);  in static_key_deferred_flush()
249 	flush_delayed_work(&key->work);  in static_key_deferred_flush()
253 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
256 	STATIC_KEY_CHECK_USE(key);  in jump_label_rate_limit()
257 	key->timeout = rl;  in jump_label_rate_limit()
258 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
298 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
300 	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);  in static_key_entries()
301 	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);  in static_key_entries()
304 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
306 	return key->type & JUMP_TYPE_TRUE;  in static_key_type()
309 static inline bool static_key_linked(struct static_key *key)  in static_key_linked()  argument
311 	return key->type & JUMP_TYPE_LINKED;  in static_key_linked()
314 static inline void static_key_clear_linked(struct static_key *key)  in static_key_clear_linked()  argument
316 	key->type &= ~JUMP_TYPE_LINKED;  in static_key_clear_linked()
319 static inline void static_key_set_linked(struct static_key *key)  in static_key_set_linked()  argument
321 	key->type |= JUMP_TYPE_LINKED;  in static_key_set_linked()
326 	return (struct static_key *)((unsigned long)entry->key & ~1UL);  in jump_entry_key()
331 	return (unsigned long)entry->key & 1UL;  in jump_entry_branch()
343 static void static_key_set_entries(struct static_key *key,  in static_key_set_entries()  argument
349 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_entries()
350 	key->entries = entries;  in static_key_set_entries()
351 	key->type |= type;  in static_key_set_entries()
356 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
357 	bool enabled = static_key_enabled(key);  in jump_label_type()
364 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
368 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
387 	struct static_key *key = NULL;  in jump_label_init()  local
414 		if (iterk == key)  in jump_label_init()
417 		key = iterk;  in jump_label_init()
418 		static_key_set_entries(key, iter);  in jump_label_init()
442 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
443 	bool type = static_key_type(key);  in jump_label_init_type()
456 static inline struct static_key_mod *static_key_mod(struct static_key *key)  in static_key_mod()  argument
458 	WARN_ON_ONCE(!(key->type & JUMP_TYPE_LINKED));  in static_key_mod()
459 	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);  in static_key_mod()
468 static void static_key_set_mod(struct static_key *key,  in static_key_set_mod()  argument
474 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_mod()
475 	key->next = mod;  in static_key_set_mod()
476 	key->type |= type;  in static_key_set_mod()
497 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
501 	for (mod = static_key_mod(key); mod; mod = mod->next) {  in __jump_label_mod_update()
517 		__jump_label_update(key, mod->entries, stop);  in __jump_label_mod_update()
551 	struct static_key *key = NULL;  in jump_label_add_module()  local
564 		if (iterk == key)  in jump_label_add_module()
567 		key = iterk;  in jump_label_add_module()
568 		if (within_module(iter->key, mod)) {  in jump_label_add_module()
569 			static_key_set_entries(key, iter);  in jump_label_add_module()
575 		if (!static_key_linked(key)) {  in jump_label_add_module()
583 			jlm2->mod = __module_address((unsigned long)key);  in jump_label_add_module()
585 			jlm2->entries = static_key_entries(key);  in jump_label_add_module()
587 			static_key_set_mod(key, jlm2);  in jump_label_add_module()
588 			static_key_set_linked(key);  in jump_label_add_module()
592 		jlm->next = static_key_mod(key);  in jump_label_add_module()
593 		static_key_set_mod(key, jlm);  in jump_label_add_module()
594 		static_key_set_linked(key);  in jump_label_add_module()
598 			__jump_label_update(key, iter, iter_stop);  in jump_label_add_module()
609 	struct static_key *key = NULL;  in jump_label_del_module()  local
613 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
616 		key = jump_entry_key(iter);  in jump_label_del_module()
618 		if (within_module(iter->key, mod))  in jump_label_del_module()
622 		if (WARN_ON(!static_key_linked(key)))  in jump_label_del_module()
625 		prev = &key->next;  in jump_label_del_module()
626 		jlm = static_key_mod(key);  in jump_label_del_module()
637 		if (prev == &key->next)  in jump_label_del_module()
638 			static_key_set_mod(key, jlm->next);  in jump_label_del_module()
644 		jlm = static_key_mod(key);  in jump_label_del_module()
647 			static_key_set_entries(key, jlm->entries);  in jump_label_del_module()
648 			static_key_clear_linked(key);  in jump_label_del_module()
739 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
746 	if (static_key_linked(key)) {  in jump_label_update()
747 		__jump_label_mod_update(key);  in jump_label_update()
752 	mod = __module_address((unsigned long)key);  in jump_label_update()
757 	entry = static_key_entries(key);  in jump_label_update()
760 		__jump_label_update(key, entry, stop);  in jump_label_update()
772 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
773 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
783 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
784 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()