206 	struct tipc_aead __rcu *aead[KEY_MAX + 1];  member
240 	struct tipc_aead *aead;  member
247 	struct tipc_aead *aead;  member
251 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
252 static inline void tipc_aead_put(struct tipc_aead *aead);
254 static int tipc_aead_users(struct tipc_aead __rcu *aead);
255 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
256 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
257 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
258 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
259 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
266 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
271 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
275 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
283 				  struct tipc_aead *aead, u8 pos,
296 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
383 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead)  in tipc_aead_get()  argument
388 	tmp = rcu_dereference(aead);  in tipc_aead_get()
396 static inline void tipc_aead_put(struct tipc_aead *aead)  in tipc_aead_put()  argument
398 	if (aead && refcount_dec_and_test(&aead->refcnt))  in tipc_aead_put()
399 		call_rcu(&aead->rcu, tipc_aead_free);  in tipc_aead_put()
408 	struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu);  in tipc_aead_free()  local
411 	if (aead->cloned) {  in tipc_aead_free()
412 		tipc_aead_put(aead->cloned);  in tipc_aead_free()
414 		head = *get_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
415 		put_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
426 	free_percpu(aead->tfm_entry);  in tipc_aead_free()
427 	kfree_sensitive(aead->key);  in tipc_aead_free()
428 	kfree(aead);  in tipc_aead_free()
431 static int tipc_aead_users(struct tipc_aead __rcu *aead)  in tipc_aead_users()  argument
437 	tmp = rcu_dereference(aead);  in tipc_aead_users()
445 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_inc()  argument
450 	tmp = rcu_dereference(aead);  in tipc_aead_users_inc()
456 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_dec()  argument
461 	tmp = rcu_dereference(aead);  in tipc_aead_users_dec()
463 		atomic_add_unless(&rcu_dereference(aead)->users, -1, lim);  in tipc_aead_users_dec()
467 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val)  in tipc_aead_users_set()  argument
473 	tmp = rcu_dereference(aead);  in tipc_aead_users_set()
488 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead)  in tipc_aead_tfm_next()  argument
493 	tfm_entry = get_cpu_ptr(aead->tfm_entry);  in tipc_aead_tfm_next()
514 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,  in tipc_aead_init()  argument
523 	if (unlikely(*aead))  in tipc_aead_init()
605 	*aead = tmp;  in tipc_aead_init()
624 	struct tipc_aead *aead;  in tipc_aead_clone()  local
636 	aead = kzalloc(sizeof(*aead), GFP_ATOMIC);  in tipc_aead_clone()
637 	if (unlikely(!aead))  in tipc_aead_clone()
640 	aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC);  in tipc_aead_clone()
641 	if (unlikely(!aead->tfm_entry)) {  in tipc_aead_clone()
642 		kfree_sensitive(aead);  in tipc_aead_clone()
647 		*per_cpu_ptr(aead->tfm_entry, cpu) =  in tipc_aead_clone()
651 	memcpy(aead->hint, src->hint, sizeof(src->hint));  in tipc_aead_clone()
652 	aead->mode = src->mode;  in tipc_aead_clone()
653 	aead->salt = src->salt;  in tipc_aead_clone()
654 	aead->authsize = src->authsize;  in tipc_aead_clone()
655 	atomic_set(&aead->users, 0);  in tipc_aead_clone()
656 	atomic64_set(&aead->seqno, 0);  in tipc_aead_clone()
657 	refcount_set(&aead->refcnt, 1);  in tipc_aead_clone()
660 	aead->cloned = src;  in tipc_aead_clone()
662 	*dst = aead;  in tipc_aead_clone()
728 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,  in tipc_aead_encrypt()  argument
733 	struct crypto_aead *tfm = tipc_aead_tfm_next(aead);  in tipc_aead_encrypt()
746 	tailen = len - skb->len + aead->authsize;  in tipc_aead_encrypt()
799 	salt = aead->salt;  in tipc_aead_encrypt()
800 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_encrypt()
817 	tx_ctx->aead = aead;  in tipc_aead_encrypt()
845 	struct tipc_aead *aead = tx_ctx->aead;  in tipc_aead_encrypt_done()  local
846 	struct tipc_crypto *tx = aead->crypto;  in tipc_aead_encrypt_done()
869 	tipc_aead_put(aead);  in tipc_aead_encrypt_done()
884 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,  in tipc_aead_decrypt()  argument
898 	if (unlikely(!aead))  in tipc_aead_decrypt()
908 	tfm = tipc_aead_tfm_next(aead);  in tipc_aead_decrypt()
924 	salt = aead->salt;  in tipc_aead_decrypt()
925 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_decrypt()
942 	rx_ctx->aead = aead;  in tipc_aead_decrypt()
969 	struct tipc_aead *aead = rx_ctx->aead;  in tipc_aead_decrypt_done()  local
970 	struct tipc_crypto_stats __percpu *stats = aead->crypto->stats;  in tipc_aead_decrypt_done()
971 	struct net *net = aead->crypto->net;  in tipc_aead_decrypt_done()
985 	tipc_crypto_rcv_complete(net, aead, b, &skb, err);  in tipc_aead_decrypt_done()
1037 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,  in tipc_ehdr_build()  argument
1056 	if (!__rx || aead->mode == CLUSTER_KEY)  in tipc_ehdr_build()
1057 		seqno = atomic64_inc_return(&aead->seqno);  in tipc_ehdr_build()
1076 	ehdr->master_key = aead->crypto->key_master;  in tipc_ehdr_build()
1129 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_init()  local
1133 	rc = tipc_aead_init(&aead, ukey, mode);  in tipc_crypto_key_init()
1137 		rc = tipc_crypto_key_attach(c, aead, 0, master_key);  in tipc_crypto_key_init()
1139 			tipc_aead_free(&aead->rcu);  in tipc_crypto_key_init()
1155 				  struct tipc_aead *aead, u8 pos,  in tipc_crypto_key_attach()  argument
1171 		if (tipc_aead_users(c->aead[key.pending]) > 0)  in tipc_crypto_key_attach()
1193 	aead->crypto = c;  in tipc_crypto_key_attach()
1194 	aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen;  in tipc_crypto_key_attach()
1195 	tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock);  in tipc_crypto_key_attach()
1228 			tipc_aead_users_dec(tx->aead[k], 0);  in tipc_crypto_key_flush()
1237 		tipc_crypto_key_detach(c->aead[k], &c->lock);  in tipc_crypto_key_flush()
1273 	if (tipc_aead_users(rx->aead[key.pending]) > 0)  in tipc_crypto_key_try_align()
1277 	tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock);  in tipc_crypto_key_try_align()
1280 	rcu_assign_pointer(rx->aead[key.pending], NULL);  in tipc_crypto_key_try_align()
1284 		tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock));  in tipc_crypto_key_try_align()
1291 	rcu_assign_pointer(rx->aead[new_pending], tmp1);  in tipc_crypto_key_try_align()
1293 		rcu_assign_pointer(rx->aead[new_passive], tmp2);  in tipc_crypto_key_try_align()
1323 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_pick_tx()  local
1340 		aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock);  in tipc_crypto_key_pick_tx()
1348 		aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock);  in tipc_crypto_key_pick_tx()
1349 		if (!aead)  in tipc_crypto_key_pick_tx()
1351 		if (aead->mode != CLUSTER_KEY ||  in tipc_crypto_key_pick_tx()
1352 		    aead == skb_cb->tx_clone_ctx.last) {  in tipc_crypto_key_pick_tx()
1353 			aead = NULL;  in tipc_crypto_key_pick_tx()
1357 		skb_cb->tx_clone_ctx.last = aead;  in tipc_crypto_key_pick_tx()
1366 	if (likely(aead))  in tipc_crypto_key_pick_tx()
1367 		WARN_ON(!refcount_inc_not_zero(&aead->refcnt));  in tipc_crypto_key_pick_tx()
1370 	return aead;  in tipc_crypto_key_pick_tx()
1433 			tipc_aead_users_inc(tx->aead[new], INT_MAX);  in tipc_crypto_key_synch()
1435 			tipc_aead_users_dec(tx->aead[cur], 0);  in tipc_crypto_key_synch()
1457 	tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_key_revoke()
1538 		tipc_aead_put(rcu_dereference(c->aead[k]));  in tipc_crypto_stop()
1559 	if (key.active && tipc_aead_users(tx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1561 	if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1568 		tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_timeout()
1578 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1592 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10)  in tipc_crypto_timeout()
1596 	tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock);  in tipc_crypto_timeout()
1605 	    tipc_aead_users(rx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1614 	tipc_aead_users_set(rx->aead[key.pending], 0);  in tipc_crypto_timeout()
1623 	    tipc_aead_users(rx->aead[key.passive]) > -10)  in tipc_crypto_timeout()
1627 	tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock);  in tipc_crypto_timeout()
1696 	struct tipc_aead *aead = NULL;  in tipc_crypto_xmit()  local
1758 	aead = tipc_aead_get(tx->aead[tx_key]);  in tipc_crypto_xmit()
1759 	if (unlikely(!aead))  in tipc_crypto_xmit()
1761 	rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx);  in tipc_crypto_xmit()
1763 		rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode);  in tipc_crypto_xmit()
1786 	tipc_aead_put(aead);  in tipc_crypto_xmit()
1817 	struct tipc_aead *aead = NULL;  in tipc_crypto_rcv()  local
1842 	aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key);  in tipc_crypto_rcv()
1843 	if (aead)  in tipc_crypto_rcv()
1849 	if (!aead)  in tipc_crypto_rcv()
1850 		aead = tipc_aead_get(rx->aead[tx_key]);  in tipc_crypto_rcv()
1851 	rc = tipc_aead_decrypt(net, aead, *skb, b);  in tipc_crypto_rcv()
1877 					      rcu_access_pointer(rx->aead[n]));  in tipc_crypto_rcv()
1891 	tipc_crypto_rcv_complete(net, aead, b, skb, rc);  in tipc_crypto_rcv()
1895 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,  in tipc_crypto_rcv_complete()  argument
1900 	struct tipc_crypto *rx = aead->crypto;  in tipc_crypto_rcv_complete()
1906 	if (unlikely(is_tx(aead->crypto))) {  in tipc_crypto_rcv_complete()
1909 			 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead,  in tipc_crypto_rcv_complete()
1913 			 aead->crypto->aead[1], aead->crypto->aead[2],  in tipc_crypto_rcv_complete()
1914 			 aead->crypto->aead[3]);  in tipc_crypto_rcv_complete()
1938 		if (tipc_aead_clone(&tmp, aead) < 0)  in tipc_crypto_rcv_complete()
1945 		tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
1946 		aead = tmp;  in tipc_crypto_rcv_complete()
1950 		tipc_aead_users_dec((struct tipc_aead __force __rcu *)aead, INT_MIN);  in tipc_crypto_rcv_complete()
1955 	tipc_aead_users_set((struct tipc_aead __force __rcu *)aead, 1);  in tipc_crypto_rcv_complete()
1970 	pskb_trim(*skb, (*skb)->len - aead->authsize);  in tipc_crypto_rcv_complete()
1996 	tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
2077 	struct tipc_aead *aead;  in tipc_crypto_key_dump()  local
2103 		aead = rcu_dereference(c->aead[k]);  in tipc_crypto_key_dump()
2104 		if (aead)  in tipc_crypto_key_dump()
2107 				       aead->hint,  in tipc_crypto_key_dump()
2108 				       (aead->mode == CLUSTER_KEY) ? "c" : "p",  in tipc_crypto_key_dump()
2109 				       atomic_read(&aead->users),  in tipc_crypto_key_dump()
2110 				       refcount_read(&aead->refcnt));  in tipc_crypto_key_dump()
2197 	struct tipc_aead *aead;  in tipc_crypto_key_distr()  local
2206 		aead = tipc_aead_get(tx->aead[key]);  in tipc_crypto_key_distr()
2207 		if (likely(aead)) {  in tipc_crypto_key_distr()
2208 			rc = tipc_crypto_key_xmit(tx->net, aead->key,  in tipc_crypto_key_distr()
2209 						  aead->gen, aead->mode,  in tipc_crypto_key_distr()
2211 			tipc_aead_put(aead);  in tipc_crypto_key_distr()
2444 	struct tipc_aead *aead;  in tipc_crypto_work_tx()  local
2452 	aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]);  in tipc_crypto_work_tx()
2453 	if (unlikely(!aead)) {  in tipc_crypto_work_tx()
2460 	skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC);  in tipc_crypto_work_tx()