202 	struct tipc_aead __rcu *aead[KEY_MAX + 1];  member
236 	struct tipc_aead *aead;  member
243 	struct tipc_aead *aead;  member
247 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
248 static inline void tipc_aead_put(struct tipc_aead *aead);
250 static int tipc_aead_users(struct tipc_aead __rcu *aead);
251 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
252 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
253 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
254 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
255 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
262 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
267 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
271 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
279 				  struct tipc_aead *aead, u8 pos,
292 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
377 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead)  in tipc_aead_get()  argument
382 	tmp = rcu_dereference(aead);  in tipc_aead_get()
390 static inline void tipc_aead_put(struct tipc_aead *aead)  in tipc_aead_put()  argument
392 	if (aead && refcount_dec_and_test(&aead->refcnt))  in tipc_aead_put()
393 		call_rcu(&aead->rcu, tipc_aead_free);  in tipc_aead_put()
402 	struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu);  in tipc_aead_free()  local
405 	if (aead->cloned) {  in tipc_aead_free()
406 		tipc_aead_put(aead->cloned);  in tipc_aead_free()
408 		head = *get_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
409 		put_cpu_ptr(aead->tfm_entry);  in tipc_aead_free()
420 	free_percpu(aead->tfm_entry);  in tipc_aead_free()
421 	kfree_sensitive(aead->key);  in tipc_aead_free()
422 	kfree(aead);  in tipc_aead_free()
425 static int tipc_aead_users(struct tipc_aead __rcu *aead)  in tipc_aead_users()  argument
431 	tmp = rcu_dereference(aead);  in tipc_aead_users()
439 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_inc()  argument
444 	tmp = rcu_dereference(aead);  in tipc_aead_users_inc()
450 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim)  in tipc_aead_users_dec()  argument
455 	tmp = rcu_dereference(aead);  in tipc_aead_users_dec()
457 		atomic_add_unless(&rcu_dereference(aead)->users, -1, lim);  in tipc_aead_users_dec()
461 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val)  in tipc_aead_users_set()  argument
467 	tmp = rcu_dereference(aead);  in tipc_aead_users_set()
481 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead)  in tipc_aead_tfm_next()  argument
486 	tfm_entry = get_cpu_ptr(aead->tfm_entry);  in tipc_aead_tfm_next()
507 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,  in tipc_aead_init()  argument
516 	if (unlikely(*aead))  in tipc_aead_init()
598 	*aead = tmp;  in tipc_aead_init()
617 	struct tipc_aead *aead;  in tipc_aead_clone()  local
629 	aead = kzalloc(sizeof(*aead), GFP_ATOMIC);  in tipc_aead_clone()
630 	if (unlikely(!aead))  in tipc_aead_clone()
633 	aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC);  in tipc_aead_clone()
634 	if (unlikely(!aead->tfm_entry)) {  in tipc_aead_clone()
635 		kfree_sensitive(aead);  in tipc_aead_clone()
640 		*per_cpu_ptr(aead->tfm_entry, cpu) =  in tipc_aead_clone()
644 	memcpy(aead->hint, src->hint, sizeof(src->hint));  in tipc_aead_clone()
645 	aead->mode = src->mode;  in tipc_aead_clone()
646 	aead->salt = src->salt;  in tipc_aead_clone()
647 	aead->authsize = src->authsize;  in tipc_aead_clone()
648 	atomic_set(&aead->users, 0);  in tipc_aead_clone()
649 	atomic64_set(&aead->seqno, 0);  in tipc_aead_clone()
650 	refcount_set(&aead->refcnt, 1);  in tipc_aead_clone()
653 	aead->cloned = src;  in tipc_aead_clone()
655 	*dst = aead;  in tipc_aead_clone()
721 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,  in tipc_aead_encrypt()  argument
726 	struct crypto_aead *tfm = tipc_aead_tfm_next(aead);  in tipc_aead_encrypt()
739 	tailen = len - skb->len + aead->authsize;  in tipc_aead_encrypt()
792 	salt = aead->salt;  in tipc_aead_encrypt()
793 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_encrypt()
810 	tx_ctx->aead = aead;  in tipc_aead_encrypt()
838 	struct tipc_aead *aead = tx_ctx->aead;  in tipc_aead_encrypt_done()  local
839 	struct tipc_crypto *tx = aead->crypto;  in tipc_aead_encrypt_done()
862 	tipc_aead_put(aead);  in tipc_aead_encrypt_done()
877 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,  in tipc_aead_decrypt()  argument
891 	if (unlikely(!aead))  in tipc_aead_decrypt()
907 	tfm = tipc_aead_tfm_next(aead);  in tipc_aead_decrypt()
923 	salt = aead->salt;  in tipc_aead_decrypt()
924 	if (aead->mode == CLUSTER_KEY)  in tipc_aead_decrypt()
941 	rx_ctx->aead = aead;  in tipc_aead_decrypt()
968 	struct tipc_aead *aead = rx_ctx->aead;  in tipc_aead_decrypt_done()  local
969 	struct tipc_crypto_stats __percpu *stats = aead->crypto->stats;  in tipc_aead_decrypt_done()
970 	struct net *net = aead->crypto->net;  in tipc_aead_decrypt_done()
984 	tipc_crypto_rcv_complete(net, aead, b, &skb, err);  in tipc_aead_decrypt_done()
1036 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,  in tipc_ehdr_build()  argument
1055 	if (!__rx || aead->mode == CLUSTER_KEY)  in tipc_ehdr_build()
1056 		seqno = atomic64_inc_return(&aead->seqno);  in tipc_ehdr_build()
1075 	ehdr->master_key = aead->crypto->key_master;  in tipc_ehdr_build()
1128 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_init()  local
1132 	rc = tipc_aead_init(&aead, ukey, mode);  in tipc_crypto_key_init()
1136 		rc = tipc_crypto_key_attach(c, aead, 0, master_key);  in tipc_crypto_key_init()
1138 			tipc_aead_free(&aead->rcu);  in tipc_crypto_key_init()
1154 				  struct tipc_aead *aead, u8 pos,  in tipc_crypto_key_attach()  argument
1170 		if (tipc_aead_users(c->aead[key.pending]) > 0)  in tipc_crypto_key_attach()
1192 	aead->crypto = c;  in tipc_crypto_key_attach()
1193 	aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen;  in tipc_crypto_key_attach()
1194 	tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock);  in tipc_crypto_key_attach()
1227 			tipc_aead_users_dec(tx->aead[k], 0);  in tipc_crypto_key_flush()
1236 		tipc_crypto_key_detach(c->aead[k], &c->lock);  in tipc_crypto_key_flush()
1272 	if (tipc_aead_users(rx->aead[key.pending]) > 0)  in tipc_crypto_key_try_align()
1276 	tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock);  in tipc_crypto_key_try_align()
1279 	rcu_assign_pointer(rx->aead[key.pending], NULL);  in tipc_crypto_key_try_align()
1283 		tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock));  in tipc_crypto_key_try_align()
1290 	rcu_assign_pointer(rx->aead[new_pending], tmp1);  in tipc_crypto_key_try_align()
1292 		rcu_assign_pointer(rx->aead[new_passive], tmp2);  in tipc_crypto_key_try_align()
1322 	struct tipc_aead *aead = NULL;  in tipc_crypto_key_pick_tx()  local
1339 		aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock);  in tipc_crypto_key_pick_tx()
1347 		aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock);  in tipc_crypto_key_pick_tx()
1348 		if (!aead)  in tipc_crypto_key_pick_tx()
1350 		if (aead->mode != CLUSTER_KEY ||  in tipc_crypto_key_pick_tx()
1351 		    aead == skb_cb->tx_clone_ctx.last) {  in tipc_crypto_key_pick_tx()
1352 			aead = NULL;  in tipc_crypto_key_pick_tx()
1356 		skb_cb->tx_clone_ctx.last = aead;  in tipc_crypto_key_pick_tx()
1365 	if (likely(aead))  in tipc_crypto_key_pick_tx()
1366 		WARN_ON(!refcount_inc_not_zero(&aead->refcnt));  in tipc_crypto_key_pick_tx()
1369 	return aead;  in tipc_crypto_key_pick_tx()
1432 			tipc_aead_users_inc(tx->aead[new], INT_MAX);  in tipc_crypto_key_synch()
1434 			tipc_aead_users_dec(tx->aead[cur], 0);  in tipc_crypto_key_synch()
1456 	tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_key_revoke()
1535 		tipc_aead_put(rcu_dereference(c->aead[k]));  in tipc_crypto_stop()
1556 	if (key.active && tipc_aead_users(tx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1558 	if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1565 		tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);  in tipc_crypto_timeout()
1575 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0)  in tipc_crypto_timeout()
1589 	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10)  in tipc_crypto_timeout()
1593 	tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock);  in tipc_crypto_timeout()
1602 	    tipc_aead_users(rx->aead[key.active]) > 0)  in tipc_crypto_timeout()
1611 	tipc_aead_users_set(rx->aead[key.pending], 0);  in tipc_crypto_timeout()
1620 	    tipc_aead_users(rx->aead[key.passive]) > -10)  in tipc_crypto_timeout()
1624 	tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock);  in tipc_crypto_timeout()
1693 	struct tipc_aead *aead = NULL;  in tipc_crypto_xmit()  local
1755 	aead = tipc_aead_get(tx->aead[tx_key]);  in tipc_crypto_xmit()
1756 	if (unlikely(!aead))  in tipc_crypto_xmit()
1758 	rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx);  in tipc_crypto_xmit()
1760 		rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode);  in tipc_crypto_xmit()
1783 	tipc_aead_put(aead);  in tipc_crypto_xmit()
1814 	struct tipc_aead *aead = NULL;  in tipc_crypto_rcv()  local
1839 	aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key);  in tipc_crypto_rcv()
1840 	if (aead)  in tipc_crypto_rcv()
1846 	if (!aead)  in tipc_crypto_rcv()
1847 		aead = tipc_aead_get(rx->aead[tx_key]);  in tipc_crypto_rcv()
1848 	rc = tipc_aead_decrypt(net, aead, *skb, b);  in tipc_crypto_rcv()
1874 					      rcu_access_pointer(rx->aead[n]));  in tipc_crypto_rcv()
1888 	tipc_crypto_rcv_complete(net, aead, b, skb, rc);  in tipc_crypto_rcv()
1892 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,  in tipc_crypto_rcv_complete()  argument
1897 	struct tipc_crypto *rx = aead->crypto;  in tipc_crypto_rcv_complete()
1903 	if (unlikely(is_tx(aead->crypto))) {  in tipc_crypto_rcv_complete()
1906 			 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead,  in tipc_crypto_rcv_complete()
1910 			 aead->crypto->aead[1], aead->crypto->aead[2],  in tipc_crypto_rcv_complete()
1911 			 aead->crypto->aead[3]);  in tipc_crypto_rcv_complete()
1935 		if (tipc_aead_clone(&tmp, aead) < 0)  in tipc_crypto_rcv_complete()
1941 		tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
1942 		aead = tipc_aead_get(tmp);  in tipc_crypto_rcv_complete()
1946 		tipc_aead_users_dec(aead, INT_MIN);  in tipc_crypto_rcv_complete()
1951 	tipc_aead_users_set(aead, 1);  in tipc_crypto_rcv_complete()
1966 	pskb_trim(*skb, (*skb)->len - aead->authsize);  in tipc_crypto_rcv_complete()
1992 	tipc_aead_put(aead);  in tipc_crypto_rcv_complete()
2073 	struct tipc_aead *aead;  in tipc_crypto_key_dump()  local
2099 		aead = rcu_dereference(c->aead[k]);  in tipc_crypto_key_dump()
2100 		if (aead)  in tipc_crypto_key_dump()
2103 				       aead->hint,  in tipc_crypto_key_dump()
2104 				       (aead->mode == CLUSTER_KEY) ? "c" : "p",  in tipc_crypto_key_dump()
2105 				       atomic_read(&aead->users),  in tipc_crypto_key_dump()
2106 				       refcount_read(&aead->refcnt));  in tipc_crypto_key_dump()
2193 	struct tipc_aead *aead;  in tipc_crypto_key_distr()  local
2202 		aead = tipc_aead_get(tx->aead[key]);  in tipc_crypto_key_distr()
2203 		if (likely(aead)) {  in tipc_crypto_key_distr()
2204 			rc = tipc_crypto_key_xmit(tx->net, aead->key,  in tipc_crypto_key_distr()
2205 						  aead->gen, aead->mode,  in tipc_crypto_key_distr()
2207 			tipc_aead_put(aead);  in tipc_crypto_key_distr()
2430 	struct tipc_aead *aead;  in tipc_crypto_work_tx()  local
2438 	aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]);  in tipc_crypto_work_tx()
2439 	if (unlikely(!aead)) {  in tipc_crypto_work_tx()
2446 	skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC);  in tipc_crypto_work_tx()