60 	struct kcm_mux *mux = psock->mux;  in kcm_abort_tx_psock()  local
64 	spin_lock_bh(&mux->lock);  in kcm_abort_tx_psock()
67 		spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
88 	spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
95 static void kcm_update_rx_mux_stats(struct kcm_mux *mux,  in kcm_update_rx_mux_stats()  argument
98 	STRP_STATS_ADD(mux->stats.rx_bytes,  in kcm_update_rx_mux_stats()
101 	mux->stats.rx_msgs +=  in kcm_update_rx_mux_stats()
107 static void kcm_update_tx_mux_stats(struct kcm_mux *mux,  in kcm_update_tx_mux_stats()  argument
110 	KCM_STATS_ADD(mux->stats.tx_bytes,  in kcm_update_tx_mux_stats()
112 	mux->stats.tx_msgs +=  in kcm_update_tx_mux_stats()
126 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()  local
133 	while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {  in kcm_rcv_ready()
136 			skb_queue_head(&mux->rx_hold_queue, skb);  in kcm_rcv_ready()
142 	while (!list_empty(&mux->psocks_ready)) {  in kcm_rcv_ready()
143 		psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,  in kcm_rcv_ready()
166 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
174 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()  local
185 		spin_lock_bh(&mux->rx_lock);  in kcm_rfree()
187 		spin_unlock_bh(&mux->rx_lock);  in kcm_rfree()
221 static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)  in requeue_rx_msgs()  argument
231 		if (list_empty(&mux->kcm_rx_waiters)) {  in requeue_rx_msgs()
232 			skb_queue_tail(&mux->rx_hold_queue, skb);  in requeue_rx_msgs()
236 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
256 	struct kcm_mux *mux = psock->mux;  in reserve_rx_kcm()  local
264 	spin_lock_bh(&mux->rx_lock);  in reserve_rx_kcm()
267 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
271 	kcm_update_rx_mux_stats(mux, psock);  in reserve_rx_kcm()
273 	if (list_empty(&mux->kcm_rx_waiters)) {  in reserve_rx_kcm()
277 			      &mux->psocks_ready);  in reserve_rx_kcm()
278 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
282 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
290 	spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
307 	struct kcm_mux *mux = psock->mux;  in unreserve_rx_kcm()  local
312 	spin_lock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
323 		spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
334 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
341 	spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
410 	struct kcm_mux *mux;  in psock_write_space()  local
418 	mux = psock->mux;  in psock_write_space()
420 	spin_lock_bh(&mux->lock);  in psock_write_space()
427 	spin_unlock_bh(&mux->lock);  in psock_write_space()
437 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()  local
452 	spin_lock_bh(&mux->lock);  in reserve_psock()
460 		spin_unlock_bh(&mux->lock);  in reserve_psock()
464 	if (!list_empty(&mux->psocks_avail)) {  in reserve_psock()
465 		psock = list_first_entry(&mux->psocks_avail,  in reserve_psock()
478 			      &mux->kcm_tx_waiters);  in reserve_psock()
482 	spin_unlock_bh(&mux->lock);  in reserve_psock()
490 	struct kcm_mux *mux = psock->mux;  in psock_now_avail()  local
493 	if (list_empty(&mux->kcm_tx_waiters)) {  in psock_now_avail()
495 			      &mux->psocks_avail);  in psock_now_avail()
497 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
519 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()  local
521 	spin_lock_bh(&mux->lock);  in unreserve_psock()
526 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
532 	kcm_update_tx_mux_stats(mux, psock);  in unreserve_psock()
544 			mux->psocks_cnt--;  in unreserve_psock()
552 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
559 	spin_unlock_bh(&mux->lock);  in unreserve_psock()
564 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()  local
566 	spin_lock_bh(&mux->lock);  in kcm_report_tx_retry()
567 	KCM_STATS_INCR(mux->stats.tx_retries);  in kcm_report_tx_retry()
568 	spin_unlock_bh(&mux->lock);  in kcm_report_tx_retry()
1230 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()  local
1235 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_disable()
1246 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1249 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_disable()
1255 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()  local
1260 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_enable()
1265 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_enable()
1333 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1346 	kcm->mux = mux;  in init_kcm_sock()
1347 	spin_lock_bh(&mux->lock);  in init_kcm_sock()
1349 	head = &mux->kcm_socks;  in init_kcm_sock()
1350 	list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {  in init_kcm_sock()
1360 	mux->kcm_socks_cnt++;  in init_kcm_sock()
1361 	spin_unlock_bh(&mux->lock);  in init_kcm_sock()
1365 	spin_lock_bh(&mux->rx_lock);  in init_kcm_sock()
1367 	spin_unlock_bh(&mux->rx_lock);  in init_kcm_sock()
1374 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()  local
1411 	psock->mux = mux;  in kcm_attach()
1448 	spin_lock_bh(&mux->lock);  in kcm_attach()
1449 	head = &mux->psocks;  in kcm_attach()
1450 	list_for_each_entry(tpsock, &mux->psocks, psock_list) {  in kcm_attach()
1460 	KCM_STATS_INCR(mux->stats.psock_attach);  in kcm_attach()
1461 	mux->psocks_cnt++;  in kcm_attach()
1463 	spin_unlock_bh(&mux->lock);  in kcm_attach()
1507 	struct kcm_mux *mux = psock->mux;  in kcm_unattach()  local
1527 	spin_lock_bh(&mux->rx_lock);  in kcm_unattach()
1536 		KCM_STATS_INCR(mux->stats.rx_ready_drops);  in kcm_unattach()
1539 	spin_unlock_bh(&mux->rx_lock);  in kcm_unattach()
1550 	spin_lock_bh(&mux->lock);  in kcm_unattach()
1552 	aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);  in kcm_unattach()
1553 	save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);  in kcm_unattach()
1555 	KCM_STATS_INCR(mux->stats.psock_unattach);  in kcm_unattach()
1562 		KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);  in kcm_unattach()
1563 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1571 		spin_lock_bh(&mux->lock);  in kcm_unattach()
1583 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1589 		mux->psocks_cnt--;  in kcm_unattach()
1590 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1603 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()  local
1621 	spin_lock_bh(&mux->lock);  in kcm_unattach_ioctl()
1623 	list_for_each_entry(psock, &mux->psocks, psock_list) {  in kcm_unattach_ioctl()
1636 		spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1645 	spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1680 	init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);  in kcm_clone()
1743 	struct kcm_mux *mux = container_of(rcu,  in free_mux()  local
1746 	kmem_cache_free(kcm_muxp, mux);  in free_mux()
1749 static void release_mux(struct kcm_mux *mux)  in release_mux()  argument
1751 	struct kcm_net *knet = mux->knet;  in release_mux()
1756 				 &mux->psocks, psock_list) {  in release_mux()
1761 	if (WARN_ON(mux->psocks_cnt))  in release_mux()
1764 	__skb_queue_purge(&mux->rx_hold_queue);  in release_mux()
1767 	aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);  in release_mux()
1768 	aggregate_psock_stats(&mux->aggregate_psock_stats,  in release_mux()
1770 	aggregate_strp_stats(&mux->aggregate_strp_stats,  in release_mux()
1772 	list_del_rcu(&mux->kcm_mux_list);  in release_mux()
1776 	call_rcu(&mux->rcu, free_mux);  in release_mux()
1781 	struct kcm_mux *mux = kcm->mux;  in kcm_done()  local
1785 	spin_lock_bh(&mux->rx_lock);  in kcm_done()
1791 		spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1800 	requeue_rx_msgs(mux, &sk->sk_receive_queue);  in kcm_done()
1802 	spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1808 	spin_lock_bh(&mux->lock);  in kcm_done()
1811 	mux->kcm_socks_cnt--;  in kcm_done()
1812 	socks_cnt = mux->kcm_socks_cnt;  in kcm_done()
1814 	spin_unlock_bh(&mux->lock);  in kcm_done()
1818 		release_mux(mux);  in kcm_done()
1833 	struct kcm_mux *mux;  in kcm_release()  local
1840 	mux = kcm->mux;  in kcm_release()
1860 	spin_lock_bh(&mux->lock);  in kcm_release()
1868 	spin_unlock_bh(&mux->lock);  in kcm_release()
1946 	struct kcm_mux *mux;  in kcm_create()  local
1967 	mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);  in kcm_create()
1968 	if (!mux) {  in kcm_create()
1973 	spin_lock_init(&mux->lock);  in kcm_create()
1974 	spin_lock_init(&mux->rx_lock);  in kcm_create()
1975 	INIT_LIST_HEAD(&mux->kcm_socks);  in kcm_create()
1976 	INIT_LIST_HEAD(&mux->kcm_rx_waiters);  in kcm_create()
1977 	INIT_LIST_HEAD(&mux->kcm_tx_waiters);  in kcm_create()
1979 	INIT_LIST_HEAD(&mux->psocks);  in kcm_create()
1980 	INIT_LIST_HEAD(&mux->psocks_ready);  in kcm_create()
1981 	INIT_LIST_HEAD(&mux->psocks_avail);  in kcm_create()
1983 	mux->knet = knet;  in kcm_create()
1987 	list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);  in kcm_create()
1991 	skb_queue_head_init(&mux->rx_hold_queue);  in kcm_create()
1995 	init_kcm_sock(kcm_sk(sk), mux);  in kcm_create()