122 static void kcm_rcv_ready(struct kcm_sock *kcm)  in kcm_rcv_ready()  argument
124 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()
128 	if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))  in kcm_rcv_ready()
132 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_ready()
135 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
144 		if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {  in kcm_rcv_ready()
146 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
163 	list_add_tail(&kcm->wait_rx_list,  in kcm_rcv_ready()
164 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
166 	WRITE_ONCE(kcm->rx_wait, true);  in kcm_rcv_ready()
172 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_rfree()  local
173 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()
182 	if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&  in kcm_rfree()
185 		kcm_rcv_ready(kcm);  in kcm_rfree()
223 	struct kcm_sock *kcm;  in requeue_rx_msgs()  local
235 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
238 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in requeue_rx_msgs()
240 			list_del(&kcm->wait_rx_list);  in requeue_rx_msgs()
242 			WRITE_ONCE(kcm->rx_wait, false);  in requeue_rx_msgs()
257 	struct kcm_sock *kcm;  in reserve_rx_kcm()  local
282 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
284 	list_del(&kcm->wait_rx_list);  in reserve_rx_kcm()
286 	WRITE_ONCE(kcm->rx_wait, false);  in reserve_rx_kcm()
288 	psock->rx_kcm = kcm;  in reserve_rx_kcm()
290 	WRITE_ONCE(kcm->rx_psock, psock);  in reserve_rx_kcm()
294 	return kcm;  in reserve_rx_kcm()
297 static void kcm_done(struct kcm_sock *kcm);
308 	struct kcm_sock *kcm = psock->rx_kcm;  in unreserve_rx_kcm()  local
311 	if (!kcm)  in unreserve_rx_kcm()
318 	WRITE_ONCE(kcm->rx_psock, NULL);  in unreserve_rx_kcm()
325 	if (unlikely(kcm->done)) {  in unreserve_rx_kcm()
331 		INIT_WORK(&kcm->done_work, kcm_done_work);  in unreserve_rx_kcm()
332 		schedule_work(&kcm->done_work);  in unreserve_rx_kcm()
336 	if (unlikely(kcm->rx_disabled)) {  in unreserve_rx_kcm()
337 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
338 	} else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {  in unreserve_rx_kcm()
342 		kcm_rcv_ready(kcm);  in unreserve_rx_kcm()
365 	struct kcm_sock *kcm;  in kcm_rcv_strparser()  local
368 	kcm = reserve_rx_kcm(psock, skb);  in kcm_rcv_strparser()
369 	if (!kcm) {  in kcm_rcv_strparser()
376 	if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_strparser()
416 	struct kcm_sock *kcm;  in psock_write_space()  local
428 	kcm = psock->tx_kcm;  in psock_write_space()
429 	if (kcm && !unlikely(kcm->tx_stopped))  in psock_write_space()
430 		queue_work(kcm_wq, &kcm->tx_work);  in psock_write_space()
437 static void unreserve_psock(struct kcm_sock *kcm);
440 static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)  in reserve_psock()  argument
442 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()
445 	psock = kcm->tx_psock;  in reserve_psock()
450 		WARN_ON(kcm->tx_wait);  in reserve_psock()
452 			unreserve_psock(kcm);  in reserve_psock()
454 			return kcm->tx_psock;  in reserve_psock()
462 	psock = kcm->tx_psock;  in reserve_psock()
464 		WARN_ON(kcm->tx_wait);  in reserve_psock()
466 		return kcm->tx_psock;  in reserve_psock()
474 		if (kcm->tx_wait) {  in reserve_psock()
475 			list_del(&kcm->wait_psock_list);  in reserve_psock()
476 			kcm->tx_wait = false;  in reserve_psock()
478 		kcm->tx_psock = psock;  in reserve_psock()
479 		psock->tx_kcm = kcm;  in reserve_psock()
481 	} else if (!kcm->tx_wait) {  in reserve_psock()
482 		list_add_tail(&kcm->wait_psock_list,  in reserve_psock()
484 		kcm->tx_wait = true;  in reserve_psock()
496 	struct kcm_sock *kcm;  in psock_now_avail()  local
502 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
505 		list_del(&kcm->wait_psock_list);  in psock_now_avail()
506 		kcm->tx_wait = false;  in psock_now_avail()
507 		psock->tx_kcm = kcm;  in psock_now_avail()
514 		kcm->tx_psock = psock;  in psock_now_avail()
516 		queue_work(kcm_wq, &kcm->tx_work);  in psock_now_avail()
521 static void unreserve_psock(struct kcm_sock *kcm)  in unreserve_psock()  argument
524 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()
528 	psock = kcm->tx_psock;  in unreserve_psock()
539 	WARN_ON(kcm->tx_wait);  in unreserve_psock()
541 	kcm->tx_psock = NULL;  in unreserve_psock()
567 static void kcm_report_tx_retry(struct kcm_sock *kcm)  in kcm_report_tx_retry()  argument
569 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()
579 static int kcm_write_msgs(struct kcm_sock *kcm)  in kcm_write_msgs()  argument
581 	struct sock *sk = &kcm->sk;  in kcm_write_msgs()
589 	kcm->tx_wait_more = false;  in kcm_write_msgs()
590 	psock = kcm->tx_psock;  in kcm_write_msgs()
595 		unreserve_psock(kcm);  in kcm_write_msgs()
596 		kcm_report_tx_retry(kcm);  in kcm_write_msgs()
624 	psock = reserve_psock(kcm);  in kcm_write_msgs()
677 				unreserve_psock(kcm);  in kcm_write_msgs()
680 				kcm_report_tx_retry(kcm);  in kcm_write_msgs()
716 		unreserve_psock(kcm);  in kcm_write_msgs()
727 	struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);  in kcm_tx_work()  local
728 	struct sock *sk = &kcm->sk;  in kcm_tx_work()
736 	err = kcm_write_msgs(kcm);  in kcm_tx_work()
740 		report_csk_error(&kcm->sk, -err);  in kcm_tx_work()
755 static void kcm_push(struct kcm_sock *kcm)  in kcm_push()  argument
757 	if (kcm->tx_wait_more)  in kcm_push()
758 		kcm_write_msgs(kcm);  in kcm_push()
766 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_sendpage()  local
787 	if (kcm->seq_skb) {  in kcm_sendpage()
789 		head = kcm->seq_skb;  in kcm_sendpage()
804 				kcm_push(kcm);  in kcm_sendpage()
822 			kcm_push(kcm);  in kcm_sendpage()
831 			kcm_push(kcm);  in kcm_sendpage()
863 		kcm->seq_skb = NULL;  in kcm_sendpage()
864 		KCM_STATS_INCR(kcm->stats.tx_msgs);  in kcm_sendpage()
867 			kcm->tx_wait_more = true;  in kcm_sendpage()
868 		} else if (kcm->tx_wait_more || not_busy) {  in kcm_sendpage()
869 			err = kcm_write_msgs(kcm);  in kcm_sendpage()
877 				report_csk_error(&kcm->sk, -err);  in kcm_sendpage()
882 		kcm->seq_skb = head;  in kcm_sendpage()
886 	KCM_STATS_ADD(kcm->stats.tx_bytes, size);  in kcm_sendpage()
892 	kcm_push(kcm);  in kcm_sendpage()
907 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_sendmsg()  local
923 	if (kcm->seq_skb) {  in kcm_sendmsg()
925 		head = kcm->seq_skb;  in kcm_sendmsg()
932 		kcm_push(kcm);  in kcm_sendmsg()
943 			kcm_push(kcm);  in kcm_sendmsg()
1021 		kcm_push(kcm);  in kcm_sendmsg()
1033 			kcm->seq_skb = NULL;  in kcm_sendmsg()
1034 			KCM_STATS_INCR(kcm->stats.tx_msgs);  in kcm_sendmsg()
1038 			kcm->tx_wait_more = true;  in kcm_sendmsg()
1039 		} else if (kcm->tx_wait_more || not_busy) {  in kcm_sendmsg()
1040 			err = kcm_write_msgs(kcm);  in kcm_sendmsg()
1048 				report_csk_error(&kcm->sk, -err);  in kcm_sendmsg()
1055 			kcm->seq_skb = head;  in kcm_sendmsg()
1060 	KCM_STATS_ADD(kcm->stats.tx_bytes, copied);  in kcm_sendmsg()
1066 	kcm_push(kcm);  in kcm_sendmsg()
1075 	if (head != kcm->seq_skb)  in kcm_sendmsg()
1092 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_recvmsg()  local
1115 		KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_recvmsg()
1128 			KCM_STATS_INCR(kcm->stats.rx_msgs);  in kcm_recvmsg()
1142 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_splice_read()  local
1167 	KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_splice_read()
1187 static void kcm_recv_disable(struct kcm_sock *kcm)  in kcm_recv_disable()  argument
1189 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()
1191 	if (kcm->rx_disabled)  in kcm_recv_disable()
1196 	kcm->rx_disabled = 1;  in kcm_recv_disable()
1199 	if (!kcm->rx_psock) {  in kcm_recv_disable()
1200 		if (kcm->rx_wait) {  in kcm_recv_disable()
1201 			list_del(&kcm->wait_rx_list);  in kcm_recv_disable()
1203 			WRITE_ONCE(kcm->rx_wait, false);  in kcm_recv_disable()
1206 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1213 static void kcm_recv_enable(struct kcm_sock *kcm)  in kcm_recv_enable()  argument
1215 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()
1217 	if (!kcm->rx_disabled)  in kcm_recv_enable()
1222 	kcm->rx_disabled = 0;  in kcm_recv_enable()
1223 	kcm_rcv_ready(kcm);  in kcm_recv_enable()
1231 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_setsockopt()  local
1248 		lock_sock(&kcm->sk);  in kcm_setsockopt()
1250 			kcm_recv_disable(kcm);  in kcm_setsockopt()
1252 			kcm_recv_enable(kcm);  in kcm_setsockopt()
1253 		release_sock(&kcm->sk);  in kcm_setsockopt()
1265 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_getsockopt()  local
1280 		val = kcm->rx_disabled;  in kcm_getsockopt()
1293 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1303 	kcm->sk.sk_state = TCP_ESTABLISHED;  in init_kcm_sock()
1306 	kcm->mux = mux;  in init_kcm_sock()
1317 	list_add(&kcm->kcm_sock_list, head);  in init_kcm_sock()
1318 	kcm->index = index;  in init_kcm_sock()
1323 	INIT_WORK(&kcm->tx_work, kcm_tx_work);  in init_kcm_sock()
1326 	kcm_rcv_ready(kcm);  in init_kcm_sock()
1333 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_attach()  local
1334 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()
1561 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_unattach_ioctl()  local
1562 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()
1738 static void kcm_done(struct kcm_sock *kcm)  in kcm_done()  argument
1740 	struct kcm_mux *mux = kcm->mux;  in kcm_done()
1741 	struct sock *sk = &kcm->sk;  in kcm_done()
1745 	if (kcm->rx_psock) {  in kcm_done()
1747 		WARN_ON(kcm->done);  in kcm_done()
1748 		kcm->rx_disabled = 1;  in kcm_done()
1749 		kcm->done = 1;  in kcm_done()
1754 	if (kcm->rx_wait) {  in kcm_done()
1755 		list_del(&kcm->wait_rx_list);  in kcm_done()
1757 		WRITE_ONCE(kcm->rx_wait, false);  in kcm_done()
1770 	list_del(&kcm->kcm_sock_list);  in kcm_done()
1781 	WARN_ON(kcm->rx_wait);  in kcm_done()
1783 	sock_put(&kcm->sk);  in kcm_done()
1792 	struct kcm_sock *kcm;  in kcm_release()  local
1799 	kcm = kcm_sk(sk);  in kcm_release()
1800 	mux = kcm->mux;  in kcm_release()
1804 	kfree_skb(kcm->seq_skb);  in kcm_release()
1816 	kcm->tx_stopped = 1;  in kcm_release()
1821 	if (kcm->tx_wait) {  in kcm_release()
1825 		list_del(&kcm->wait_psock_list);  in kcm_release()
1826 		kcm->tx_wait = false;  in kcm_release()
1833 	cancel_work_sync(&kcm->tx_work);  in kcm_release()
1836 	psock = kcm->tx_psock;  in kcm_release()
1843 		unreserve_psock(kcm);  in kcm_release()
1847 	WARN_ON(kcm->tx_wait);  in kcm_release()
1848 	WARN_ON(kcm->tx_psock);  in kcm_release()
1852 	kcm_done(kcm);  in kcm_release()