121 static void kcm_rcv_ready(struct kcm_sock *kcm)  in kcm_rcv_ready()  argument
123 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()
127 	if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))  in kcm_rcv_ready()
131 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_ready()
134 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
143 		if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {  in kcm_rcv_ready()
145 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
162 	list_add_tail(&kcm->wait_rx_list,  in kcm_rcv_ready()
163 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
164 	kcm->rx_wait = true;  in kcm_rcv_ready()
170 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_rfree()  local
171 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()
180 	if (!kcm->rx_wait && !kcm->rx_psock &&  in kcm_rfree()
183 		kcm_rcv_ready(kcm);  in kcm_rfree()
221 	struct kcm_sock *kcm;  in requeue_rx_msgs()  local
233 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
236 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in requeue_rx_msgs()
238 			list_del(&kcm->wait_rx_list);  in requeue_rx_msgs()
239 			kcm->rx_wait = false;  in requeue_rx_msgs()
254 	struct kcm_sock *kcm;  in reserve_rx_kcm()  local
279 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
281 	list_del(&kcm->wait_rx_list);  in reserve_rx_kcm()
282 	kcm->rx_wait = false;  in reserve_rx_kcm()
284 	psock->rx_kcm = kcm;  in reserve_rx_kcm()
285 	kcm->rx_psock = psock;  in reserve_rx_kcm()
289 	return kcm;  in reserve_rx_kcm()
292 static void kcm_done(struct kcm_sock *kcm);
303 	struct kcm_sock *kcm = psock->rx_kcm;  in unreserve_rx_kcm()  local
306 	if (!kcm)  in unreserve_rx_kcm()
312 	kcm->rx_psock = NULL;  in unreserve_rx_kcm()
319 	if (unlikely(kcm->done)) {  in unreserve_rx_kcm()
325 		INIT_WORK(&kcm->done_work, kcm_done_work);  in unreserve_rx_kcm()
326 		schedule_work(&kcm->done_work);  in unreserve_rx_kcm()
330 	if (unlikely(kcm->rx_disabled)) {  in unreserve_rx_kcm()
331 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
332 	} else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {  in unreserve_rx_kcm()
336 		kcm_rcv_ready(kcm);  in unreserve_rx_kcm()
359 	struct kcm_sock *kcm;  in kcm_rcv_strparser()  local
362 	kcm = reserve_rx_kcm(psock, skb);  in kcm_rcv_strparser()
363 	if (!kcm) {  in kcm_rcv_strparser()
370 	if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_strparser()
412 	struct kcm_sock *kcm;  in psock_write_space()  local
424 	kcm = psock->tx_kcm;  in psock_write_space()
425 	if (kcm && !unlikely(kcm->tx_stopped))  in psock_write_space()
426 		queue_work(kcm_wq, &kcm->tx_work);  in psock_write_space()
433 static void unreserve_psock(struct kcm_sock *kcm);
436 static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)  in reserve_psock()  argument
438 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()
441 	psock = kcm->tx_psock;  in reserve_psock()
446 		WARN_ON(kcm->tx_wait);  in reserve_psock()
448 			unreserve_psock(kcm);  in reserve_psock()
450 			return kcm->tx_psock;  in reserve_psock()
458 	psock = kcm->tx_psock;  in reserve_psock()
460 		WARN_ON(kcm->tx_wait);  in reserve_psock()
462 		return kcm->tx_psock;  in reserve_psock()
470 		if (kcm->tx_wait) {  in reserve_psock()
471 			list_del(&kcm->wait_psock_list);  in reserve_psock()
472 			kcm->tx_wait = false;  in reserve_psock()
474 		kcm->tx_psock = psock;  in reserve_psock()
475 		psock->tx_kcm = kcm;  in reserve_psock()
477 	} else if (!kcm->tx_wait) {  in reserve_psock()
478 		list_add_tail(&kcm->wait_psock_list,  in reserve_psock()
480 		kcm->tx_wait = true;  in reserve_psock()
492 	struct kcm_sock *kcm;  in psock_now_avail()  local
498 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
501 		list_del(&kcm->wait_psock_list);  in psock_now_avail()
502 		kcm->tx_wait = false;  in psock_now_avail()
503 		psock->tx_kcm = kcm;  in psock_now_avail()
510 		kcm->tx_psock = psock;  in psock_now_avail()
512 		queue_work(kcm_wq, &kcm->tx_work);  in psock_now_avail()
517 static void unreserve_psock(struct kcm_sock *kcm)  in unreserve_psock()  argument
520 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()
524 	psock = kcm->tx_psock;  in unreserve_psock()
535 	WARN_ON(kcm->tx_wait);  in unreserve_psock()
537 	kcm->tx_psock = NULL;  in unreserve_psock()
563 static void kcm_report_tx_retry(struct kcm_sock *kcm)  in kcm_report_tx_retry()  argument
565 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()
575 static int kcm_write_msgs(struct kcm_sock *kcm)  in kcm_write_msgs()  argument
577 	struct sock *sk = &kcm->sk;  in kcm_write_msgs()
585 	kcm->tx_wait_more = false;  in kcm_write_msgs()
586 	psock = kcm->tx_psock;  in kcm_write_msgs()
591 		unreserve_psock(kcm);  in kcm_write_msgs()
592 		kcm_report_tx_retry(kcm);  in kcm_write_msgs()
620 	psock = reserve_psock(kcm);  in kcm_write_msgs()
673 				unreserve_psock(kcm);  in kcm_write_msgs()
676 				kcm_report_tx_retry(kcm);  in kcm_write_msgs()
712 		unreserve_psock(kcm);  in kcm_write_msgs()
723 	struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);  in kcm_tx_work()  local
724 	struct sock *sk = &kcm->sk;  in kcm_tx_work()
732 	err = kcm_write_msgs(kcm);  in kcm_tx_work()
736 		report_csk_error(&kcm->sk, -err);  in kcm_tx_work()
751 static void kcm_push(struct kcm_sock *kcm)  in kcm_push()  argument
753 	if (kcm->tx_wait_more)  in kcm_push()
754 		kcm_write_msgs(kcm);  in kcm_push()
762 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_sendpage()  local
783 	if (kcm->seq_skb) {  in kcm_sendpage()
785 		head = kcm->seq_skb;  in kcm_sendpage()
800 				kcm_push(kcm);  in kcm_sendpage()
818 			kcm_push(kcm);  in kcm_sendpage()
827 			kcm_push(kcm);  in kcm_sendpage()
859 		kcm->seq_skb = NULL;  in kcm_sendpage()
860 		KCM_STATS_INCR(kcm->stats.tx_msgs);  in kcm_sendpage()
863 			kcm->tx_wait_more = true;  in kcm_sendpage()
864 		} else if (kcm->tx_wait_more || not_busy) {  in kcm_sendpage()
865 			err = kcm_write_msgs(kcm);  in kcm_sendpage()
873 				report_csk_error(&kcm->sk, -err);  in kcm_sendpage()
878 		kcm->seq_skb = head;  in kcm_sendpage()
882 	KCM_STATS_ADD(kcm->stats.tx_bytes, size);  in kcm_sendpage()
888 	kcm_push(kcm);  in kcm_sendpage()
903 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_sendmsg()  local
919 	if (kcm->seq_skb) {  in kcm_sendmsg()
921 		head = kcm->seq_skb;  in kcm_sendmsg()
928 		kcm_push(kcm);  in kcm_sendmsg()
939 			kcm_push(kcm);  in kcm_sendmsg()
1017 		kcm_push(kcm);  in kcm_sendmsg()
1029 			kcm->seq_skb = NULL;  in kcm_sendmsg()
1030 			KCM_STATS_INCR(kcm->stats.tx_msgs);  in kcm_sendmsg()
1034 			kcm->tx_wait_more = true;  in kcm_sendmsg()
1035 		} else if (kcm->tx_wait_more || not_busy) {  in kcm_sendmsg()
1036 			err = kcm_write_msgs(kcm);  in kcm_sendmsg()
1044 				report_csk_error(&kcm->sk, -err);  in kcm_sendmsg()
1051 			kcm->seq_skb = head;  in kcm_sendmsg()
1056 	KCM_STATS_ADD(kcm->stats.tx_bytes, copied);  in kcm_sendmsg()
1062 	kcm_push(kcm);  in kcm_sendmsg()
1071 	if (head != kcm->seq_skb)  in kcm_sendmsg()
1119 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_recvmsg()  local
1147 		KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_recvmsg()
1160 			KCM_STATS_INCR(kcm->stats.rx_msgs);  in kcm_recvmsg()
1177 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_splice_read()  local
1207 	KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_splice_read()
1229 static void kcm_recv_disable(struct kcm_sock *kcm)  in kcm_recv_disable()  argument
1231 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()
1233 	if (kcm->rx_disabled)  in kcm_recv_disable()
1238 	kcm->rx_disabled = 1;  in kcm_recv_disable()
1241 	if (!kcm->rx_psock) {  in kcm_recv_disable()
1242 		if (kcm->rx_wait) {  in kcm_recv_disable()
1243 			list_del(&kcm->wait_rx_list);  in kcm_recv_disable()
1244 			kcm->rx_wait = false;  in kcm_recv_disable()
1247 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1254 static void kcm_recv_enable(struct kcm_sock *kcm)  in kcm_recv_enable()  argument
1256 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()
1258 	if (!kcm->rx_disabled)  in kcm_recv_enable()
1263 	kcm->rx_disabled = 0;  in kcm_recv_enable()
1264 	kcm_rcv_ready(kcm);  in kcm_recv_enable()
1272 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_setsockopt()  local
1289 		lock_sock(&kcm->sk);  in kcm_setsockopt()
1291 			kcm_recv_disable(kcm);  in kcm_setsockopt()
1293 			kcm_recv_enable(kcm);  in kcm_setsockopt()
1294 		release_sock(&kcm->sk);  in kcm_setsockopt()
1306 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_getsockopt()  local
1321 		val = kcm->rx_disabled;  in kcm_getsockopt()
1334 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1344 	kcm->sk.sk_state = TCP_ESTABLISHED;  in init_kcm_sock()
1347 	kcm->mux = mux;  in init_kcm_sock()
1358 	list_add(&kcm->kcm_sock_list, head);  in init_kcm_sock()
1359 	kcm->index = index;  in init_kcm_sock()
1364 	INIT_WORK(&kcm->tx_work, kcm_tx_work);  in init_kcm_sock()
1367 	kcm_rcv_ready(kcm);  in init_kcm_sock()
1374 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_attach()  local
1375 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()
1603 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_unattach_ioctl()  local
1604 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()
1780 static void kcm_done(struct kcm_sock *kcm)  in kcm_done()  argument
1782 	struct kcm_mux *mux = kcm->mux;  in kcm_done()
1783 	struct sock *sk = &kcm->sk;  in kcm_done()
1787 	if (kcm->rx_psock) {  in kcm_done()
1789 		WARN_ON(kcm->done);  in kcm_done()
1790 		kcm->rx_disabled = 1;  in kcm_done()
1791 		kcm->done = 1;  in kcm_done()
1796 	if (kcm->rx_wait) {  in kcm_done()
1797 		list_del(&kcm->wait_rx_list);  in kcm_done()
1798 		kcm->rx_wait = false;  in kcm_done()
1811 	list_del(&kcm->kcm_sock_list);  in kcm_done()
1822 	WARN_ON(kcm->rx_wait);  in kcm_done()
1824 	sock_put(&kcm->sk);  in kcm_done()
1833 	struct kcm_sock *kcm;  in kcm_release()  local
1840 	kcm = kcm_sk(sk);  in kcm_release()
1841 	mux = kcm->mux;  in kcm_release()
1844 	kfree_skb(kcm->seq_skb);  in kcm_release()
1857 	kcm->tx_stopped = 1;  in kcm_release()
1862 	if (kcm->tx_wait) {  in kcm_release()
1866 		list_del(&kcm->wait_psock_list);  in kcm_release()
1867 		kcm->tx_wait = false;  in kcm_release()
1874 	cancel_work_sync(&kcm->tx_work);  in kcm_release()
1877 	psock = kcm->tx_psock;  in kcm_release()
1884 		unreserve_psock(kcm);  in kcm_release()
1888 	WARN_ON(kcm->tx_wait);  in kcm_release()
1889 	WARN_ON(kcm->tx_psock);  in kcm_release()
1893 	kcm_done(kcm);  in kcm_release()