123 static void kcm_rcv_ready(struct kcm_sock *kcm)  in kcm_rcv_ready()  argument
125 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()
129 	if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))  in kcm_rcv_ready()
133 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_ready()
136 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
145 		if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {  in kcm_rcv_ready()
147 			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));  in kcm_rcv_ready()
164 	list_add_tail(&kcm->wait_rx_list,  in kcm_rcv_ready()
165 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
167 	WRITE_ONCE(kcm->rx_wait, true);  in kcm_rcv_ready()
173 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_rfree()  local
174 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()
183 	if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&  in kcm_rfree()
186 		kcm_rcv_ready(kcm);  in kcm_rfree()
224 	struct kcm_sock *kcm;  in requeue_rx_msgs()  local
236 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
239 		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in requeue_rx_msgs()
241 			list_del(&kcm->wait_rx_list);  in requeue_rx_msgs()
243 			WRITE_ONCE(kcm->rx_wait, false);  in requeue_rx_msgs()
258 	struct kcm_sock *kcm;  in reserve_rx_kcm()  local
283 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
285 	list_del(&kcm->wait_rx_list);  in reserve_rx_kcm()
287 	WRITE_ONCE(kcm->rx_wait, false);  in reserve_rx_kcm()
289 	psock->rx_kcm = kcm;  in reserve_rx_kcm()
291 	WRITE_ONCE(kcm->rx_psock, psock);  in reserve_rx_kcm()
295 	return kcm;  in reserve_rx_kcm()
298 static void kcm_done(struct kcm_sock *kcm);
309 	struct kcm_sock *kcm = psock->rx_kcm;  in unreserve_rx_kcm()  local
312 	if (!kcm)  in unreserve_rx_kcm()
319 	WRITE_ONCE(kcm->rx_psock, NULL);  in unreserve_rx_kcm()
326 	if (unlikely(kcm->done)) {  in unreserve_rx_kcm()
332 		INIT_WORK(&kcm->done_work, kcm_done_work);  in unreserve_rx_kcm()
333 		schedule_work(&kcm->done_work);  in unreserve_rx_kcm()
337 	if (unlikely(kcm->rx_disabled)) {  in unreserve_rx_kcm()
338 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
339 	} else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {  in unreserve_rx_kcm()
343 		kcm_rcv_ready(kcm);  in unreserve_rx_kcm()
368 	struct kcm_sock *kcm;  in kcm_rcv_strparser()  local
371 	kcm = reserve_rx_kcm(psock, skb);  in kcm_rcv_strparser()
372 	if (!kcm) {  in kcm_rcv_strparser()
379 	if (kcm_queue_rcv_skb(&kcm->sk, skb)) {  in kcm_rcv_strparser()
419 	struct kcm_sock *kcm;  in psock_write_space()  local
431 	kcm = psock->tx_kcm;  in psock_write_space()
432 	if (kcm && !unlikely(kcm->tx_stopped))  in psock_write_space()
433 		queue_work(kcm_wq, &kcm->tx_work);  in psock_write_space()
440 static void unreserve_psock(struct kcm_sock *kcm);
443 static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)  in reserve_psock()  argument
445 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()
448 	psock = kcm->tx_psock;  in reserve_psock()
453 		WARN_ON(kcm->tx_wait);  in reserve_psock()
455 			unreserve_psock(kcm);  in reserve_psock()
457 			return kcm->tx_psock;  in reserve_psock()
465 	psock = kcm->tx_psock;  in reserve_psock()
467 		WARN_ON(kcm->tx_wait);  in reserve_psock()
469 		return kcm->tx_psock;  in reserve_psock()
477 		if (kcm->tx_wait) {  in reserve_psock()
478 			list_del(&kcm->wait_psock_list);  in reserve_psock()
479 			kcm->tx_wait = false;  in reserve_psock()
481 		kcm->tx_psock = psock;  in reserve_psock()
482 		psock->tx_kcm = kcm;  in reserve_psock()
484 	} else if (!kcm->tx_wait) {  in reserve_psock()
485 		list_add_tail(&kcm->wait_psock_list,  in reserve_psock()
487 		kcm->tx_wait = true;  in reserve_psock()
499 	struct kcm_sock *kcm;  in psock_now_avail()  local
505 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
508 		list_del(&kcm->wait_psock_list);  in psock_now_avail()
509 		kcm->tx_wait = false;  in psock_now_avail()
510 		psock->tx_kcm = kcm;  in psock_now_avail()
517 		kcm->tx_psock = psock;  in psock_now_avail()
519 		queue_work(kcm_wq, &kcm->tx_work);  in psock_now_avail()
524 static void unreserve_psock(struct kcm_sock *kcm)  in unreserve_psock()  argument
527 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()
531 	psock = kcm->tx_psock;  in unreserve_psock()
542 	WARN_ON(kcm->tx_wait);  in unreserve_psock()
544 	kcm->tx_psock = NULL;  in unreserve_psock()
570 static void kcm_report_tx_retry(struct kcm_sock *kcm)  in kcm_report_tx_retry()  argument
572 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()
582 static int kcm_write_msgs(struct kcm_sock *kcm)  in kcm_write_msgs()  argument
585 	struct sock *sk = &kcm->sk;  in kcm_write_msgs()
590 	kcm->tx_wait_more = false;  in kcm_write_msgs()
591 	psock = kcm->tx_psock;  in kcm_write_msgs()
596 		unreserve_psock(kcm);  in kcm_write_msgs()
597 		kcm_report_tx_retry(kcm);  in kcm_write_msgs()
615 			psock = reserve_psock(kcm);  in kcm_write_msgs()
663 				unreserve_psock(kcm);  in kcm_write_msgs()
667 				kcm_report_tx_retry(kcm);  in kcm_write_msgs()
701 			unreserve_psock(kcm);  in kcm_write_msgs()
712 	struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);  in kcm_tx_work()  local
713 	struct sock *sk = &kcm->sk;  in kcm_tx_work()
721 	err = kcm_write_msgs(kcm);  in kcm_tx_work()
725 		report_csk_error(&kcm->sk, -err);  in kcm_tx_work()
740 static void kcm_push(struct kcm_sock *kcm)  in kcm_push()  argument
742 	if (kcm->tx_wait_more)  in kcm_push()
743 		kcm_write_msgs(kcm);  in kcm_push()
749 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_sendmsg()  local
765 	if (kcm->seq_skb) {  in kcm_sendmsg()
767 		head = kcm->seq_skb;  in kcm_sendmsg()
774 		kcm_push(kcm);  in kcm_sendmsg()
785 			kcm_push(kcm);  in kcm_sendmsg()
886 		kcm_push(kcm);  in kcm_sendmsg()
898 			kcm->seq_skb = NULL;  in kcm_sendmsg()
899 			KCM_STATS_INCR(kcm->stats.tx_msgs);  in kcm_sendmsg()
903 			kcm->tx_wait_more = true;  in kcm_sendmsg()
904 		} else if (kcm->tx_wait_more || not_busy) {  in kcm_sendmsg()
905 			err = kcm_write_msgs(kcm);  in kcm_sendmsg()
913 				report_csk_error(&kcm->sk, -err);  in kcm_sendmsg()
920 			kcm->seq_skb = head;  in kcm_sendmsg()
925 	KCM_STATS_ADD(kcm->stats.tx_bytes, copied);  in kcm_sendmsg()
931 	kcm_push(kcm);  in kcm_sendmsg()
939 		if (head != kcm->seq_skb)  in kcm_sendmsg()
943 		kcm->seq_skb = NULL;  in kcm_sendmsg()
959 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_splice_eof()  local
965 	kcm_write_msgs(kcm);  in kcm_splice_eof()
973 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_recvmsg()  local
996 		KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_recvmsg()
1009 			KCM_STATS_INCR(kcm->stats.rx_msgs);  in kcm_recvmsg()
1023 	struct kcm_sock *kcm = kcm_sk(sk);  in kcm_splice_read()  local
1048 	KCM_STATS_ADD(kcm->stats.rx_bytes, copied);  in kcm_splice_read()
1068 static void kcm_recv_disable(struct kcm_sock *kcm)  in kcm_recv_disable()  argument
1070 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()
1072 	if (kcm->rx_disabled)  in kcm_recv_disable()
1077 	kcm->rx_disabled = 1;  in kcm_recv_disable()
1080 	if (!kcm->rx_psock) {  in kcm_recv_disable()
1081 		if (kcm->rx_wait) {  in kcm_recv_disable()
1082 			list_del(&kcm->wait_rx_list);  in kcm_recv_disable()
1084 			WRITE_ONCE(kcm->rx_wait, false);  in kcm_recv_disable()
1087 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1094 static void kcm_recv_enable(struct kcm_sock *kcm)  in kcm_recv_enable()  argument
1096 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()
1098 	if (!kcm->rx_disabled)  in kcm_recv_enable()
1103 	kcm->rx_disabled = 0;  in kcm_recv_enable()
1104 	kcm_rcv_ready(kcm);  in kcm_recv_enable()
1112 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_setsockopt()  local
1129 		lock_sock(&kcm->sk);  in kcm_setsockopt()
1131 			kcm_recv_disable(kcm);  in kcm_setsockopt()
1133 			kcm_recv_enable(kcm);  in kcm_setsockopt()
1134 		release_sock(&kcm->sk);  in kcm_setsockopt()
1146 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_getsockopt()  local
1161 		val = kcm->rx_disabled;  in kcm_getsockopt()
1174 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1184 	kcm->sk.sk_state = TCP_ESTABLISHED;  in init_kcm_sock()
1187 	kcm->mux = mux;  in init_kcm_sock()
1198 	list_add(&kcm->kcm_sock_list, head);  in init_kcm_sock()
1199 	kcm->index = index;  in init_kcm_sock()
1204 	INIT_WORK(&kcm->tx_work, kcm_tx_work);  in init_kcm_sock()
1207 	kcm_rcv_ready(kcm);  in init_kcm_sock()
1214 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_attach()  local
1215 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()
1442 	struct kcm_sock *kcm = kcm_sk(sock->sk);  in kcm_unattach_ioctl()  local
1443 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()
1619 static void kcm_done(struct kcm_sock *kcm)  in kcm_done()  argument
1621 	struct kcm_mux *mux = kcm->mux;  in kcm_done()
1622 	struct sock *sk = &kcm->sk;  in kcm_done()
1626 	if (kcm->rx_psock) {  in kcm_done()
1628 		WARN_ON(kcm->done);  in kcm_done()
1629 		kcm->rx_disabled = 1;  in kcm_done()
1630 		kcm->done = 1;  in kcm_done()
1635 	if (kcm->rx_wait) {  in kcm_done()
1636 		list_del(&kcm->wait_rx_list);  in kcm_done()
1638 		WRITE_ONCE(kcm->rx_wait, false);  in kcm_done()
1651 	list_del(&kcm->kcm_sock_list);  in kcm_done()
1662 	WARN_ON(kcm->rx_wait);  in kcm_done()
1664 	sock_put(&kcm->sk);  in kcm_done()
1673 	struct kcm_sock *kcm;  in kcm_release()  local
1680 	kcm = kcm_sk(sk);  in kcm_release()
1681 	mux = kcm->mux;  in kcm_release()
1685 	kfree_skb(kcm->seq_skb);  in kcm_release()
1697 	kcm->tx_stopped = 1;  in kcm_release()
1702 	if (kcm->tx_wait) {  in kcm_release()
1706 		list_del(&kcm->wait_psock_list);  in kcm_release()
1707 		kcm->tx_wait = false;  in kcm_release()
1714 	cancel_work_sync(&kcm->tx_work);  in kcm_release()
1717 	psock = kcm->tx_psock;  in kcm_release()
1724 		unreserve_psock(kcm);  in kcm_release()
1728 	WARN_ON(kcm->tx_wait);  in kcm_release()
1729 	WARN_ON(kcm->tx_psock);  in kcm_release()
1733 	kcm_done(kcm);  in kcm_release()