264 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,  in __mptcp_move_skb()  argument
268 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skb()
272 	__skb_unlink(skb, &ssk->sk_receive_queue);  in __mptcp_move_skb()
279 		if (ssk->sk_forward_alloc < skb->truesize)  in __mptcp_move_skb()
281 		__sk_mem_reclaim(ssk, skb->truesize);  in __mptcp_move_skb()
381 static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)  in mptcp_set_timeout()  argument
383 	long tout = ssk && inet_csk(ssk)->icsk_pending ?  in mptcp_set_timeout()
384 				      inet_csk(ssk)->icsk_timeout - jiffies : 0;  in mptcp_set_timeout()
441 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_check_data_fin()  local
443 			lock_sock(ssk);  in mptcp_check_data_fin()
444 			tcp_send_ack(ssk);  in mptcp_check_data_fin()
445 			release_sock(ssk);  in mptcp_check_data_fin()
459 					   struct sock *ssk,  in __mptcp_move_skbs_from_subflow()  argument
462 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in __mptcp_move_skbs_from_subflow()
470 	pr_debug("msk=%p ssk=%p", msk, ssk);  in __mptcp_move_skbs_from_subflow()
471 	tp = tcp_sk(ssk);  in __mptcp_move_skbs_from_subflow()
483 		skb = skb_peek(&ssk->sk_receive_queue);  in __mptcp_move_skbs_from_subflow()
516 			if (__mptcp_move_skb(msk, ssk, skb, offset, len))  in __mptcp_move_skbs_from_subflow()
524 			sk_eat_skb(ssk, skb);  in __mptcp_move_skbs_from_subflow()
529 		more_data_avail = mptcp_subflow_data_available(ssk);  in __mptcp_move_skbs_from_subflow()
539 		tcp_cleanup_rbuf(ssk, 1);  in __mptcp_move_skbs_from_subflow()
590 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)  in move_skbs_to_msk()  argument
603 		__mptcp_move_skbs_from_subflow(msk, ssk, &moved);  in move_skbs_to_msk()
621 void mptcp_data_ready(struct sock *sk, struct sock *ssk)  in mptcp_data_ready()  argument
623 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);  in mptcp_data_ready()
636 	    move_skbs_to_msk(msk, ssk))  in mptcp_data_ready()
651 		move_skbs_to_msk(msk, ssk);  in mptcp_data_ready()
883 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,  in mptcp_sendmsg_frag()  argument
914 	mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags);  in mptcp_sendmsg_frag()
918 	skb = tcp_write_queue_tail(ssk);  in mptcp_sendmsg_frag()
972 	ret = do_tcp_sendpages(ssk, page, offset, psize,  in mptcp_sendmsg_frag()
1008 	tail = tcp_write_queue_tail(ssk);  in mptcp_sendmsg_frag()
1015 	skb = tcp_write_queue_tail(ssk);  in mptcp_sendmsg_frag()
1021 	mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;  in mptcp_sendmsg_frag()
1034 	mptcp_subflow_ctx(ssk)->rel_write_seq += ret;  in mptcp_sendmsg_frag()
1047 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_nospace()  local
1048 		struct socket *sock = READ_ONCE(ssk->sk_socket);  in mptcp_nospace()
1058 	struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_active()  local
1065 	return ((1 << ssk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT));  in mptcp_subflow_active()
1075 	struct sock *ssk;  member
1085 	struct sock *ssk;  in mptcp_subflow_get_send()  local
1107 			ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1108 			*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);  in mptcp_subflow_get_send()
1115 		send_info[i].ssk = NULL;  in mptcp_subflow_get_send()
1119 		ssk =  mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_send()
1124 		*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);  in mptcp_subflow_get_send()
1128 		pace = READ_ONCE(ssk->sk_pacing_rate);  in mptcp_subflow_get_send()
1132 		ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32,  in mptcp_subflow_get_send()
1135 			send_info[subflow->backup].ssk = ssk;  in mptcp_subflow_get_send()
1141 		 msk, nr_active, send_info[0].ssk, send_info[0].ratio,  in mptcp_subflow_get_send()
1142 		 send_info[1].ssk, send_info[1].ratio);  in mptcp_subflow_get_send()
1146 		send_info[0].ssk = send_info[1].ssk;  in mptcp_subflow_get_send()
1148 	if (send_info[0].ssk) {  in mptcp_subflow_get_send()
1149 		msk->last_snd = send_info[0].ssk;  in mptcp_subflow_get_send()
1169 	struct sock *ssk;  in mptcp_sendmsg()  local
1197 	ssk = mptcp_subflow_get_send(msk, &sndbuf);  in mptcp_sendmsg()
1199 	       !ssk ||  in mptcp_sendmsg()
1200 	       !mptcp_page_frag_refill(ssk, pfrag)) {  in mptcp_sendmsg()
1201 		if (ssk) {  in mptcp_sendmsg()
1209 			mptcp_set_timeout(sk, ssk);  in mptcp_sendmsg()
1221 		ssk = mptcp_subflow_get_send(msk, &sndbuf);  in mptcp_sendmsg()
1233 	pr_debug("conn_list->subflow=%p", ssk);  in mptcp_sendmsg()
1235 	lock_sock(ssk);  in mptcp_sendmsg()
1238 		ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now,  in mptcp_sendmsg()
1242 				mptcp_set_timeout(sk, ssk);  in mptcp_sendmsg()
1243 				release_sock(ssk);  in mptcp_sendmsg()
1259 		if (!sk_stream_memory_free(ssk) ||  in mptcp_sendmsg()
1260 		    !mptcp_page_frag_refill(ssk, pfrag) ||  in mptcp_sendmsg()
1262 			tcp_push(ssk, msg->msg_flags, mss_now,  in mptcp_sendmsg()
1263 				 tcp_sk(ssk)->nonagle, size_goal);  in mptcp_sendmsg()
1264 			mptcp_set_timeout(sk, ssk);  in mptcp_sendmsg()
1265 			release_sock(ssk);  in mptcp_sendmsg()
1283 			tcp_push(ssk, msg->msg_flags, mss_now,  in mptcp_sendmsg()
1284 				 tcp_sk(ssk)->nonagle, size_goal);  in mptcp_sendmsg()
1292 				mptcp_set_timeout(sk, ssk);  in mptcp_sendmsg()
1293 				release_sock(ssk);  in mptcp_sendmsg()
1299 	mptcp_set_timeout(sk, ssk);  in mptcp_sendmsg()
1301 		tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle,  in mptcp_sendmsg()
1309 	release_sock(ssk);  in mptcp_sendmsg()
1448 				struct sock *ssk;  in mptcp_rcv_space_adjust()  local
1451 				ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_rcv_space_adjust()
1452 				slow = lock_sock_fast(ssk);  in mptcp_rcv_space_adjust()
1453 				WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);  in mptcp_rcv_space_adjust()
1454 				tcp_sk(ssk)->window_clamp = window_clamp;  in mptcp_rcv_space_adjust()
1455 				tcp_cleanup_rbuf(ssk, 1);  in mptcp_rcv_space_adjust()
1456 				unlock_sock_fast(ssk, slow);  in mptcp_rcv_space_adjust()
1478 		struct sock *ssk = mptcp_subflow_recv_lookup(msk);  in __mptcp_move_skbs()  local
1480 		if (!ssk)  in __mptcp_move_skbs()
1483 		lock_sock(ssk);  in __mptcp_move_skbs()
1484 		done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);  in __mptcp_move_skbs()
1485 		release_sock(ssk);  in __mptcp_move_skbs()
1645 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_subflow_get_retrans()  local
1651 		if (!tcp_write_queue_empty(ssk))  in mptcp_subflow_get_retrans()
1656 				backup = ssk;  in mptcp_subflow_get_retrans()
1660 		return ssk;  in mptcp_subflow_get_retrans()
1674 void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,  in __mptcp_close_ssk()  argument
1678 	struct socket *sock = READ_ONCE(ssk->sk_socket);  in __mptcp_close_ssk()
1687 		tcp_close(ssk, timeout);  in __mptcp_close_ssk()
1728 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in __mptcp_close_subflow()  local
1730 		if (inet_sk_state_load(ssk) != TCP_CLOSE)  in __mptcp_close_subflow()
1733 		__mptcp_close_ssk((struct sock *)msk, ssk, subflow, 0);  in __mptcp_close_subflow()
1740 	struct sock *ssk, *sk = &msk->sk.icsk_inet.sk;  in mptcp_worker()  local
1777 	ssk = mptcp_subflow_get_retrans(msk);  in mptcp_worker()
1778 	if (!ssk)  in mptcp_worker()
1781 	lock_sock(ssk);  in mptcp_worker()
1787 		int ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo,  in mptcp_worker()
1801 		tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle,  in mptcp_worker()
1808 	mptcp_set_timeout(sk, ssk);  in mptcp_worker()
1809 	release_sock(ssk);  in mptcp_worker()
1889 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)  in mptcp_subflow_shutdown()  argument
1891 	lock_sock(ssk);  in mptcp_subflow_shutdown()
1893 	switch (ssk->sk_state) {  in mptcp_subflow_shutdown()
1899 		tcp_disconnect(ssk, O_NONBLOCK);  in mptcp_subflow_shutdown()
1904 			ssk->sk_shutdown |= how;  in mptcp_subflow_shutdown()
1905 			tcp_shutdown(ssk, how);  in mptcp_subflow_shutdown()
1907 			pr_debug("Sending DATA_FIN on subflow %p", ssk);  in mptcp_subflow_shutdown()
1908 			mptcp_set_timeout(sk, ssk);  in mptcp_subflow_shutdown()
1909 			tcp_send_ack(ssk);  in mptcp_subflow_shutdown()
1914 	release_sock(ssk);  in mptcp_subflow_shutdown()
1993 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_close()  local
1994 		__mptcp_close_ssk(sk, ssk, subflow, timeout);  in mptcp_close()
2004 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)  in mptcp_copy_inaddrs()  argument
2007 	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);  in mptcp_copy_inaddrs()
2010 	msk->sk_v6_daddr = ssk->sk_v6_daddr;  in mptcp_copy_inaddrs()
2011 	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;  in mptcp_copy_inaddrs()
2019 	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;  in mptcp_copy_inaddrs()
2020 	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;  in mptcp_copy_inaddrs()
2021 	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;  in mptcp_copy_inaddrs()
2022 	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;  in mptcp_copy_inaddrs()
2023 	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;  in mptcp_copy_inaddrs()
2024 	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;  in mptcp_copy_inaddrs()
2091 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)  in mptcp_rcv_space_init()  argument
2093 	const struct tcp_sock *tp = tcp_sk(ssk);  in mptcp_rcv_space_init()
2129 		struct sock *ssk = newsk;  in mptcp_accept()  local
2151 		mptcp_copy_inaddrs(newsk, ssk);  in mptcp_accept()
2154 		mptcp_rcv_space_init(msk, ssk);  in mptcp_accept()
2247 	struct sock *ssk;  in mptcp_setsockopt()  local
2261 	ssk = __mptcp_tcp_fallback(msk);  in mptcp_setsockopt()
2263 	if (ssk)  in mptcp_setsockopt()
2264 		return tcp_setsockopt(ssk, level, optname, optval, optlen);  in mptcp_setsockopt()
2276 	struct sock *ssk;  in mptcp_getsockopt()  local
2287 	ssk = __mptcp_tcp_fallback(msk);  in mptcp_getsockopt()
2289 	if (ssk)  in mptcp_getsockopt()
2290 		return tcp_getsockopt(ssk, level, optname, optval, option);  in mptcp_getsockopt()
2316 		struct sock *ssk;  in mptcp_release_cb()  local
2318 		ssk = mptcp_subflow_recv_lookup(msk);  in mptcp_release_cb()
2319 		if (!ssk || !schedule_work(&msk->work))  in mptcp_release_cb()
2356 void mptcp_finish_connect(struct sock *ssk)  in mptcp_finish_connect()  argument
2363 	subflow = mptcp_subflow_ctx(ssk);  in mptcp_finish_connect()
2386 	mptcp_rcv_space_init(msk, ssk);  in mptcp_finish_connect()
2619 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);  in mptcp_stream_accept()  local
2621 			if (!ssk->sk_socket)  in mptcp_stream_accept()
2622 				mptcp_sock_graft(ssk, newsock);  in mptcp_stream_accept()