775 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)  in dfrag_clear()  argument
777 	int len = dfrag->data_len + dfrag->overhead;  in dfrag_clear()
779 	list_del(&dfrag->list);  in dfrag_clear()
781 	put_page(dfrag->page);  in dfrag_clear()
801 	struct mptcp_data_frag *dtmp, *dfrag;  in mptcp_clean_una()  local
812 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {  in mptcp_clean_una()
813 		if (after64(dfrag->data_seq + dfrag->data_len, snd_una))  in mptcp_clean_una()
816 		dfrag_clear(sk, dfrag);  in mptcp_clean_una()
820 	dfrag = mptcp_rtx_head(sk);  in mptcp_clean_una()
821 	if (dfrag && after64(snd_una, dfrag->data_seq)) {  in mptcp_clean_una()
822 		u64 delta = snd_una - dfrag->data_seq;  in mptcp_clean_una()
824 		if (WARN_ON_ONCE(delta > dfrag->data_len))  in mptcp_clean_una()
827 		dfrag->data_seq += delta;  in mptcp_clean_una()
828 		dfrag->offset += delta;  in mptcp_clean_una()
829 		dfrag->data_len -= delta;  in mptcp_clean_una()
871 	struct mptcp_data_frag *dfrag;  in mptcp_carve_data_frag()  local
873 	dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);  in mptcp_carve_data_frag()
874 	dfrag->data_len = 0;  in mptcp_carve_data_frag()
875 	dfrag->data_seq = msk->write_seq;  in mptcp_carve_data_frag()
876 	dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
877 	dfrag->offset = offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
878 	dfrag->page = pfrag->page;  in mptcp_carve_data_frag()
880 	return dfrag;  in mptcp_carve_data_frag()
884 			      struct msghdr *msg, struct mptcp_data_frag *dfrag,  in mptcp_sendmsg_frag()  argument
892 	bool retransmission = !!dfrag;  in mptcp_sendmsg_frag()
909 		write_seq = &dfrag->data_seq;  in mptcp_sendmsg_frag()
910 		page = dfrag->page;  in mptcp_sendmsg_frag()
940 		dfrag = mptcp_rtx_tail(sk);  in mptcp_sendmsg_frag()
942 		dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);  in mptcp_sendmsg_frag()
944 			dfrag = mptcp_carve_data_frag(msk, pfrag, offset);  in mptcp_sendmsg_frag()
945 			offset = dfrag->offset;  in mptcp_sendmsg_frag()
946 			frag_truesize = dfrag->overhead;  in mptcp_sendmsg_frag()
960 		if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) {  in mptcp_sendmsg_frag()
965 		offset = dfrag->offset;  in mptcp_sendmsg_frag()
966 		psize = min_t(size_t, dfrag->data_len, avail_size);  in mptcp_sendmsg_frag()
988 		dfrag->data_len += ret;  in mptcp_sendmsg_frag()
990 			get_page(dfrag->page);  in mptcp_sendmsg_frag()
991 			list_add_tail(&dfrag->list, &msk->rtx_queue);  in mptcp_sendmsg_frag()
1742 	struct mptcp_data_frag *dfrag;  in mptcp_worker()  local
1770 	dfrag = mptcp_rtx_head(sk);  in mptcp_worker()
1771 	if (!dfrag)  in mptcp_worker()
1783 	orig_len = dfrag->data_len;  in mptcp_worker()
1784 	orig_offset = dfrag->offset;  in mptcp_worker()
1785 	orig_write_seq = dfrag->data_seq;  in mptcp_worker()
1786 	while (dfrag->data_len > 0) {  in mptcp_worker()
1787 		int ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo,  in mptcp_worker()
1794 		dfrag->data_len -= ret;  in mptcp_worker()
1795 		dfrag->offset += ret;  in mptcp_worker()
1804 	dfrag->data_seq = orig_write_seq;  in mptcp_worker()
1805 	dfrag->offset = orig_offset;  in mptcp_worker()
1806 	dfrag->data_len = orig_len;  in mptcp_worker()
1873 	struct mptcp_data_frag *dtmp, *dfrag;  in __mptcp_clear_xmit()  local
1877 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)  in __mptcp_clear_xmit()
1878 		dfrag_clear(sk, dfrag);  in __mptcp_clear_xmit()