982 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)  in dfrag_clear()  argument
984 	int len = dfrag->data_len + dfrag->overhead;  in dfrag_clear()
986 	list_del(&dfrag->list);  in dfrag_clear()
988 	put_page(dfrag->page);  in dfrag_clear()
994 	struct mptcp_data_frag *dtmp, *dfrag;  in __mptcp_clean_una()  local
1004 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {  in __mptcp_clean_una()
1005 		if (after64(dfrag->data_seq + dfrag->data_len, snd_una))  in __mptcp_clean_una()
1008 		if (unlikely(dfrag == msk->first_pending)) {  in __mptcp_clean_una()
1016 		dfrag_clear(sk, dfrag);  in __mptcp_clean_una()
1019 	dfrag = mptcp_rtx_head(sk);  in __mptcp_clean_una()
1020 	if (dfrag && after64(snd_una, dfrag->data_seq)) {  in __mptcp_clean_una()
1021 		u64 delta = snd_una - dfrag->data_seq;  in __mptcp_clean_una()
1024 		if (unlikely(delta > dfrag->already_sent)) {  in __mptcp_clean_una()
1027 			if (WARN_ON_ONCE(delta > dfrag->data_len))  in __mptcp_clean_una()
1029 			dfrag->already_sent += delta - dfrag->already_sent;  in __mptcp_clean_una()
1032 		dfrag->data_seq += delta;  in __mptcp_clean_una()
1033 		dfrag->offset += delta;  in __mptcp_clean_una()
1034 		dfrag->data_len -= delta;  in __mptcp_clean_una()
1035 		dfrag->already_sent -= delta;  in __mptcp_clean_una()
1104 	struct mptcp_data_frag *dfrag;  in mptcp_carve_data_frag()  local
1106 	dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);  in mptcp_carve_data_frag()
1107 	dfrag->data_len = 0;  in mptcp_carve_data_frag()
1108 	dfrag->data_seq = msk->write_seq;  in mptcp_carve_data_frag()
1109 	dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
1110 	dfrag->offset = offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
1111 	dfrag->already_sent = 0;  in mptcp_carve_data_frag()
1112 	dfrag->page = pfrag->page;  in mptcp_carve_data_frag()
1114 	return dfrag;  in mptcp_carve_data_frag()
1228 			      struct mptcp_data_frag *dfrag,  in mptcp_sendmsg_frag()  argument
1231 	u64 data_seq = dfrag->data_seq + info->sent;  in mptcp_sendmsg_frag()
1232 	int offset = dfrag->offset + info->sent;  in mptcp_sendmsg_frag()
1243 		 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);  in mptcp_sendmsg_frag()
1246 			 info->limit > dfrag->data_len))  in mptcp_sendmsg_frag()
1271 		can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset);  in mptcp_sendmsg_frag()
1318 		get_page(dfrag->page);  in mptcp_sendmsg_frag()
1319 		skb_fill_page_desc(skb, i, dfrag->page, offset, copy);  in mptcp_sendmsg_frag()
1510 				   struct mptcp_data_frag *dfrag,  in mptcp_update_post_push()  argument
1513 	u64 snd_nxt_new = dfrag->data_seq;  in mptcp_update_post_push()
1515 	dfrag->already_sent += sent;  in mptcp_update_post_push()
1519 	snd_nxt_new += dfrag->already_sent;  in mptcp_update_post_push()
1548 	struct mptcp_data_frag *dfrag;  in __mptcp_push_pending()  local
1551 	while ((dfrag = mptcp_send_head(sk))) {  in __mptcp_push_pending()
1552 		info.sent = dfrag->already_sent;  in __mptcp_push_pending()
1553 		info.limit = dfrag->data_len;  in __mptcp_push_pending()
1554 		len = dfrag->data_len - dfrag->already_sent;  in __mptcp_push_pending()
1576 			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_push_pending()
1588 			mptcp_update_post_push(msk, dfrag, ret);  in __mptcp_push_pending()
1611 	struct mptcp_data_frag *dfrag;  in __mptcp_subflow_push_pending()  local
1617 	while ((dfrag = mptcp_send_head(sk))) {  in __mptcp_subflow_push_pending()
1618 		info.sent = dfrag->already_sent;  in __mptcp_subflow_push_pending()
1619 		info.limit = dfrag->data_len;  in __mptcp_subflow_push_pending()
1620 		len = dfrag->data_len - dfrag->already_sent;  in __mptcp_subflow_push_pending()
1637 			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_subflow_push_pending()
1646 			mptcp_update_post_push(msk, dfrag, ret);  in __mptcp_subflow_push_pending()
1753 		struct mptcp_data_frag *dfrag;  in mptcp_sendmsg()  local
1760 		dfrag = mptcp_pending_tail(sk);  in mptcp_sendmsg()
1761 		dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);  in mptcp_sendmsg()
1769 			dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset);  in mptcp_sendmsg()
1770 			frag_truesize = dfrag->overhead;  in mptcp_sendmsg()
1777 		offset = dfrag->offset + dfrag->data_len;  in mptcp_sendmsg()
1785 		if (copy_page_from_iter(dfrag->page, offset, psize,  in mptcp_sendmsg()
1794 		dfrag->data_len += psize;  in mptcp_sendmsg()
1804 			get_page(dfrag->page);  in mptcp_sendmsg()
1805 			list_add_tail(&dfrag->list, &msk->rtx_queue);  in mptcp_sendmsg()
1807 				WRITE_ONCE(msk->first_pending, dfrag);  in mptcp_sendmsg()
1810 			 dfrag->data_seq, dfrag->data_len, dfrag->already_sent,  in mptcp_sendmsg()
2504 	struct mptcp_data_frag *dfrag;  in __mptcp_retrans()  local
2513 	dfrag = mptcp_rtx_head(sk);  in __mptcp_retrans()
2514 	if (!dfrag) {  in __mptcp_retrans()
2538 	info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent;  in __mptcp_retrans()
2540 		ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_retrans()
2549 		dfrag->already_sent = max(dfrag->already_sent, info.sent);  in __mptcp_retrans()
2743 	struct mptcp_data_frag *dtmp, *dfrag;  in __mptcp_clear_xmit()  local
2746 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)  in __mptcp_clear_xmit()
2747 		dfrag_clear(sk, dfrag);  in __mptcp_clear_xmit()