1038 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)  in dfrag_clear()  argument
1040 	int len = dfrag->data_len + dfrag->overhead;  in dfrag_clear()
1042 	list_del(&dfrag->list);  in dfrag_clear()
1044 	put_page(dfrag->page);  in dfrag_clear()
1050 	struct mptcp_data_frag *dtmp, *dfrag;  in __mptcp_clean_una()  local
1061 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {  in __mptcp_clean_una()
1062 		if (after64(dfrag->data_seq + dfrag->data_len, snd_una))  in __mptcp_clean_una()
1065 		if (unlikely(dfrag == msk->first_pending)) {  in __mptcp_clean_una()
1073 		dfrag_clear(sk, dfrag);  in __mptcp_clean_una()
1077 	dfrag = mptcp_rtx_head(sk);  in __mptcp_clean_una()
1078 	if (dfrag && after64(snd_una, dfrag->data_seq)) {  in __mptcp_clean_una()
1079 		u64 delta = snd_una - dfrag->data_seq;  in __mptcp_clean_una()
1082 		if (unlikely(delta > dfrag->already_sent)) {  in __mptcp_clean_una()
1085 			if (WARN_ON_ONCE(delta > dfrag->data_len))  in __mptcp_clean_una()
1087 			dfrag->already_sent += delta - dfrag->already_sent;  in __mptcp_clean_una()
1090 		dfrag->data_seq += delta;  in __mptcp_clean_una()
1091 		dfrag->offset += delta;  in __mptcp_clean_una()
1092 		dfrag->data_len -= delta;  in __mptcp_clean_una()
1093 		dfrag->already_sent -= delta;  in __mptcp_clean_una()
1166 	struct mptcp_data_frag *dfrag;  in mptcp_carve_data_frag()  local
1168 	dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);  in mptcp_carve_data_frag()
1169 	dfrag->data_len = 0;  in mptcp_carve_data_frag()
1170 	dfrag->data_seq = msk->write_seq;  in mptcp_carve_data_frag()
1171 	dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
1172 	dfrag->offset = offset + sizeof(struct mptcp_data_frag);  in mptcp_carve_data_frag()
1173 	dfrag->already_sent = 0;  in mptcp_carve_data_frag()
1174 	dfrag->page = pfrag->page;  in mptcp_carve_data_frag()
1176 	return dfrag;  in mptcp_carve_data_frag()
1283 			      struct mptcp_data_frag *dfrag,  in mptcp_sendmsg_frag()  argument
1286 	u64 data_seq = dfrag->data_seq + info->sent;  in mptcp_sendmsg_frag()
1297 		 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);  in mptcp_sendmsg_frag()
1342 			 info->limit > dfrag->data_len))  in mptcp_sendmsg_frag()
1347 			      dfrag->page, dfrag->offset + info->sent, &ret);  in mptcp_sendmsg_frag()
1515 	struct mptcp_data_frag *dfrag;  in __mptcp_push_pending()  local
1518 	while ((dfrag = mptcp_send_head(sk))) {  in __mptcp_push_pending()
1519 		info.sent = dfrag->already_sent;  in __mptcp_push_pending()
1520 		info.limit = dfrag->data_len;  in __mptcp_push_pending()
1521 		len = dfrag->data_len - dfrag->already_sent;  in __mptcp_push_pending()
1544 			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_push_pending()
1551 			dfrag->already_sent += ret;  in __mptcp_push_pending()
1579 	struct mptcp_data_frag *dfrag;  in __mptcp_subflow_push_pending()  local
1585 	while ((dfrag = mptcp_send_head(sk))) {  in __mptcp_subflow_push_pending()
1586 		info.sent = dfrag->already_sent;  in __mptcp_subflow_push_pending()
1587 		info.limit = dfrag->data_len;  in __mptcp_subflow_push_pending()
1588 		len = dfrag->data_len - dfrag->already_sent;  in __mptcp_subflow_push_pending()
1604 			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_subflow_push_pending()
1609 			dfrag->already_sent += ret;  in __mptcp_subflow_push_pending()
1675 		struct mptcp_data_frag *dfrag;  in mptcp_sendmsg()  local
1687 		dfrag = mptcp_pending_tail(sk);  in mptcp_sendmsg()
1688 		dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);  in mptcp_sendmsg()
1696 			dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset);  in mptcp_sendmsg()
1697 			frag_truesize = dfrag->overhead;  in mptcp_sendmsg()
1704 		offset = dfrag->offset + dfrag->data_len;  in mptcp_sendmsg()
1712 		if (copy_page_from_iter(dfrag->page, offset, psize,  in mptcp_sendmsg()
1721 		dfrag->data_len += psize;  in mptcp_sendmsg()
1732 			get_page(dfrag->page);  in mptcp_sendmsg()
1733 			list_add_tail(&dfrag->list, &msk->rtx_queue);  in mptcp_sendmsg()
1735 				WRITE_ONCE(msk->first_pending, dfrag);  in mptcp_sendmsg()
1738 			 dfrag->data_seq, dfrag->data_len, dfrag->already_sent,  in mptcp_sendmsg()
2350 	struct mptcp_data_frag *dfrag;  in __mptcp_retrans()  local
2356 	dfrag = mptcp_rtx_head(sk);  in __mptcp_retrans()
2357 	if (!dfrag) {  in __mptcp_retrans()
2379 	info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent;  in __mptcp_retrans()
2381 		ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);  in __mptcp_retrans()
2390 		dfrag->already_sent = max(dfrag->already_sent, info.sent);  in __mptcp_retrans()
2521 	struct mptcp_data_frag *dtmp, *dfrag;  in __mptcp_clear_xmit()  local
2524 	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)  in __mptcp_clear_xmit()
2525 		dfrag_clear(sk, dfrag);  in __mptcp_clear_xmit()