Lines Matching refs:msk
34 struct mptcp_sock msk; member
65 struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) in __mptcp_nmpc_socket() argument
67 if (!msk->subflow || READ_ONCE(msk->can_ack)) in __mptcp_nmpc_socket()
70 return msk->subflow; in __mptcp_nmpc_socket()
74 static u64 mptcp_wnd_end(const struct mptcp_sock *msk) in mptcp_wnd_end() argument
76 return READ_ONCE(msk->wnd_end); in mptcp_wnd_end()
103 static int __mptcp_socket_create(struct mptcp_sock *msk) in __mptcp_socket_create() argument
106 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create()
114 msk->first = ssock->sk; in __mptcp_socket_create()
115 msk->subflow = ssock; in __mptcp_socket_create()
117 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
123 mptcp_sock_graft(msk->first, sk->sk_socket); in __mptcp_socket_create()
165 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, in mptcp_ooo_try_coalesce() argument
171 return mptcp_try_coalesce((struct sock *)msk, to, from); in mptcp_ooo_try_coalesce()
183 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_rmem_uncharge() local
186 msk->rmem_fwd_alloc += size; in mptcp_rmem_uncharge()
187 reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk); in mptcp_rmem_uncharge()
216 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) in mptcp_data_queue_ofo() argument
218 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo()
225 max_seq = atomic64_read(&msk->rcv_wnd_sent); in mptcp_data_queue_ofo()
227 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, in mptcp_data_queue_ofo()
228 RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_data_queue_ofo()
234 (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); in mptcp_data_queue_ofo()
239 p = &msk->out_of_order_queue.rb_node; in mptcp_data_queue_ofo()
241 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { in mptcp_data_queue_ofo()
243 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
244 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
251 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { in mptcp_data_queue_ofo()
258 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { in mptcp_data_queue_ofo()
260 parent = &msk->ooo_last_skb->rbnode; in mptcp_data_queue_ofo()
292 &msk->out_of_order_queue); in mptcp_data_queue_ofo()
297 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { in mptcp_data_queue_ofo()
307 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
314 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
320 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
329 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_rmem_schedule() local
332 if (size <= msk->rmem_fwd_alloc) in mptcp_rmem_schedule()
335 size -= msk->rmem_fwd_alloc; in mptcp_rmem_schedule()
341 msk->rmem_fwd_alloc += amount; in mptcp_rmem_schedule()
345 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
350 struct sock *sk = (struct sock *)msk; in __mptcp_move_skb()
374 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { in __mptcp_move_skb()
376 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); in __mptcp_move_skb()
384 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { in __mptcp_move_skb()
385 mptcp_data_queue_ofo(msk, skb); in __mptcp_move_skb()
421 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin_ack() local
423 return !__mptcp_check_fallback(msk) && in mptcp_pending_data_fin_ack()
426 msk->write_seq == READ_ONCE(msk->snd_una); in mptcp_pending_data_fin_ack()
431 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack() local
435 WRITE_ONCE(msk->snd_data_fin_enable, 0); in mptcp_check_data_fin_ack()
453 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin() local
455 if (READ_ONCE(msk->rcv_data_fin) && in mptcp_pending_data_fin()
458 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); in mptcp_pending_data_fin()
460 if (msk->ack_seq == rcv_data_fin_seq) { in mptcp_pending_data_fin()
526 static void mptcp_send_ack(struct mptcp_sock *msk) in mptcp_send_ack() argument
530 mptcp_for_each_subflow(msk, subflow) in mptcp_send_ack()
557 static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) in mptcp_cleanup_rbuf() argument
559 int old_space = READ_ONCE(msk->old_wspace); in mptcp_cleanup_rbuf()
561 struct sock *sk = (struct sock *)msk; in mptcp_cleanup_rbuf()
568 mptcp_for_each_subflow(msk, subflow) { in mptcp_cleanup_rbuf()
578 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin() local
582 if (__mptcp_check_fallback(msk)) in mptcp_check_data_fin()
599 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); in mptcp_check_data_fin()
600 WRITE_ONCE(msk->rcv_data_fin, 0); in mptcp_check_data_fin()
622 mptcp_send_ack(msk); in mptcp_check_data_fin()
628 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, in __mptcp_move_skbs_from_subflow() argument
633 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow()
651 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
674 if (__mptcp_check_fallback(msk)) { in __mptcp_move_skbs_from_subflow()
696 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
721 static bool __mptcp_ofo_queue(struct mptcp_sock *msk) in __mptcp_ofo_queue() argument
723 struct sock *sk = (struct sock *)msk; in __mptcp_ofo_queue()
729 p = rb_first(&msk->out_of_order_queue); in __mptcp_ofo_queue()
730 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); in __mptcp_ofo_queue()
733 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) in __mptcp_ofo_queue()
737 rb_erase(&skb->rbnode, &msk->out_of_order_queue); in __mptcp_ofo_queue()
740 msk->ack_seq))) { in __mptcp_ofo_queue()
748 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { in __mptcp_ofo_queue()
749 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in __mptcp_ofo_queue()
753 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, in __mptcp_ofo_queue()
759 msk->ack_seq = end_seq; in __mptcp_ofo_queue()
768 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
770 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk()
773 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
774 __mptcp_ofo_queue(msk); in move_skbs_to_msk()
779 __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); in move_skbs_to_msk()
795 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready() local
818 if (move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
824 static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) in __mptcp_finish_join() argument
826 struct sock *sk = (struct sock *)msk; in __mptcp_finish_join()
837 mptcp_propagate_sndbuf((struct sock *)msk, ssk); in __mptcp_finish_join()
838 mptcp_sockopt_sync_locked(msk, ssk); in __mptcp_finish_join()
845 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_flush_join_list() local
847 list_for_each_entry_safe(subflow, tmp, &msk->join_list, node) { in __mptcp_flush_join_list()
851 list_move_tail(&subflow->node, &msk->conn_list); in __mptcp_flush_join_list()
852 if (!__mptcp_finish_join(msk, ssk)) in __mptcp_flush_join_list()
895 static void mptcp_check_for_eof(struct mptcp_sock *msk) in mptcp_check_for_eof() argument
898 struct sock *sk = (struct sock *)msk; in mptcp_check_for_eof()
901 mptcp_for_each_subflow(msk, subflow) in mptcp_check_for_eof()
932 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) in mptcp_subflow_recv_lookup() argument
935 struct sock *sk = (struct sock *)msk; in mptcp_subflow_recv_lookup()
939 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
966 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, in mptcp_frag_can_collapse_to() argument
973 df->data_seq + df->data_len == msk->write_seq; in mptcp_frag_can_collapse_to()
993 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clean_una() local
1000 if (__mptcp_check_fallback(msk)) in __mptcp_clean_una()
1001 msk->snd_una = READ_ONCE(msk->snd_nxt); in __mptcp_clean_una()
1003 snd_una = msk->snd_una; in __mptcp_clean_una()
1004 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { in __mptcp_clean_una()
1008 if (unlikely(dfrag == msk->first_pending)) { in __mptcp_clean_una()
1010 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1013 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_clean_una()
1025 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1041 if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) in __mptcp_clean_una()
1042 msk->recovery = false; in __mptcp_clean_una()
1045 if (snd_una == READ_ONCE(msk->snd_nxt) && in __mptcp_clean_una()
1046 snd_una == READ_ONCE(msk->write_seq)) { in __mptcp_clean_una()
1047 if (mptcp_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) in __mptcp_clean_una()
1072 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_enter_memory_pressure() local
1076 mptcp_for_each_subflow(msk, subflow) { in mptcp_enter_memory_pressure()
1100 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, in mptcp_carve_data_frag() argument
1108 dfrag->data_seq = msk->write_seq; in mptcp_carve_data_frag()
1126 static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, in mptcp_check_allowed_size() argument
1129 u64 window_end = mptcp_wnd_end(msk); in mptcp_check_allowed_size()
1132 if (__mptcp_check_fallback(msk)) in mptcp_check_allowed_size()
1211 static void mptcp_update_infinite_map(struct mptcp_sock *msk, in mptcp_update_infinite_map() argument
1223 pr_fallback(msk); in mptcp_update_infinite_map()
1233 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag() local
1243 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); in mptcp_sendmsg_frag()
1290 copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); in mptcp_sendmsg_frag()
1292 u64 snd_una = READ_ONCE(msk->snd_una); in mptcp_sendmsg_frag()
1294 if (snd_una != msk->snd_nxt) { in mptcp_sendmsg_frag()
1353 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1359 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1362 mptcp_update_infinite_map(msk, ssk, mpext); in mptcp_sendmsg_frag()
1409 static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) in mptcp_subflow_get_send() argument
1413 struct sock *sk = (struct sock *)msk; in mptcp_subflow_get_send()
1422 if (__mptcp_check_fallback(msk)) { in mptcp_subflow_get_send()
1423 if (!msk->first) in mptcp_subflow_get_send()
1425 return __tcp_can_send(msk->first) && in mptcp_subflow_get_send()
1426 sk_stream_memory_free(msk->first) ? msk->first : NULL; in mptcp_subflow_get_send()
1430 if (msk->last_snd && msk->snd_burst > 0 && in mptcp_subflow_get_send()
1431 sk_stream_memory_free(msk->last_snd) && in mptcp_subflow_get_send()
1432 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) { in mptcp_subflow_get_send()
1434 return msk->last_snd; in mptcp_subflow_get_send()
1443 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1487 burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); in mptcp_subflow_get_send()
1490 msk->last_snd = NULL; in mptcp_subflow_get_send()
1498 msk->last_snd = ssk; in mptcp_subflow_get_send()
1499 msk->snd_burst = burst; in mptcp_subflow_get_send()
1509 static void mptcp_update_post_push(struct mptcp_sock *msk, in mptcp_update_post_push() argument
1517 msk->snd_burst -= sent; in mptcp_update_post_push()
1530 if (likely(after64(snd_nxt_new, msk->snd_nxt))) in mptcp_update_post_push()
1531 msk->snd_nxt = snd_nxt_new; in mptcp_update_post_push()
1543 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_push_pending() local
1559 ssk = mptcp_subflow_get_send(msk); in __mptcp_push_pending()
1588 mptcp_update_post_push(msk, dfrag, ret); in __mptcp_push_pending()
1590 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_push_pending()
1607 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_subflow_push_pending() local
1646 mptcp_update_post_push(msk, dfrag, ret); in __mptcp_subflow_push_pending()
1648 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_subflow_push_pending()
1661 if (msk->snd_data_fin_enable && in __mptcp_subflow_push_pending()
1662 msk->snd_nxt + 1 == msk->write_seq) in __mptcp_subflow_push_pending()
1680 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_fastopen() local
1685 msk->connect_flags = O_NONBLOCK; in mptcp_sendmsg_fastopen()
1686 msk->is_sendmsg = 1; in mptcp_sendmsg_fastopen()
1688 msk->is_sendmsg = 0; in mptcp_sendmsg_fastopen()
1709 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg() local
1725 ssock = __mptcp_nmpc_socket(msk); in mptcp_sendmsg()
1761 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); in mptcp_sendmsg()
1769 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); in mptcp_sendmsg()
1797 WRITE_ONCE(msk->write_seq, msk->write_seq + psize); in mptcp_sendmsg()
1805 list_add_tail(&dfrag->list, &msk->rtx_queue); in mptcp_sendmsg()
1806 if (!msk->first_pending) in mptcp_sendmsg()
1807 WRITE_ONCE(msk->first_pending, dfrag); in mptcp_sendmsg()
1809 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, in mptcp_sendmsg()
1838 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, in __mptcp_recvmsg_mskq() argument
1847 skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { in __mptcp_recvmsg_mskq()
1880 WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); in __mptcp_recvmsg_mskq()
1881 __skb_unlink(skb, &msk->receive_queue); in __mptcp_recvmsg_mskq()
1896 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) in mptcp_rcv_space_adjust() argument
1899 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust()
1908 msk->rcvq_space.copied += copied; in mptcp_rcv_space_adjust()
1911 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); in mptcp_rcv_space_adjust()
1913 rtt_us = msk->rcvq_space.rtt_us; in mptcp_rcv_space_adjust()
1918 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1932 msk->rcvq_space.rtt_us = rtt_us; in mptcp_rcv_space_adjust()
1936 if (msk->rcvq_space.copied <= msk->rcvq_space.space) in mptcp_rcv_space_adjust()
1944 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; in mptcp_rcv_space_adjust()
1946 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1948 do_div(grow, msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1970 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1984 msk->rcvq_space.space = msk->rcvq_space.copied; in mptcp_rcv_space_adjust()
1986 msk->rcvq_space.copied = 0; in mptcp_rcv_space_adjust()
1987 msk->rcvq_space.time = mstamp; in mptcp_rcv_space_adjust()
1992 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_update_rmem() local
1994 if (!msk->rmem_released) in __mptcp_update_rmem()
1997 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); in __mptcp_update_rmem()
1998 mptcp_rmem_uncharge(sk, msk->rmem_released); in __mptcp_update_rmem()
1999 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_update_rmem()
2004 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_splice_receive_queue() local
2006 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); in __mptcp_splice_receive_queue()
2009 static bool __mptcp_move_skbs(struct mptcp_sock *msk) in __mptcp_move_skbs() argument
2011 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs()
2016 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs()
2029 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
2039 if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || in __mptcp_move_skbs()
2043 ret |= __mptcp_ofo_queue(msk); in __mptcp_move_skbs()
2048 mptcp_check_data_fin((struct sock *)msk); in __mptcp_move_skbs()
2049 return !skb_queue_empty(&msk->receive_queue); in __mptcp_move_skbs()
2054 const struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_inq_hint() local
2057 skb = skb_peek(&msk->receive_queue); in mptcp_inq_hint()
2059 u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in mptcp_inq_hint()
2076 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg() local
2097 if (unlikely(msk->recvmsg_inq)) in mptcp_recvmsg()
2103 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); in mptcp_recvmsg()
2113 mptcp_cleanup_rbuf(msk); in mptcp_recvmsg()
2115 if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) in mptcp_recvmsg()
2137 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_recvmsg()
2138 mptcp_check_for_eof(msk); in mptcp_recvmsg()
2144 if (__mptcp_move_skbs(msk)) in mptcp_recvmsg()
2182 msk, skb_queue_empty_lockless(&sk->sk_receive_queue), in mptcp_recvmsg()
2183 skb_queue_empty(&msk->receive_queue), copied); in mptcp_recvmsg()
2185 mptcp_rcv_space_adjust(msk, copied); in mptcp_recvmsg()
2196 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_timer() local
2201 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_retransmit_timer()
2205 __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); in mptcp_retransmit_timer()
2224 static struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) in mptcp_subflow_get_retrans() argument
2230 sock_owned_by_me((const struct sock *)msk); in mptcp_subflow_get_retrans()
2232 if (__mptcp_check_fallback(msk)) in mptcp_subflow_get_retrans()
2235 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
2243 mptcp_pm_subflow_chk_stale(msk, ssk); in mptcp_subflow_get_retrans()
2265 static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk) in mptcp_dispose_initial_subflow() argument
2267 if (msk->subflow) { in mptcp_dispose_initial_subflow()
2268 iput(SOCK_INODE(msk->subflow)); in mptcp_dispose_initial_subflow()
2269 msk->subflow = NULL; in mptcp_dispose_initial_subflow()
2276 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retransmit_pending_data() local
2296 msk->recovery_snd_nxt = msk->snd_nxt; in __mptcp_retransmit_pending_data()
2297 msk->recovery = true; in __mptcp_retransmit_pending_data()
2300 msk->first_pending = rtx_head; in __mptcp_retransmit_pending_data()
2301 msk->snd_burst = 0; in __mptcp_retransmit_pending_data()
2304 list_for_each_entry(cur, &msk->rtx_queue, list) { in __mptcp_retransmit_pending_data()
2329 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_ssk() local
2332 dispose_it = !msk->subflow || ssk != msk->subflow->sk; in __mptcp_close_ssk()
2350 msk->subflow->state = SS_UNCONNECTED; in __mptcp_close_ssk()
2382 if (ssk == msk->first) in __mptcp_close_ssk()
2383 msk->first = NULL; in __mptcp_close_ssk()
2386 if (ssk == msk->last_snd) in __mptcp_close_ssk()
2387 msk->last_snd = NULL; in __mptcp_close_ssk()
2412 static void __mptcp_close_subflow(struct mptcp_sock *msk) in __mptcp_close_subflow() argument
2418 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in __mptcp_close_subflow()
2428 mptcp_close_ssk((struct sock *)msk, ssk, subflow); in __mptcp_close_subflow()
2451 static void mptcp_check_fastclose(struct mptcp_sock *msk) in mptcp_check_fastclose() argument
2454 struct sock *sk = &msk->sk.icsk_inet.sk; in mptcp_check_fastclose()
2456 if (likely(!READ_ONCE(msk->rcv_fastclose))) in mptcp_check_fastclose()
2459 mptcp_token_destroy(msk); in mptcp_check_fastclose()
2461 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in mptcp_check_fastclose()
2490 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); in mptcp_check_fastclose()
2502 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retrans() local
2512 ssk = mptcp_subflow_get_retrans(msk); in __mptcp_retrans()
2515 if (mptcp_data_fin_enabled(msk)) { in __mptcp_retrans()
2520 mptcp_send_ack(msk); in __mptcp_retrans()
2538 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; in __mptcp_retrans()
2552 WRITE_ONCE(msk->allow_infinite_fallback, false); in __mptcp_retrans()
2567 void mptcp_reset_timeout(struct mptcp_sock *msk, unsigned long fail_tout) in mptcp_reset_timeout() argument
2569 struct sock *sk = (struct sock *)msk; in mptcp_reset_timeout()
2585 static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) in mptcp_mp_fail_no_response() argument
2587 struct sock *ssk = msk->first; in mptcp_mp_fail_no_response()
2600 mptcp_reset_timeout(msk, 0); in mptcp_mp_fail_no_response()
2606 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_do_fastclose() local
2608 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_do_fastclose()
2615 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); in mptcp_worker() local
2616 struct sock *sk = &msk->sk.icsk_inet.sk; in mptcp_worker()
2627 mptcp_check_fastclose(msk); in mptcp_worker()
2629 mptcp_pm_nl_work(msk); in mptcp_worker()
2631 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_worker()
2632 mptcp_check_for_eof(msk); in mptcp_worker()
2652 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) in mptcp_worker()
2653 __mptcp_close_subflow(msk); in mptcp_worker()
2655 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_worker()
2658 fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; in mptcp_worker()
2660 mptcp_mp_fail_no_response(msk); in mptcp_worker()
2669 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock() local
2671 INIT_LIST_HEAD(&msk->conn_list); in __mptcp_init_sock()
2672 INIT_LIST_HEAD(&msk->join_list); in __mptcp_init_sock()
2673 INIT_LIST_HEAD(&msk->rtx_queue); in __mptcp_init_sock()
2674 INIT_WORK(&msk->work, mptcp_worker); in __mptcp_init_sock()
2675 __skb_queue_head_init(&msk->receive_queue); in __mptcp_init_sock()
2676 msk->out_of_order_queue = RB_ROOT; in __mptcp_init_sock()
2677 msk->first_pending = NULL; in __mptcp_init_sock()
2678 msk->rmem_fwd_alloc = 0; in __mptcp_init_sock()
2679 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_init_sock()
2680 msk->timer_ival = TCP_RTO_MIN; in __mptcp_init_sock()
2682 msk->first = NULL; in __mptcp_init_sock()
2684 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in __mptcp_init_sock()
2685 WRITE_ONCE(msk->allow_infinite_fallback, true); in __mptcp_init_sock()
2686 msk->recovery = false; in __mptcp_init_sock()
2688 mptcp_pm_data_init(msk); in __mptcp_init_sock()
2691 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
2742 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit() local
2745 WRITE_ONCE(msk->first_pending, NULL); in __mptcp_clear_xmit()
2746 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) in __mptcp_clear_xmit()
2752 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work() local
2754 if (cancel_work_sync(&msk->work)) in mptcp_cancel_work()
2817 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_check_send_data_fin() local
2820 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), in __mptcp_check_send_data_fin()
2821 msk->snd_nxt, msk->write_seq); in __mptcp_check_send_data_fin()
2826 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || in __mptcp_check_send_data_fin()
2830 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in __mptcp_check_send_data_fin()
2835 if (__mptcp_check_fallback(msk)) { in __mptcp_check_send_data_fin()
2836 WRITE_ONCE(msk->snd_una, msk->write_seq); in __mptcp_check_send_data_fin()
2845 mptcp_for_each_subflow(msk, subflow) { in __mptcp_check_send_data_fin()
2854 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_wr_shutdown() local
2857 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, in __mptcp_wr_shutdown()
2861 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in __mptcp_wr_shutdown()
2862 WRITE_ONCE(msk->snd_data_fin_enable, 1); in __mptcp_wr_shutdown()
2869 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_destroy_sock() local
2871 pr_debug("msk=%p", msk); in __mptcp_destroy_sock()
2877 msk->pm.status = 0; in __mptcp_destroy_sock()
2881 WARN_ON_ONCE(msk->rmem_fwd_alloc); in __mptcp_destroy_sock()
2882 WARN_ON_ONCE(msk->rmem_released); in __mptcp_destroy_sock()
2890 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) in mptcp_check_readable() argument
2895 if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && in mptcp_check_readable()
2896 skb_queue_empty_lockless(&msk->receive_queue)) in mptcp_check_readable()
2905 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close() local
2915 if (mptcp_check_readable(msk)) { in __mptcp_close()
2928 mptcp_for_each_subflow(msk, subflow) { in __mptcp_close()
2935 if (ssk == msk->first) in __mptcp_close()
2950 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); in __mptcp_close()
2956 mptcp_reset_timeout(msk, 0); in __mptcp_close()
2976 void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
2980 struct ipv6_pinfo *msk6 = inet6_sk(msk); in mptcp_copy_inaddrs()
2982 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
2983 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
2991 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
2992 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
2993 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
2994 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
2995 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
2996 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
3001 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_disconnect() local
3014 mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); in mptcp_disconnect()
3015 msk->last_snd = NULL; in mptcp_disconnect()
3016 WRITE_ONCE(msk->flags, 0); in mptcp_disconnect()
3017 msk->cb_flags = 0; in mptcp_disconnect()
3018 msk->push_pending = 0; in mptcp_disconnect()
3019 msk->recovery = false; in mptcp_disconnect()
3020 msk->can_ack = false; in mptcp_disconnect()
3021 msk->fully_established = false; in mptcp_disconnect()
3022 msk->rcv_data_fin = false; in mptcp_disconnect()
3023 msk->snd_data_fin_enable = false; in mptcp_disconnect()
3024 msk->rcv_fastclose = false; in mptcp_disconnect()
3025 msk->use_64bit_ack = false; in mptcp_disconnect()
3026 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in mptcp_disconnect()
3027 mptcp_pm_data_reset(msk); in mptcp_disconnect()
3050 struct mptcp_sock *msk; in mptcp_sk_clone() local
3063 msk = mptcp_sk(nsk); in mptcp_sk_clone()
3064 msk->local_key = subflow_req->local_key; in mptcp_sk_clone()
3065 msk->token = subflow_req->token; in mptcp_sk_clone()
3066 msk->subflow = NULL; in mptcp_sk_clone()
3067 WRITE_ONCE(msk->fully_established, false); in mptcp_sk_clone()
3069 WRITE_ONCE(msk->csum_enabled, true); in mptcp_sk_clone()
3071 msk->write_seq = subflow_req->idsn + 1; in mptcp_sk_clone()
3072 msk->snd_nxt = msk->write_seq; in mptcp_sk_clone()
3073 msk->snd_una = msk->write_seq; in mptcp_sk_clone()
3074 msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; in mptcp_sk_clone()
3075 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; in mptcp_sk_clone()
3078 msk->can_ack = true; in mptcp_sk_clone()
3079 msk->remote_key = mp_opt->sndr_key; in mptcp_sk_clone()
3080 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); in mptcp_sk_clone()
3082 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_sk_clone()
3083 atomic64_set(&msk->rcv_wnd_sent, ack_seq); in mptcp_sk_clone()
3098 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
3102 msk->rcvq_space.copied = 0; in mptcp_rcv_space_init()
3103 msk->rcvq_space.rtt_us = 0; in mptcp_rcv_space_init()
3105 msk->rcvq_space.time = tp->tcp_mstamp; in mptcp_rcv_space_init()
3108 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, in mptcp_rcv_space_init()
3110 if (msk->rcvq_space.space == 0) in mptcp_rcv_space_init()
3111 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; in mptcp_rcv_space_init()
3113 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); in mptcp_rcv_space_init()
3119 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_accept() local
3123 listener = __mptcp_nmpc_socket(msk); in mptcp_accept()
3129 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); in mptcp_accept()
3134 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); in mptcp_accept()
3164 void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) in mptcp_destroy_common() argument
3167 struct sock *sk = (struct sock *)msk; in mptcp_destroy_common()
3172 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_destroy_common()
3177 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); in mptcp_destroy_common()
3179 skb_rbtree_purge(&msk->out_of_order_queue); in mptcp_destroy_common()
3185 sk->sk_forward_alloc += msk->rmem_fwd_alloc; in mptcp_destroy_common()
3186 msk->rmem_fwd_alloc = 0; in mptcp_destroy_common()
3187 mptcp_token_destroy(msk); in mptcp_destroy_common()
3188 mptcp_pm_free_anno_list(msk); in mptcp_destroy_common()
3189 mptcp_free_local_addr_list(msk); in mptcp_destroy_common()
3194 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy() local
3199 mptcp_dispose_initial_subflow(msk); in mptcp_destroy()
3200 mptcp_destroy_common(msk, 0); in mptcp_destroy()
3240 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_release_cb() local
3243 unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED) | in mptcp_release_cb()
3244 msk->push_pending; in mptcp_release_cb()
3255 msk->push_pending = 0; in mptcp_release_cb()
3256 msk->cb_flags &= ~flags; in mptcp_release_cb()
3269 if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) in mptcp_release_cb()
3271 if (unlikely(&msk->cb_flags)) { in mptcp_release_cb()
3275 if (__test_and_clear_bit(MPTCP_CONNECTED, &msk->cb_flags)) in mptcp_release_cb()
3277 if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) in mptcp_release_cb()
3279 if (__test_and_clear_bit(MPTCP_RESET_SCHEDULER, &msk->cb_flags)) in mptcp_release_cb()
3280 msk->last_snd = NULL; in mptcp_release_cb()
3348 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port() local
3351 ssock = __mptcp_nmpc_socket(msk); in mptcp_get_port()
3352 pr_debug("msk=%p, subflow=%p", msk, ssock); in mptcp_get_port()
3362 struct mptcp_sock *msk; in mptcp_finish_connect() local
3368 msk = mptcp_sk(sk); in mptcp_finish_connect()
3380 WRITE_ONCE(msk->remote_key, subflow->remote_key); in mptcp_finish_connect()
3381 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
3382 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); in mptcp_finish_connect()
3383 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in mptcp_finish_connect()
3384 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_finish_connect()
3385 WRITE_ONCE(msk->can_ack, 1); in mptcp_finish_connect()
3386 WRITE_ONCE(msk->snd_una, msk->write_seq); in mptcp_finish_connect()
3387 atomic64_set(&msk->rcv_wnd_sent, ack_seq); in mptcp_finish_connect()
3389 mptcp_pm_new_connection(msk, ssk, 0); in mptcp_finish_connect()
3391 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
3406 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join() local
3407 struct sock *parent = (void *)msk; in mptcp_finish_join()
3410 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
3421 if (!mptcp_pm_allow_new_subflow(msk)) in mptcp_finish_join()
3430 ret = __mptcp_finish_join(msk, ssk); in mptcp_finish_join()
3433 list_add_tail(&subflow->node, &msk->conn_list); in mptcp_finish_join()
3437 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
3438 __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); in mptcp_finish_join()
3448 subflow->map_seq = READ_ONCE(msk->ack_seq); in mptcp_finish_join()
3449 WRITE_ONCE(msk->allow_infinite_fallback, false); in mptcp_finish_join()
3452 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); in mptcp_finish_join()
3469 static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) in mptcp_ioctl_outq() argument
3471 const struct sock *sk = (void *)msk; in mptcp_ioctl_outq()
3480 delta = msk->write_seq - v; in mptcp_ioctl_outq()
3481 if (__mptcp_check_fallback(msk) && msk->first) { in mptcp_ioctl_outq()
3482 struct tcp_sock *tp = tcp_sk(msk->first); in mptcp_ioctl_outq()
3488 if (!((1 << msk->first->sk_state) & in mptcp_ioctl_outq()
3500 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_ioctl() local
3510 __mptcp_move_skbs(msk); in mptcp_ioctl()
3516 answ = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); in mptcp_ioctl()
3521 answ = mptcp_ioctl_outq(msk, msk->snd_nxt); in mptcp_ioctl()
3531 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, in mptcp_subflow_early_fallback() argument
3535 __mptcp_do_fallback(msk); in mptcp_subflow_early_fallback()
3541 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_connect() local
3545 ssock = __mptcp_nmpc_socket(msk); in mptcp_connect()
3549 mptcp_token_destroy(msk); in mptcp_connect()
3557 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3561 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3563 if (likely(!__mptcp_check_fallback(msk))) in mptcp_connect()
3569 if (msk->is_sendmsg) in mptcp_connect()
3570 err = __inet_stream_connect(ssock, uaddr, addr_len, msk->connect_flags, 1); in mptcp_connect()
3572 err = inet_stream_connect(ssock, uaddr, addr_len, msk->connect_flags); in mptcp_connect()
3629 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind() local
3634 ssock = __mptcp_nmpc_socket(msk); in mptcp_bind()
3663 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen() local
3667 pr_debug("msk=%p", msk); in mptcp_listen()
3670 ssock = __mptcp_nmpc_socket(msk); in mptcp_listen()
3676 mptcp_token_destroy(msk); in mptcp_listen()
3693 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept() local
3697 pr_debug("msk=%p", msk); in mptcp_stream_accept()
3699 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_accept()
3705 struct mptcp_sock *msk = mptcp_sk(newsock->sk); in mptcp_stream_accept() local
3718 subflow = mptcp_subflow_ctx(msk->first); in mptcp_stream_accept()
3719 list_add(&subflow->node, &msk->conn_list); in mptcp_stream_accept()
3720 sock_hold(msk->first); in mptcp_stream_accept()
3722 mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); in mptcp_stream_accept()
3724 mptcp_rcv_space_init(msk, msk->first); in mptcp_stream_accept()
3725 mptcp_propagate_sndbuf(newsk, msk->first); in mptcp_stream_accept()
3730 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
3742 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) in mptcp_check_writeable() argument
3744 struct sock *sk = (struct sock *)msk; in mptcp_check_writeable()
3764 struct mptcp_sock *msk; in mptcp_poll() local
3768 msk = mptcp_sk(sk); in mptcp_poll()
3772 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); in mptcp_poll()
3774 if (WARN_ON_ONCE(!msk->subflow || !msk->subflow->sk)) in mptcp_poll()
3777 return inet_csk_listen_poll(msk->subflow->sk); in mptcp_poll()
3781 mask |= mptcp_check_readable(msk); in mptcp_poll()
3782 mask |= mptcp_check_writeable(msk); in mptcp_poll()