Lines Matching refs:msk
33 struct mptcp_sock msk; member
63 struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) in __mptcp_nmpc_socket() argument
65 if (!msk->subflow || READ_ONCE(msk->can_ack)) in __mptcp_nmpc_socket()
68 return msk->subflow; in __mptcp_nmpc_socket()
72 static u64 mptcp_wnd_end(const struct mptcp_sock *msk) in mptcp_wnd_end() argument
74 return READ_ONCE(msk->wnd_end); in mptcp_wnd_end()
101 static int __mptcp_socket_create(struct mptcp_sock *msk) in __mptcp_socket_create() argument
104 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create()
112 msk->first = ssock->sk; in __mptcp_socket_create()
113 msk->subflow = ssock; in __mptcp_socket_create()
115 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
118 mptcp_sock_graft(msk->first, sk->sk_socket); in __mptcp_socket_create()
149 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, in mptcp_ooo_try_coalesce() argument
155 return mptcp_try_coalesce((struct sock *)msk, to, from); in mptcp_ooo_try_coalesce()
162 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) in mptcp_data_queue_ofo() argument
164 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo()
171 max_seq = READ_ONCE(msk->rcv_wnd_sent); in mptcp_data_queue_ofo()
173 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, in mptcp_data_queue_ofo()
174 RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_data_queue_ofo()
180 (unsigned long long)msk->rcv_wnd_sent); in mptcp_data_queue_ofo()
185 p = &msk->out_of_order_queue.rb_node; in mptcp_data_queue_ofo()
187 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { in mptcp_data_queue_ofo()
189 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
190 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
197 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { in mptcp_data_queue_ofo()
204 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { in mptcp_data_queue_ofo()
206 parent = &msk->ooo_last_skb->rbnode; in mptcp_data_queue_ofo()
238 &msk->out_of_order_queue); in mptcp_data_queue_ofo()
243 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { in mptcp_data_queue_ofo()
253 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
260 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
266 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
273 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
278 struct sock *sk = (struct sock *)msk; in __mptcp_move_skb()
309 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { in __mptcp_move_skb()
311 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); in __mptcp_move_skb()
319 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { in __mptcp_move_skb()
320 mptcp_data_queue_ofo(msk, skb); in __mptcp_move_skb()
356 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin_ack() local
358 return !__mptcp_check_fallback(msk) && in mptcp_pending_data_fin_ack()
361 msk->write_seq == READ_ONCE(msk->snd_una); in mptcp_pending_data_fin_ack()
366 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack() local
370 WRITE_ONCE(msk->snd_data_fin_enable, 0); in mptcp_check_data_fin_ack()
388 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin() local
390 if (READ_ONCE(msk->rcv_data_fin) && in mptcp_pending_data_fin()
393 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); in mptcp_pending_data_fin()
395 if (msk->ack_seq == rcv_data_fin_seq) { in mptcp_pending_data_fin()
453 static void mptcp_send_ack(struct mptcp_sock *msk) in mptcp_send_ack() argument
457 mptcp_for_each_subflow(msk, subflow) in mptcp_send_ack()
484 static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) in mptcp_cleanup_rbuf() argument
486 int old_space = READ_ONCE(msk->old_wspace); in mptcp_cleanup_rbuf()
488 struct sock *sk = (struct sock *)msk; in mptcp_cleanup_rbuf()
495 mptcp_for_each_subflow(msk, subflow) { in mptcp_cleanup_rbuf()
505 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin() local
509 if (__mptcp_check_fallback(msk)) in mptcp_check_data_fin()
526 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); in mptcp_check_data_fin()
527 WRITE_ONCE(msk->rcv_data_fin, 0); in mptcp_check_data_fin()
549 mptcp_send_ack(msk); in mptcp_check_data_fin()
555 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, in __mptcp_move_skbs_from_subflow() argument
560 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow()
578 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
601 if (__mptcp_check_fallback(msk)) { in __mptcp_move_skbs_from_subflow()
623 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
648 static bool __mptcp_ofo_queue(struct mptcp_sock *msk) in __mptcp_ofo_queue() argument
650 struct sock *sk = (struct sock *)msk; in __mptcp_ofo_queue()
656 p = rb_first(&msk->out_of_order_queue); in __mptcp_ofo_queue()
657 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); in __mptcp_ofo_queue()
660 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) in __mptcp_ofo_queue()
664 rb_erase(&skb->rbnode, &msk->out_of_order_queue); in __mptcp_ofo_queue()
667 msk->ack_seq))) { in __mptcp_ofo_queue()
675 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { in __mptcp_ofo_queue()
676 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in __mptcp_ofo_queue()
680 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, in __mptcp_ofo_queue()
685 msk->ack_seq = end_seq; in __mptcp_ofo_queue()
694 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
696 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk()
699 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
700 __mptcp_ofo_queue(msk); in move_skbs_to_msk()
705 set_bit(MPTCP_ERROR_REPORT, &msk->flags); in move_skbs_to_msk()
721 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready() local
744 if (move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
750 static bool mptcp_do_flush_join_list(struct mptcp_sock *msk) in mptcp_do_flush_join_list() argument
755 if (likely(list_empty(&msk->join_list))) in mptcp_do_flush_join_list()
758 spin_lock_bh(&msk->join_list_lock); in mptcp_do_flush_join_list()
759 list_for_each_entry(subflow, &msk->join_list, node) { in mptcp_do_flush_join_list()
762 mptcp_propagate_sndbuf((struct sock *)msk, mptcp_subflow_tcp_sock(subflow)); in mptcp_do_flush_join_list()
763 if (READ_ONCE(msk->setsockopt_seq) != sseq) in mptcp_do_flush_join_list()
766 list_splice_tail_init(&msk->join_list, &msk->conn_list); in mptcp_do_flush_join_list()
767 spin_unlock_bh(&msk->join_list_lock); in mptcp_do_flush_join_list()
772 void __mptcp_flush_join_list(struct mptcp_sock *msk) in __mptcp_flush_join_list() argument
774 if (likely(!mptcp_do_flush_join_list(msk))) in __mptcp_flush_join_list()
777 if (!test_and_set_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags)) in __mptcp_flush_join_list()
778 mptcp_schedule_work((struct sock *)msk); in __mptcp_flush_join_list()
781 static void mptcp_flush_join_list(struct mptcp_sock *msk) in mptcp_flush_join_list() argument
783 bool sync_needed = test_and_clear_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags); in mptcp_flush_join_list()
787 if (!mptcp_do_flush_join_list(msk) && !sync_needed) in mptcp_flush_join_list()
790 mptcp_sockopt_sync_all(msk); in mptcp_flush_join_list()
830 static void mptcp_check_for_eof(struct mptcp_sock *msk) in mptcp_check_for_eof() argument
833 struct sock *sk = (struct sock *)msk; in mptcp_check_for_eof()
836 mptcp_for_each_subflow(msk, subflow) in mptcp_check_for_eof()
867 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) in mptcp_subflow_recv_lookup() argument
870 struct sock *sk = (struct sock *)msk; in mptcp_subflow_recv_lookup()
874 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
901 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, in mptcp_frag_can_collapse_to() argument
908 df->data_seq + df->data_len == msk->write_seq; in mptcp_frag_can_collapse_to()
919 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_wmem_reserve() local
921 WARN_ON_ONCE(msk->wmem_reserved); in __mptcp_wmem_reserve()
943 msk->wmem_reserved = amount; in __mptcp_wmem_reserve()
949 msk->wmem_reserved = -1; in __mptcp_wmem_reserve()
954 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_update_wmem() local
960 if (!msk->wmem_reserved) in __mptcp_update_wmem()
963 if (msk->wmem_reserved < 0) in __mptcp_update_wmem()
964 msk->wmem_reserved = 0; in __mptcp_update_wmem()
965 if (msk->wmem_reserved > 0) { in __mptcp_update_wmem()
966 sk->sk_forward_alloc += msk->wmem_reserved; in __mptcp_update_wmem()
967 msk->wmem_reserved = 0; in __mptcp_update_wmem()
973 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_wmem_alloc() local
976 if (msk->wmem_reserved < 0) in mptcp_wmem_alloc()
979 if (msk->wmem_reserved >= size) in mptcp_wmem_alloc()
989 msk->wmem_reserved += size; in mptcp_wmem_alloc()
993 msk->wmem_reserved -= size; in mptcp_wmem_alloc()
999 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_wmem_uncharge() local
1001 if (msk->wmem_reserved < 0) in mptcp_wmem_uncharge()
1002 msk->wmem_reserved = 0; in mptcp_wmem_uncharge()
1003 msk->wmem_reserved += size; in mptcp_wmem_uncharge()
1015 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_mem_reclaim_partial() local
1021 if (msk->wmem_reserved < 0) in mptcp_mem_reclaim_partial()
1025 sk->sk_forward_alloc += msk->wmem_reserved; in mptcp_mem_reclaim_partial()
1027 msk->wmem_reserved = sk->sk_forward_alloc; in mptcp_mem_reclaim_partial()
1049 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clean_una() local
1057 if (__mptcp_check_fallback(msk)) in __mptcp_clean_una()
1058 msk->snd_una = READ_ONCE(msk->snd_nxt); in __mptcp_clean_una()
1060 snd_una = msk->snd_una; in __mptcp_clean_una()
1061 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { in __mptcp_clean_una()
1065 if (unlikely(dfrag == msk->first_pending)) { in __mptcp_clean_una()
1067 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1070 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_clean_una()
1083 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1100 if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) in __mptcp_clean_una()
1101 msk->recovery = false; in __mptcp_clean_una()
1107 if (snd_una == READ_ONCE(msk->snd_nxt) && !msk->recovery) { in __mptcp_clean_una()
1108 if (mptcp_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) in __mptcp_clean_una()
1134 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_enter_memory_pressure() local
1138 mptcp_for_each_subflow(msk, subflow) { in mptcp_enter_memory_pressure()
1162 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, in mptcp_carve_data_frag() argument
1170 dfrag->data_seq = msk->write_seq; in mptcp_carve_data_frag()
1188 static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq, in mptcp_check_allowed_size() argument
1191 u64 window_end = mptcp_wnd_end(msk); in mptcp_check_allowed_size()
1193 if (__mptcp_check_fallback(msk)) in mptcp_check_allowed_size()
1287 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag() local
1297 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); in mptcp_sendmsg_frag()
1330 avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size); in mptcp_sendmsg_frag()
1332 u64 snd_una = READ_ONCE(msk->snd_una); in mptcp_sendmsg_frag()
1334 if (skb || snd_una != msk->snd_nxt) in mptcp_sendmsg_frag()
1382 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1388 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1431 static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) in mptcp_subflow_get_send() argument
1435 struct sock *sk = (struct sock *)msk; in mptcp_subflow_get_send()
1444 if (__mptcp_check_fallback(msk)) { in mptcp_subflow_get_send()
1445 if (!msk->first) in mptcp_subflow_get_send()
1447 return sk_stream_memory_free(msk->first) ? msk->first : NULL; in mptcp_subflow_get_send()
1451 if (msk->last_snd && msk->snd_burst > 0 && in mptcp_subflow_get_send()
1452 sk_stream_memory_free(msk->last_snd) && in mptcp_subflow_get_send()
1453 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) { in mptcp_subflow_get_send()
1455 return msk->last_snd; in mptcp_subflow_get_send()
1463 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1492 msk->last_snd = send_info[0].ssk; in mptcp_subflow_get_send()
1493 msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE, in mptcp_subflow_get_send()
1494 tcp_sk(msk->last_snd)->snd_wnd); in mptcp_subflow_get_send()
1495 return msk->last_snd; in mptcp_subflow_get_send()
1511 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_push_pending() local
1526 mptcp_flush_join_list(msk); in __mptcp_push_pending()
1527 ssk = mptcp_subflow_get_send(msk); in __mptcp_push_pending()
1552 msk->snd_nxt += ret; in __mptcp_push_pending()
1553 msk->snd_burst -= ret; in __mptcp_push_pending()
1554 msk->tx_pending_data -= ret; in __mptcp_push_pending()
1558 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_push_pending()
1575 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_subflow_push_pending() local
1610 msk->snd_nxt += ret; in __mptcp_subflow_push_pending()
1611 msk->snd_burst -= ret; in __mptcp_subflow_push_pending()
1612 msk->tx_pending_data -= ret; in __mptcp_subflow_push_pending()
1617 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_subflow_push_pending()
1631 if (msk->snd_data_fin_enable && in __mptcp_subflow_push_pending()
1632 msk->snd_nxt + 1 == msk->write_seq) in __mptcp_subflow_push_pending()
1648 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg() local
1688 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); in mptcp_sendmsg()
1696 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); in mptcp_sendmsg()
1724 WRITE_ONCE(msk->write_seq, msk->write_seq + psize); in mptcp_sendmsg()
1725 msk->tx_pending_data += psize; in mptcp_sendmsg()
1733 list_add_tail(&dfrag->list, &msk->rtx_queue); in mptcp_sendmsg()
1734 if (!msk->first_pending) in mptcp_sendmsg()
1735 WRITE_ONCE(msk->first_pending, dfrag); in mptcp_sendmsg()
1737 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, in mptcp_sendmsg()
1759 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, in __mptcp_recvmsg_mskq() argument
1768 skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { in __mptcp_recvmsg_mskq()
1799 WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); in __mptcp_recvmsg_mskq()
1800 __skb_unlink(skb, &msk->receive_queue); in __mptcp_recvmsg_mskq()
1815 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) in mptcp_rcv_space_adjust() argument
1818 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust()
1827 msk->rcvq_space.copied += copied; in mptcp_rcv_space_adjust()
1830 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); in mptcp_rcv_space_adjust()
1832 rtt_us = msk->rcvq_space.rtt_us; in mptcp_rcv_space_adjust()
1837 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1851 msk->rcvq_space.rtt_us = rtt_us; in mptcp_rcv_space_adjust()
1855 if (msk->rcvq_space.copied <= msk->rcvq_space.space) in mptcp_rcv_space_adjust()
1863 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; in mptcp_rcv_space_adjust()
1865 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1867 do_div(grow, msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1889 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1903 msk->rcvq_space.space = msk->rcvq_space.copied; in mptcp_rcv_space_adjust()
1905 msk->rcvq_space.copied = 0; in mptcp_rcv_space_adjust()
1906 msk->rcvq_space.time = mstamp; in mptcp_rcv_space_adjust()
1911 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_update_rmem() local
1913 if (!msk->rmem_released) in __mptcp_update_rmem()
1916 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); in __mptcp_update_rmem()
1917 sk_mem_uncharge(sk, msk->rmem_released); in __mptcp_update_rmem()
1918 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_update_rmem()
1923 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_splice_receive_queue() local
1925 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); in __mptcp_splice_receive_queue()
1928 static bool __mptcp_move_skbs(struct mptcp_sock *msk) in __mptcp_move_skbs() argument
1930 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs()
1934 mptcp_flush_join_list(msk); in __mptcp_move_skbs()
1936 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs()
1949 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
1959 if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || in __mptcp_move_skbs()
1963 ret |= __mptcp_ofo_queue(msk); in __mptcp_move_skbs()
1968 mptcp_check_data_fin((struct sock *)msk); in __mptcp_move_skbs()
1969 return !skb_queue_empty(&msk->receive_queue); in __mptcp_move_skbs()
1975 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg() local
1999 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); in mptcp_recvmsg()
2009 mptcp_cleanup_rbuf(msk); in mptcp_recvmsg()
2011 if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) in mptcp_recvmsg()
2033 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_recvmsg()
2034 mptcp_check_for_eof(msk); in mptcp_recvmsg()
2040 if (__mptcp_move_skbs(msk)) in mptcp_recvmsg()
2072 msk, skb_queue_empty_lockless(&sk->sk_receive_queue), in mptcp_recvmsg()
2073 skb_queue_empty(&msk->receive_queue), copied); in mptcp_recvmsg()
2075 mptcp_rcv_space_adjust(msk, copied); in mptcp_recvmsg()
2086 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_timer() local
2091 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_retransmit_timer()
2095 set_bit(MPTCP_RETRANSMIT, &msk->flags); in mptcp_retransmit_timer()
2114 static struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) in mptcp_subflow_get_retrans() argument
2120 sock_owned_by_me((const struct sock *)msk); in mptcp_subflow_get_retrans()
2122 if (__mptcp_check_fallback(msk)) in mptcp_subflow_get_retrans()
2125 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
2133 mptcp_pm_subflow_chk_stale(msk, ssk); in mptcp_subflow_get_retrans()
2155 static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk) in mptcp_dispose_initial_subflow() argument
2157 if (msk->subflow) { in mptcp_dispose_initial_subflow()
2158 iput(SOCK_INODE(msk->subflow)); in mptcp_dispose_initial_subflow()
2159 msk->subflow = NULL; in mptcp_dispose_initial_subflow()
2166 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retransmit_pending_data() local
2187 if (!msk->recovery || after64(msk->snd_nxt, msk->recovery_snd_nxt)) in __mptcp_retransmit_pending_data()
2188 msk->recovery_snd_nxt = msk->snd_nxt; in __mptcp_retransmit_pending_data()
2189 msk->recovery = true; in __mptcp_retransmit_pending_data()
2192 msk->first_pending = rtx_head; in __mptcp_retransmit_pending_data()
2193 msk->tx_pending_data += msk->snd_nxt - rtx_head->data_seq; in __mptcp_retransmit_pending_data()
2194 msk->snd_nxt = rtx_head->data_seq; in __mptcp_retransmit_pending_data()
2195 msk->snd_burst = 0; in __mptcp_retransmit_pending_data()
2198 list_for_each_entry(cur, &msk->rtx_queue, list) { in __mptcp_retransmit_pending_data()
2218 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_ssk() local
2251 if (ssk == msk->last_snd) in __mptcp_close_ssk()
2252 msk->last_snd = NULL; in __mptcp_close_ssk()
2254 if (ssk == msk->first) in __mptcp_close_ssk()
2255 msk->first = NULL; in __mptcp_close_ssk()
2257 if (msk->subflow && ssk == msk->subflow->sk) in __mptcp_close_ssk()
2258 mptcp_dispose_initial_subflow(msk); in __mptcp_close_ssk()
2277 static void __mptcp_close_subflow(struct mptcp_sock *msk) in __mptcp_close_subflow() argument
2283 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { in __mptcp_close_subflow()
2293 mptcp_close_ssk((struct sock *)msk, ssk, subflow); in __mptcp_close_subflow()
2316 static void mptcp_check_fastclose(struct mptcp_sock *msk) in mptcp_check_fastclose() argument
2319 struct sock *sk = &msk->sk.icsk_inet.sk; in mptcp_check_fastclose()
2321 if (likely(!READ_ONCE(msk->rcv_fastclose))) in mptcp_check_fastclose()
2324 mptcp_token_destroy(msk); in mptcp_check_fastclose()
2326 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { in mptcp_check_fastclose()
2341 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); in mptcp_check_fastclose()
2348 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retrans() local
2358 if (mptcp_data_fin_enabled(msk)) { in __mptcp_retrans()
2363 mptcp_send_ack(msk); in __mptcp_retrans()
2371 ssk = mptcp_subflow_get_retrans(msk); in __mptcp_retrans()
2379 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; in __mptcp_retrans()
2404 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); in mptcp_worker() local
2405 struct sock *sk = &msk->sk.icsk_inet.sk; in mptcp_worker()
2414 mptcp_flush_join_list(msk); in mptcp_worker()
2416 mptcp_check_fastclose(msk); in mptcp_worker()
2418 if (msk->pm.status) in mptcp_worker()
2419 mptcp_pm_nl_work(msk); in mptcp_worker()
2421 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_worker()
2422 mptcp_check_for_eof(msk); in mptcp_worker()
2438 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) in mptcp_worker()
2439 __mptcp_close_subflow(msk); in mptcp_worker()
2441 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_worker()
2451 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock() local
2453 spin_lock_init(&msk->join_list_lock); in __mptcp_init_sock()
2455 INIT_LIST_HEAD(&msk->conn_list); in __mptcp_init_sock()
2456 INIT_LIST_HEAD(&msk->join_list); in __mptcp_init_sock()
2457 INIT_LIST_HEAD(&msk->rtx_queue); in __mptcp_init_sock()
2458 INIT_WORK(&msk->work, mptcp_worker); in __mptcp_init_sock()
2459 __skb_queue_head_init(&msk->receive_queue); in __mptcp_init_sock()
2460 msk->out_of_order_queue = RB_ROOT; in __mptcp_init_sock()
2461 msk->first_pending = NULL; in __mptcp_init_sock()
2462 msk->wmem_reserved = 0; in __mptcp_init_sock()
2463 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_init_sock()
2464 msk->tx_pending_data = 0; in __mptcp_init_sock()
2465 msk->timer_ival = TCP_RTO_MIN; in __mptcp_init_sock()
2467 msk->first = NULL; in __mptcp_init_sock()
2469 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in __mptcp_init_sock()
2470 msk->recovery = false; in __mptcp_init_sock()
2472 mptcp_pm_data_init(msk); in __mptcp_init_sock()
2475 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
2520 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit() local
2523 WRITE_ONCE(msk->first_pending, NULL); in __mptcp_clear_xmit()
2524 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) in __mptcp_clear_xmit()
2530 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work() local
2532 if (cancel_work_sync(&msk->work)) in mptcp_cancel_work()
2595 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_check_send_data_fin() local
2598 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), in __mptcp_check_send_data_fin()
2599 msk->snd_nxt, msk->write_seq); in __mptcp_check_send_data_fin()
2604 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || in __mptcp_check_send_data_fin()
2608 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in __mptcp_check_send_data_fin()
2613 if (__mptcp_check_fallback(msk)) { in __mptcp_check_send_data_fin()
2622 mptcp_flush_join_list(msk); in __mptcp_check_send_data_fin()
2623 mptcp_for_each_subflow(msk, subflow) { in __mptcp_check_send_data_fin()
2632 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_wr_shutdown() local
2635 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, in __mptcp_wr_shutdown()
2639 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in __mptcp_wr_shutdown()
2640 WRITE_ONCE(msk->snd_data_fin_enable, 1); in __mptcp_wr_shutdown()
2648 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_destroy_sock() local
2651 pr_debug("msk=%p", msk); in __mptcp_destroy_sock()
2658 spin_lock_bh(&msk->join_list_lock); in __mptcp_destroy_sock()
2659 list_splice_tail_init(&msk->join_list, &msk->conn_list); in __mptcp_destroy_sock()
2660 spin_unlock_bh(&msk->join_list_lock); in __mptcp_destroy_sock()
2661 list_splice_init(&msk->conn_list, &conn_list); in __mptcp_destroy_sock()
2663 sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); in __mptcp_destroy_sock()
2665 msk->pm.status = 0; in __mptcp_destroy_sock()
2674 WARN_ON_ONCE(msk->wmem_reserved); in __mptcp_destroy_sock()
2675 WARN_ON_ONCE(msk->rmem_released); in __mptcp_destroy_sock()
2680 mptcp_dispose_initial_subflow(msk); in __mptcp_destroy_sock()
2732 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
2736 struct ipv6_pinfo *msk6 = inet6_sk(msk); in mptcp_copy_inaddrs()
2738 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
2739 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
2747 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
2748 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
2749 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
2750 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
2751 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
2752 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
2758 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_disconnect() local
2760 mptcp_do_flush_join_list(msk); in mptcp_disconnect()
2762 mptcp_for_each_subflow(msk, subflow) { in mptcp_disconnect()
2787 struct mptcp_sock *msk; in mptcp_sk_clone() local
2800 msk = mptcp_sk(nsk); in mptcp_sk_clone()
2801 msk->local_key = subflow_req->local_key; in mptcp_sk_clone()
2802 msk->token = subflow_req->token; in mptcp_sk_clone()
2803 msk->subflow = NULL; in mptcp_sk_clone()
2804 WRITE_ONCE(msk->fully_established, false); in mptcp_sk_clone()
2806 WRITE_ONCE(msk->csum_enabled, true); in mptcp_sk_clone()
2808 msk->write_seq = subflow_req->idsn + 1; in mptcp_sk_clone()
2809 msk->snd_nxt = msk->write_seq; in mptcp_sk_clone()
2810 msk->snd_una = msk->write_seq; in mptcp_sk_clone()
2811 msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; in mptcp_sk_clone()
2812 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; in mptcp_sk_clone()
2815 msk->can_ack = true; in mptcp_sk_clone()
2816 msk->remote_key = mp_opt->sndr_key; in mptcp_sk_clone()
2817 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); in mptcp_sk_clone()
2819 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_sk_clone()
2820 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); in mptcp_sk_clone()
2835 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
2839 msk->rcvq_space.copied = 0; in mptcp_rcv_space_init()
2840 msk->rcvq_space.rtt_us = 0; in mptcp_rcv_space_init()
2842 msk->rcvq_space.time = tp->tcp_mstamp; in mptcp_rcv_space_init()
2845 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, in mptcp_rcv_space_init()
2847 if (msk->rcvq_space.space == 0) in mptcp_rcv_space_init()
2848 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; in mptcp_rcv_space_init()
2850 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); in mptcp_rcv_space_init()
2856 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_accept() local
2860 listener = __mptcp_nmpc_socket(msk); in mptcp_accept()
2866 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); in mptcp_accept()
2871 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); in mptcp_accept()
2899 void mptcp_destroy_common(struct mptcp_sock *msk) in mptcp_destroy_common() argument
2901 struct sock *sk = (struct sock *)msk; in mptcp_destroy_common()
2906 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); in mptcp_destroy_common()
2908 skb_rbtree_purge(&msk->out_of_order_queue); in mptcp_destroy_common()
2909 mptcp_token_destroy(msk); in mptcp_destroy_common()
2910 mptcp_pm_free_anno_list(msk); in mptcp_destroy_common()
2915 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy() local
2917 mptcp_destroy_common(msk); in mptcp_destroy()
3027 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port() local
3030 ssock = __mptcp_nmpc_socket(msk); in mptcp_get_port()
3031 pr_debug("msk=%p, subflow=%p", msk, ssock); in mptcp_get_port()
3041 struct mptcp_sock *msk; in mptcp_finish_connect() local
3047 msk = mptcp_sk(sk); in mptcp_finish_connect()
3059 WRITE_ONCE(msk->remote_key, subflow->remote_key); in mptcp_finish_connect()
3060 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
3061 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); in mptcp_finish_connect()
3062 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in mptcp_finish_connect()
3063 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_finish_connect()
3064 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); in mptcp_finish_connect()
3065 WRITE_ONCE(msk->can_ack, 1); in mptcp_finish_connect()
3066 WRITE_ONCE(msk->snd_una, msk->write_seq); in mptcp_finish_connect()
3068 mptcp_pm_new_connection(msk, ssk, 0); in mptcp_finish_connect()
3070 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
3085 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join() local
3086 struct sock *parent = (void *)msk; in mptcp_finish_join()
3090 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
3098 if (!msk->pm.server_side) in mptcp_finish_join()
3101 if (!mptcp_pm_allow_new_subflow(msk)) { in mptcp_finish_join()
3111 spin_lock_bh(&msk->join_list_lock); in mptcp_finish_join()
3114 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
3117 spin_unlock_bh(&msk->join_list_lock); in mptcp_finish_join()
3129 subflow->map_seq = READ_ONCE(msk->ack_seq); in mptcp_finish_join()
3131 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); in mptcp_finish_join()
3173 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind() local
3178 ssock = __mptcp_nmpc_socket(msk); in mptcp_bind()
3193 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, in mptcp_subflow_early_fallback() argument
3197 __mptcp_do_fallback(msk); in mptcp_subflow_early_fallback()
3203 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_connect() local
3209 if (sock->state != SS_UNCONNECTED && msk->subflow) { in mptcp_stream_connect()
3213 ssock = msk->subflow; in mptcp_stream_connect()
3217 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_connect()
3223 mptcp_token_destroy(msk); in mptcp_stream_connect()
3231 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
3235 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
3237 if (likely(!__mptcp_check_fallback(msk))) in mptcp_stream_connect()
3259 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen() local
3263 pr_debug("msk=%p", msk); in mptcp_listen()
3266 ssock = __mptcp_nmpc_socket(msk); in mptcp_listen()
3272 mptcp_token_destroy(msk); in mptcp_listen()
3289 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept() local
3293 pr_debug("msk=%p", msk); in mptcp_stream_accept()
3299 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_accept()
3303 clear_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
3309 struct mptcp_sock *msk = mptcp_sk(newsock->sk); in mptcp_stream_accept() local
3322 subflow = mptcp_subflow_ctx(msk->first); in mptcp_stream_accept()
3323 list_add(&subflow->node, &msk->conn_list); in mptcp_stream_accept()
3324 sock_hold(msk->first); in mptcp_stream_accept()
3326 mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); in mptcp_stream_accept()
3328 mptcp_copy_inaddrs(newsk, msk->first); in mptcp_stream_accept()
3329 mptcp_rcv_space_init(msk, msk->first); in mptcp_stream_accept()
3330 mptcp_propagate_sndbuf(newsk, msk->first); in mptcp_stream_accept()
3335 mptcp_flush_join_list(msk); in mptcp_stream_accept()
3336 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
3346 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
3355 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) in mptcp_check_readable() argument
3360 if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && in mptcp_check_readable()
3361 skb_queue_empty_lockless(&msk->receive_queue)) in mptcp_check_readable()
3367 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) in mptcp_check_writeable() argument
3369 struct sock *sk = (struct sock *)msk; in mptcp_check_writeable()
3389 struct mptcp_sock *msk; in mptcp_poll() local
3393 msk = mptcp_sk(sk); in mptcp_poll()
3397 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); in mptcp_poll()
3399 return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : 0; in mptcp_poll()
3402 mask |= mptcp_check_readable(msk); in mptcp_poll()
3403 mask |= mptcp_check_writeable(msk); in mptcp_poll()