Lines Matching refs:msk
34 struct mptcp_sock msk; member
53 static u64 mptcp_wnd_end(const struct mptcp_sock *msk) in mptcp_wnd_end() argument
55 return READ_ONCE(msk->wnd_end); in mptcp_wnd_end()
82 static int __mptcp_socket_create(struct mptcp_sock *msk) in __mptcp_socket_create() argument
85 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create()
93 msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; in __mptcp_socket_create()
94 WRITE_ONCE(msk->first, ssock->sk); in __mptcp_socket_create()
96 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
99 subflow->subflow_id = msk->subflow_id++; in __mptcp_socket_create()
103 mptcp_sock_graft(msk->first, sk->sk_socket); in __mptcp_socket_create()
112 struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) in __mptcp_nmpc_sk() argument
114 struct sock *sk = (struct sock *)msk; in __mptcp_nmpc_sk()
120 if (!msk->first) { in __mptcp_nmpc_sk()
121 ret = __mptcp_socket_create(msk); in __mptcp_nmpc_sk()
125 mptcp_sockopt_sync(msk, msk->first); in __mptcp_nmpc_sk()
128 return msk->first; in __mptcp_nmpc_sk()
174 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, in mptcp_ooo_try_coalesce() argument
180 return mptcp_try_coalesce((struct sock *)msk, to, from); in mptcp_ooo_try_coalesce()
192 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_rmem_uncharge() local
196 reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk); in mptcp_rmem_uncharge()
225 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) in mptcp_data_queue_ofo() argument
227 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo()
234 max_seq = atomic64_read(&msk->rcv_wnd_sent); in mptcp_data_queue_ofo()
236 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, in mptcp_data_queue_ofo()
237 RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_data_queue_ofo()
243 (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); in mptcp_data_queue_ofo()
248 p = &msk->out_of_order_queue.rb_node; in mptcp_data_queue_ofo()
250 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { in mptcp_data_queue_ofo()
252 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
253 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
260 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { in mptcp_data_queue_ofo()
267 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { in mptcp_data_queue_ofo()
269 parent = &msk->ooo_last_skb->rbnode; in mptcp_data_queue_ofo()
301 &msk->out_of_order_queue); in mptcp_data_queue_ofo()
306 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { in mptcp_data_queue_ofo()
316 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
323 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
329 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
338 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_rmem_schedule() local
341 if (size <= msk->rmem_fwd_alloc) in mptcp_rmem_schedule()
344 size -= msk->rmem_fwd_alloc; in mptcp_rmem_schedule()
354 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
359 struct sock *sk = (struct sock *)msk; in __mptcp_move_skb()
383 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { in __mptcp_move_skb()
385 msk->bytes_received += copy_len; in __mptcp_move_skb()
386 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); in __mptcp_move_skb()
394 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { in __mptcp_move_skb()
395 mptcp_data_queue_ofo(msk, skb); in __mptcp_move_skb()
431 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin_ack() local
435 msk->write_seq == READ_ONCE(msk->snd_una); in mptcp_pending_data_fin_ack()
440 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack() local
444 WRITE_ONCE(msk->snd_data_fin_enable, 0); in mptcp_check_data_fin_ack()
462 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin() local
464 if (READ_ONCE(msk->rcv_data_fin) && in mptcp_pending_data_fin()
467 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); in mptcp_pending_data_fin()
469 if (msk->ack_seq == rcv_data_fin_seq) { in mptcp_pending_data_fin()
535 static void mptcp_send_ack(struct mptcp_sock *msk) in mptcp_send_ack() argument
539 mptcp_for_each_subflow(msk, subflow) in mptcp_send_ack()
566 static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) in mptcp_cleanup_rbuf() argument
568 int old_space = READ_ONCE(msk->old_wspace); in mptcp_cleanup_rbuf()
570 struct sock *sk = (struct sock *)msk; in mptcp_cleanup_rbuf()
577 mptcp_for_each_subflow(msk, subflow) { in mptcp_cleanup_rbuf()
587 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin() local
605 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); in mptcp_check_data_fin()
606 WRITE_ONCE(msk->rcv_data_fin, 0); in mptcp_check_data_fin()
628 if (!__mptcp_check_fallback(msk)) in mptcp_check_data_fin()
629 mptcp_send_ack(msk); in mptcp_check_data_fin()
635 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, in __mptcp_move_skbs_from_subflow() argument
640 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow()
658 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
681 if (__mptcp_check_fallback(msk)) { in __mptcp_move_skbs_from_subflow()
703 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
728 static bool __mptcp_ofo_queue(struct mptcp_sock *msk) in __mptcp_ofo_queue() argument
730 struct sock *sk = (struct sock *)msk; in __mptcp_ofo_queue()
736 p = rb_first(&msk->out_of_order_queue); in __mptcp_ofo_queue()
737 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); in __mptcp_ofo_queue()
740 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) in __mptcp_ofo_queue()
744 rb_erase(&skb->rbnode, &msk->out_of_order_queue); in __mptcp_ofo_queue()
747 msk->ack_seq))) { in __mptcp_ofo_queue()
755 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { in __mptcp_ofo_queue()
756 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in __mptcp_ofo_queue()
760 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, in __mptcp_ofo_queue()
766 msk->bytes_received += end_seq - msk->ack_seq; in __mptcp_ofo_queue()
767 msk->ack_seq = end_seq; in __mptcp_ofo_queue()
806 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_error_report() local
808 mptcp_for_each_subflow(msk, subflow) in __mptcp_error_report()
816 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
818 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk()
821 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
822 __mptcp_ofo_queue(msk); in move_skbs_to_msk()
827 __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); in move_skbs_to_msk()
843 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready() local
866 if (move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
872 static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) in mptcp_subflow_joined() argument
874 mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); in mptcp_subflow_joined()
875 WRITE_ONCE(msk->allow_infinite_fallback, false); in mptcp_subflow_joined()
876 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); in mptcp_subflow_joined()
879 static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) in __mptcp_finish_join() argument
881 struct sock *sk = (struct sock *)msk; in __mptcp_finish_join()
892 mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; in __mptcp_finish_join()
893 mptcp_sockopt_sync_locked(msk, ssk); in __mptcp_finish_join()
894 mptcp_subflow_joined(msk, ssk); in __mptcp_finish_join()
902 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_flush_join_list() local
908 list_move_tail(&subflow->node, &msk->conn_list); in __mptcp_flush_join_list()
909 if (!__mptcp_finish_join(msk, ssk)) in __mptcp_flush_join_list()
946 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) in mptcp_subflow_recv_lookup() argument
950 msk_owned_by_me(msk); in mptcp_subflow_recv_lookup()
952 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
979 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, in mptcp_frag_can_collapse_to() argument
986 df->data_seq + df->data_len == msk->write_seq; in mptcp_frag_can_collapse_to()
1006 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clean_una() local
1010 snd_una = msk->snd_una; in __mptcp_clean_una()
1011 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { in __mptcp_clean_una()
1015 if (unlikely(dfrag == msk->first_pending)) { in __mptcp_clean_una()
1017 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1020 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __mptcp_clean_una()
1032 if (WARN_ON_ONCE(!msk->recovery)) in __mptcp_clean_una()
1048 if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) in __mptcp_clean_una()
1049 msk->recovery = false; in __mptcp_clean_una()
1052 if (snd_una == READ_ONCE(msk->snd_nxt) && in __mptcp_clean_una()
1053 snd_una == READ_ONCE(msk->write_seq)) { in __mptcp_clean_una()
1054 if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) in __mptcp_clean_una()
1079 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_enter_memory_pressure() local
1083 mptcp_for_each_subflow(msk, subflow) { in mptcp_enter_memory_pressure()
1107 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, in mptcp_carve_data_frag() argument
1115 dfrag->data_seq = msk->write_seq; in mptcp_carve_data_frag()
1133 static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, in mptcp_check_allowed_size() argument
1136 u64 window_end = mptcp_wnd_end(msk); in mptcp_check_allowed_size()
1139 if (__mptcp_check_fallback(msk)) in mptcp_check_allowed_size()
1218 static void mptcp_update_infinite_map(struct mptcp_sock *msk, in mptcp_update_infinite_map() argument
1230 pr_fallback(msk); in mptcp_update_infinite_map()
1240 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag() local
1250 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); in mptcp_sendmsg_frag()
1297 copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); in mptcp_sendmsg_frag()
1299 u64 snd_una = READ_ONCE(msk->snd_una); in mptcp_sendmsg_frag()
1301 if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) { in mptcp_sendmsg_frag()
1354 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1360 if (READ_ONCE(msk->csum_enabled)) in mptcp_sendmsg_frag()
1363 mptcp_update_infinite_map(msk, ssk, mpext); in mptcp_sendmsg_frag()
1410 struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) in mptcp_subflow_get_send() argument
1414 struct sock *sk = (struct sock *)msk; in mptcp_subflow_get_send()
1427 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1471 burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); in mptcp_subflow_get_send()
1480 msk->snd_burst = burst; in mptcp_subflow_get_send()
1490 static void mptcp_update_post_push(struct mptcp_sock *msk, in mptcp_update_post_push() argument
1498 msk->snd_burst -= sent; in mptcp_update_post_push()
1511 if (likely(after64(snd_nxt_new, msk->snd_nxt))) { in mptcp_update_post_push()
1512 msk->bytes_sent += snd_nxt_new - msk->snd_nxt; in mptcp_update_post_push()
1513 msk->snd_nxt = snd_nxt_new; in mptcp_update_post_push()
1526 struct mptcp_sock *msk = mptcp_sk(sk); in __subflow_push_pending() local
1547 mptcp_update_post_push(msk, dfrag, ret); in __subflow_push_pending()
1549 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); in __subflow_push_pending()
1551 if (msk->snd_burst <= 0 || in __subflow_push_pending()
1568 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_push_pending() local
1579 if (mptcp_sched_get_send(msk)) in __mptcp_push_pending()
1584 mptcp_for_each_subflow(msk, subflow) { in __mptcp_push_pending()
1632 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_subflow_push_pending() local
1658 if (mptcp_sched_get_send(msk)) in __mptcp_subflow_push_pending()
1669 mptcp_for_each_subflow(msk, subflow) { in __mptcp_subflow_push_pending()
1691 if (msk->snd_data_fin_enable && in __mptcp_subflow_push_pending()
1692 msk->snd_nxt + 1 == msk->write_seq) in __mptcp_subflow_push_pending()
1712 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_fastopen() local
1723 ssk = __mptcp_nmpc_sk(msk); in mptcp_sendmsg_fastopen()
1727 if (!msk->first) in mptcp_sendmsg_fastopen()
1730 ssk = msk->first; in mptcp_sendmsg_fastopen()
1734 msk->fastopening = 1; in mptcp_sendmsg_fastopen()
1736 msk->fastopening = 0; in mptcp_sendmsg_fastopen()
1766 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg() local
1813 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); in mptcp_sendmsg()
1821 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); in mptcp_sendmsg()
1849 WRITE_ONCE(msk->write_seq, msk->write_seq + psize); in mptcp_sendmsg()
1857 list_add_tail(&dfrag->list, &msk->rtx_queue); in mptcp_sendmsg()
1858 if (!msk->first_pending) in mptcp_sendmsg()
1859 WRITE_ONCE(msk->first_pending, dfrag); in mptcp_sendmsg()
1861 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, in mptcp_sendmsg()
1890 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, in __mptcp_recvmsg_mskq() argument
1899 skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { in __mptcp_recvmsg_mskq()
1932 WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); in __mptcp_recvmsg_mskq()
1933 __skb_unlink(skb, &msk->receive_queue); in __mptcp_recvmsg_mskq()
1948 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) in mptcp_rcv_space_adjust() argument
1951 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust()
1956 msk_owned_by_me(msk); in mptcp_rcv_space_adjust()
1961 msk->rcvq_space.copied += copied; in mptcp_rcv_space_adjust()
1964 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); in mptcp_rcv_space_adjust()
1966 rtt_us = msk->rcvq_space.rtt_us; in mptcp_rcv_space_adjust()
1971 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1986 msk->rcvq_space.rtt_us = rtt_us; in mptcp_rcv_space_adjust()
1987 msk->scaling_ratio = scaling_ratio; in mptcp_rcv_space_adjust()
1991 if (msk->rcvq_space.copied <= msk->rcvq_space.space) in mptcp_rcv_space_adjust()
1999 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; in mptcp_rcv_space_adjust()
2001 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); in mptcp_rcv_space_adjust()
2003 do_div(grow, msk->rcvq_space.space); in mptcp_rcv_space_adjust()
2020 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
2034 msk->rcvq_space.space = msk->rcvq_space.copied; in mptcp_rcv_space_adjust()
2036 msk->rcvq_space.copied = 0; in mptcp_rcv_space_adjust()
2037 msk->rcvq_space.time = mstamp; in mptcp_rcv_space_adjust()
2042 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_update_rmem() local
2044 if (!msk->rmem_released) in __mptcp_update_rmem()
2047 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); in __mptcp_update_rmem()
2048 mptcp_rmem_uncharge(sk, msk->rmem_released); in __mptcp_update_rmem()
2049 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_update_rmem()
2054 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_splice_receive_queue() local
2056 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); in __mptcp_splice_receive_queue()
2059 static bool __mptcp_move_skbs(struct mptcp_sock *msk) in __mptcp_move_skbs() argument
2061 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs()
2066 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs()
2079 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
2089 if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || in __mptcp_move_skbs()
2093 ret |= __mptcp_ofo_queue(msk); in __mptcp_move_skbs()
2098 mptcp_check_data_fin((struct sock *)msk); in __mptcp_move_skbs()
2099 return !skb_queue_empty(&msk->receive_queue); in __mptcp_move_skbs()
2104 const struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_inq_hint() local
2107 skb = skb_peek(&msk->receive_queue); in mptcp_inq_hint()
2109 u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in mptcp_inq_hint()
2126 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg() local
2147 if (unlikely(msk->recvmsg_inq)) in mptcp_recvmsg()
2153 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); in mptcp_recvmsg()
2163 mptcp_cleanup_rbuf(msk); in mptcp_recvmsg()
2165 if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) in mptcp_recvmsg()
2191 if (__mptcp_move_skbs(msk)) in mptcp_recvmsg()
2229 msk, skb_queue_empty_lockless(&sk->sk_receive_queue), in mptcp_recvmsg()
2230 skb_queue_empty(&msk->receive_queue), copied); in mptcp_recvmsg()
2232 mptcp_rcv_space_adjust(msk, copied); in mptcp_recvmsg()
2243 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_timer() local
2248 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_retransmit_timer()
2252 __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); in mptcp_retransmit_timer()
2271 struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) in mptcp_subflow_get_retrans() argument
2277 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
2285 mptcp_pm_subflow_chk_stale(msk, ssk); in mptcp_subflow_get_retrans()
2310 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retransmit_pending_data() local
2312 if (__mptcp_check_fallback(msk)) in __mptcp_retransmit_pending_data()
2330 msk->recovery_snd_nxt = msk->snd_nxt; in __mptcp_retransmit_pending_data()
2331 msk->recovery = true; in __mptcp_retransmit_pending_data()
2334 msk->first_pending = rtx_head; in __mptcp_retransmit_pending_data()
2335 msk->snd_burst = 0; in __mptcp_retransmit_pending_data()
2338 list_for_each_entry(cur, &msk->rtx_queue, list) { in __mptcp_retransmit_pending_data()
2383 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_ssk() local
2391 if (msk->in_accept_queue && msk->first == ssk && in __mptcp_close_ssk()
2401 dispose_it = msk->free_first || ssk != msk->first; in __mptcp_close_ssk()
2407 if ((flags & MPTCP_CF_FASTCLOSE) && !__mptcp_check_fallback(msk)) { in __mptcp_close_ssk()
2447 if (ssk == msk->first) in __mptcp_close_ssk()
2448 WRITE_ONCE(msk->first, NULL); in __mptcp_close_ssk()
2459 if (list_is_singular(&msk->conn_list) && msk->first && in __mptcp_close_ssk()
2460 inet_sk_state_load(msk->first) == TCP_CLOSE) { in __mptcp_close_ssk()
2462 msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { in __mptcp_close_ssk()
2493 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_subflow() local
2497 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in __mptcp_close_subflow()
2522 static void mptcp_check_fastclose(struct mptcp_sock *msk) in mptcp_check_fastclose() argument
2525 struct sock *sk = (struct sock *)msk; in mptcp_check_fastclose()
2527 if (likely(!READ_ONCE(msk->rcv_fastclose))) in mptcp_check_fastclose()
2530 mptcp_token_destroy(msk); in mptcp_check_fastclose()
2532 mptcp_for_each_subflow_safe(msk, subflow, tmp) { in mptcp_check_fastclose()
2561 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); in mptcp_check_fastclose()
2573 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retrans() local
2584 err = mptcp_sched_get_retrans(msk); in __mptcp_retrans()
2587 if (mptcp_data_fin_enabled(msk)) { in __mptcp_retrans()
2592 mptcp_send_ack(msk); in __mptcp_retrans()
2606 mptcp_for_each_subflow(msk, subflow) { in __mptcp_retrans()
2618 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : in __mptcp_retrans()
2633 WRITE_ONCE(msk->allow_infinite_fallback, false); in __mptcp_retrans()
2640 msk->bytes_retrans += len; in __mptcp_retrans()
2653 void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) in mptcp_reset_tout_timer() argument
2655 struct sock *sk = (struct sock *)msk; in mptcp_reset_tout_timer()
2672 static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) in mptcp_mp_fail_no_response() argument
2674 struct sock *ssk = msk->first; in mptcp_mp_fail_no_response()
2691 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_do_fastclose() local
2694 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_do_fastclose()
2701 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); in mptcp_worker() local
2702 struct sock *sk = (struct sock *)msk; in mptcp_worker()
2711 mptcp_check_fastclose(msk); in mptcp_worker()
2713 mptcp_pm_nl_work(msk); in mptcp_worker()
2719 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) in mptcp_worker()
2732 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_worker()
2735 fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; in mptcp_worker()
2737 mptcp_mp_fail_no_response(msk); in mptcp_worker()
2746 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock() local
2748 INIT_LIST_HEAD(&msk->conn_list); in __mptcp_init_sock()
2749 INIT_LIST_HEAD(&msk->join_list); in __mptcp_init_sock()
2750 INIT_LIST_HEAD(&msk->rtx_queue); in __mptcp_init_sock()
2751 INIT_WORK(&msk->work, mptcp_worker); in __mptcp_init_sock()
2752 __skb_queue_head_init(&msk->receive_queue); in __mptcp_init_sock()
2753 msk->out_of_order_queue = RB_ROOT; in __mptcp_init_sock()
2754 msk->first_pending = NULL; in __mptcp_init_sock()
2755 msk->rmem_fwd_alloc = 0; in __mptcp_init_sock()
2756 WRITE_ONCE(msk->rmem_released, 0); in __mptcp_init_sock()
2757 msk->timer_ival = TCP_RTO_MIN; in __mptcp_init_sock()
2759 WRITE_ONCE(msk->first, NULL); in __mptcp_init_sock()
2761 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in __mptcp_init_sock()
2762 WRITE_ONCE(msk->allow_infinite_fallback, true); in __mptcp_init_sock()
2763 msk->recovery = false; in __mptcp_init_sock()
2764 msk->subflow_id = 1; in __mptcp_init_sock()
2766 mptcp_pm_data_init(msk); in __mptcp_init_sock()
2769 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
2819 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit() local
2822 WRITE_ONCE(msk->first_pending, NULL); in __mptcp_clear_xmit()
2823 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) in __mptcp_clear_xmit()
2829 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work() local
2831 if (cancel_work_sync(&msk->work)) in mptcp_cancel_work()
2900 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_send_data_fin() local
2903 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), in mptcp_check_send_data_fin()
2904 msk->snd_nxt, msk->write_seq); in mptcp_check_send_data_fin()
2909 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || in mptcp_check_send_data_fin()
2913 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in mptcp_check_send_data_fin()
2915 mptcp_for_each_subflow(msk, subflow) { in mptcp_check_send_data_fin()
2924 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_wr_shutdown() local
2927 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, in __mptcp_wr_shutdown()
2931 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in __mptcp_wr_shutdown()
2932 WRITE_ONCE(msk->snd_data_fin_enable, 1); in __mptcp_wr_shutdown()
2939 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_destroy_sock() local
2941 pr_debug("msk=%p", msk); in __mptcp_destroy_sock()
2947 msk->pm.status = 0; in __mptcp_destroy_sock()
2948 mptcp_release_sched(msk); in __mptcp_destroy_sock()
2952 WARN_ON_ONCE(msk->rmem_fwd_alloc); in __mptcp_destroy_sock()
2953 WARN_ON_ONCE(msk->rmem_released); in __mptcp_destroy_sock()
2967 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) in mptcp_check_readable() argument
2972 if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && in mptcp_check_readable()
2973 skb_queue_empty_lockless(&msk->receive_queue)) in mptcp_check_readable()
3002 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close() local
3014 if (mptcp_check_readable(msk) || timeout < 0) { in __mptcp_close()
3028 mptcp_for_each_subflow(msk, subflow) { in __mptcp_close()
3037 if (ssk == msk->first) in __mptcp_close()
3057 if (msk->token) in __mptcp_close()
3058 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); in __mptcp_close()
3084 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
3088 struct ipv6_pinfo *msk6 = inet6_sk(msk); in mptcp_copy_inaddrs()
3090 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
3091 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
3099 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
3100 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
3101 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
3102 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
3103 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
3104 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
3109 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_disconnect() local
3115 if (msk->fastopening) in mptcp_disconnect()
3124 if (msk->token) in mptcp_disconnect()
3125 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); in mptcp_disconnect()
3130 mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); in mptcp_disconnect()
3131 WRITE_ONCE(msk->flags, 0); in mptcp_disconnect()
3132 msk->cb_flags = 0; in mptcp_disconnect()
3133 msk->push_pending = 0; in mptcp_disconnect()
3134 msk->recovery = false; in mptcp_disconnect()
3135 msk->can_ack = false; in mptcp_disconnect()
3136 msk->fully_established = false; in mptcp_disconnect()
3137 msk->rcv_data_fin = false; in mptcp_disconnect()
3138 msk->snd_data_fin_enable = false; in mptcp_disconnect()
3139 msk->rcv_fastclose = false; in mptcp_disconnect()
3140 msk->use_64bit_ack = false; in mptcp_disconnect()
3141 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in mptcp_disconnect()
3142 mptcp_pm_data_reset(msk); in mptcp_disconnect()
3144 msk->bytes_acked = 0; in mptcp_disconnect()
3145 msk->bytes_received = 0; in mptcp_disconnect()
3146 msk->bytes_sent = 0; in mptcp_disconnect()
3147 msk->bytes_retrans = 0; in mptcp_disconnect()
3170 struct mptcp_sock *msk; in mptcp_sk_clone_init() local
3182 msk = mptcp_sk(nsk); in mptcp_sk_clone_init()
3183 msk->local_key = subflow_req->local_key; in mptcp_sk_clone_init()
3184 msk->token = subflow_req->token; in mptcp_sk_clone_init()
3185 msk->in_accept_queue = 1; in mptcp_sk_clone_init()
3186 WRITE_ONCE(msk->fully_established, false); in mptcp_sk_clone_init()
3188 WRITE_ONCE(msk->csum_enabled, true); in mptcp_sk_clone_init()
3190 msk->write_seq = subflow_req->idsn + 1; in mptcp_sk_clone_init()
3191 msk->snd_nxt = msk->write_seq; in mptcp_sk_clone_init()
3192 msk->snd_una = msk->write_seq; in mptcp_sk_clone_init()
3193 msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; in mptcp_sk_clone_init()
3194 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; in mptcp_sk_clone_init()
3195 mptcp_init_sched(msk, mptcp_sk(sk)->sched); in mptcp_sk_clone_init()
3198 msk->subflow_id = 2; in mptcp_sk_clone_init()
3209 WRITE_ONCE(msk->first, ssk); in mptcp_sk_clone_init()
3210 list_add(&mptcp_subflow_ctx(ssk)->node, &msk->conn_list); in mptcp_sk_clone_init()
3216 mptcp_token_accept(subflow_req, msk); in mptcp_sk_clone_init()
3224 mptcp_rcv_space_init(msk, ssk); in mptcp_sk_clone_init()
3231 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
3235 msk->rcvq_space.copied = 0; in mptcp_rcv_space_init()
3236 msk->rcvq_space.rtt_us = 0; in mptcp_rcv_space_init()
3238 msk->rcvq_space.time = tp->tcp_mstamp; in mptcp_rcv_space_init()
3241 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, in mptcp_rcv_space_init()
3243 if (msk->rcvq_space.space == 0) in mptcp_rcv_space_init()
3244 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; in mptcp_rcv_space_init()
3246 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); in mptcp_rcv_space_init()
3287 void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) in mptcp_destroy_common() argument
3290 struct sock *sk = (struct sock *)msk; in mptcp_destroy_common()
3295 mptcp_for_each_subflow_safe(msk, subflow, tmp) in mptcp_destroy_common()
3300 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); in mptcp_destroy_common()
3302 skb_rbtree_purge(&msk->out_of_order_queue); in mptcp_destroy_common()
3308 sk_forward_alloc_add(sk, msk->rmem_fwd_alloc); in mptcp_destroy_common()
3309 WRITE_ONCE(msk->rmem_fwd_alloc, 0); in mptcp_destroy_common()
3310 mptcp_token_destroy(msk); in mptcp_destroy_common()
3311 mptcp_pm_free_anno_list(msk); in mptcp_destroy_common()
3312 mptcp_free_local_addr_list(msk); in mptcp_destroy_common()
3317 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy() local
3320 msk->free_first = 1; in mptcp_destroy()
3321 mptcp_destroy_common(msk, 0); in mptcp_destroy()
3355 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_release_cb() local
3358 unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED) | in mptcp_release_cb()
3359 msk->push_pending; in mptcp_release_cb()
3366 list_splice_init(&msk->join_list, &join_list); in mptcp_release_cb()
3375 msk->push_pending = 0; in mptcp_release_cb()
3376 msk->cb_flags &= ~flags; in mptcp_release_cb()
3390 if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) in mptcp_release_cb()
3392 if (unlikely(msk->cb_flags)) { in mptcp_release_cb()
3396 if (__test_and_clear_bit(MPTCP_CONNECTED, &msk->cb_flags)) in mptcp_release_cb()
3398 if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) in mptcp_release_cb()
3464 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port() local
3466 pr_debug("msk=%p, ssk=%p", msk, msk->first); in mptcp_get_port()
3467 if (WARN_ON_ONCE(!msk->first)) in mptcp_get_port()
3470 return inet_csk_get_port(msk->first, snum); in mptcp_get_port()
3476 struct mptcp_sock *msk; in mptcp_finish_connect() local
3481 msk = mptcp_sk(sk); in mptcp_finish_connect()
3491 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
3492 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); in mptcp_finish_connect()
3493 WRITE_ONCE(msk->snd_nxt, msk->write_seq); in mptcp_finish_connect()
3494 WRITE_ONCE(msk->snd_una, msk->write_seq); in mptcp_finish_connect()
3496 mptcp_pm_new_connection(msk, ssk, 0); in mptcp_finish_connect()
3498 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
3513 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join() local
3514 struct sock *parent = (void *)msk; in mptcp_finish_join()
3517 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
3527 mptcp_subflow_joined(msk, ssk); in mptcp_finish_join()
3531 if (!mptcp_pm_allow_new_subflow(msk)) in mptcp_finish_join()
3539 ret = __mptcp_finish_join(msk, ssk); in mptcp_finish_join()
3542 list_add_tail(&subflow->node, &msk->conn_list); in mptcp_finish_join()
3546 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
3547 __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); in mptcp_finish_join()
3574 static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) in mptcp_ioctl_outq() argument
3576 const struct sock *sk = (void *)msk; in mptcp_ioctl_outq()
3585 delta = msk->write_seq - v; in mptcp_ioctl_outq()
3586 if (__mptcp_check_fallback(msk) && msk->first) { in mptcp_ioctl_outq()
3587 struct tcp_sock *tp = tcp_sk(msk->first); in mptcp_ioctl_outq()
3593 if (!((1 << msk->first->sk_state) & in mptcp_ioctl_outq()
3605 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_ioctl() local
3614 __mptcp_move_skbs(msk); in mptcp_ioctl()
3620 *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); in mptcp_ioctl()
3625 *karg = mptcp_ioctl_outq(msk, msk->snd_nxt); in mptcp_ioctl()
3635 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, in mptcp_subflow_early_fallback() argument
3639 __mptcp_do_fallback(msk); in mptcp_subflow_early_fallback()
3645 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_connect() local
3649 ssk = __mptcp_nmpc_sk(msk); in mptcp_connect()
3660 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3664 mptcp_subflow_early_fallback(msk, subflow); in mptcp_connect()
3666 if (likely(!__mptcp_check_fallback(msk))) in mptcp_connect()
3672 if (!msk->fastopening) in mptcp_connect()
3694 if (!msk->fastopening) in mptcp_connect()
3702 mptcp_token_destroy(msk); in mptcp_connect()
3747 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind() local
3752 ssk = __mptcp_nmpc_sk(msk); in mptcp_bind()
3774 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen() local
3779 pr_debug("msk=%p", msk); in mptcp_listen()
3787 ssk = __mptcp_nmpc_sk(msk); in mptcp_listen()
3815 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept() local
3819 pr_debug("msk=%p", msk); in mptcp_stream_accept()
3824 ssk = READ_ONCE(msk->first); in mptcp_stream_accept()
3836 struct mptcp_sock *msk = mptcp_sk(newsk); in mptcp_stream_accept() local
3840 msk->in_accept_queue = 0; in mptcp_stream_accept()
3845 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
3855 if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) { in mptcp_stream_accept()
3856 __mptcp_close_ssk(newsk, msk->first, in mptcp_stream_accept()
3857 mptcp_subflow_ctx(msk->first), 0); in mptcp_stream_accept()
3858 if (unlikely(list_is_singular(&msk->conn_list))) in mptcp_stream_accept()
3867 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) in mptcp_check_writeable() argument
3869 struct sock *sk = (struct sock *)msk; in mptcp_check_writeable()
3886 struct mptcp_sock *msk; in mptcp_poll() local
3891 msk = mptcp_sk(sk); in mptcp_poll()
3895 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); in mptcp_poll()
3897 struct sock *ssk = READ_ONCE(msk->first); in mptcp_poll()
3912 mask |= mptcp_check_readable(msk); in mptcp_poll()
3916 mask |= mptcp_check_writeable(msk); in mptcp_poll()