Lines Matching refs:msk
29 struct mptcp_sock msk; member
48 static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) in __mptcp_nmpc_socket() argument
50 if (!msk->subflow || READ_ONCE(msk->can_ack)) in __mptcp_nmpc_socket()
53 return msk->subflow; in __mptcp_nmpc_socket()
80 static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) in __mptcp_tcp_fallback() argument
82 sock_owned_by_me((const struct sock *)msk); in __mptcp_tcp_fallback()
84 if (likely(!__mptcp_check_fallback(msk))) in __mptcp_tcp_fallback()
87 return msk->first; in __mptcp_tcp_fallback()
90 static int __mptcp_socket_create(struct mptcp_sock *msk) in __mptcp_socket_create() argument
93 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create()
101 msk->first = ssock->sk; in __mptcp_socket_create()
102 msk->subflow = ssock; in __mptcp_socket_create()
104 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
110 RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); in __mptcp_socket_create()
141 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, in mptcp_ooo_try_coalesce() argument
147 return mptcp_try_coalesce((struct sock *)msk, to, from); in mptcp_ooo_try_coalesce()
154 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) in mptcp_data_queue_ofo() argument
156 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo()
165 max_seq = space > 0 ? space + msk->ack_seq : msk->ack_seq; in mptcp_data_queue_ofo()
167 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, in mptcp_data_queue_ofo()
168 RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_data_queue_ofo()
176 p = &msk->out_of_order_queue.rb_node; in mptcp_data_queue_ofo()
178 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { in mptcp_data_queue_ofo()
180 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
181 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
188 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { in mptcp_data_queue_ofo()
195 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { in mptcp_data_queue_ofo()
197 parent = &msk->ooo_last_skb->rbnode; in mptcp_data_queue_ofo()
229 &msk->out_of_order_queue); in mptcp_data_queue_ofo()
234 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { in mptcp_data_queue_ofo()
244 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
251 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
257 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
264 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
269 struct sock *sk = (struct sock *)msk; in __mptcp_move_skb()
294 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { in __mptcp_move_skb()
296 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); in __mptcp_move_skb()
304 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { in __mptcp_move_skb()
305 mptcp_data_queue_ofo(msk, skb); in __mptcp_move_skb()
328 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack() local
330 if (__mptcp_check_fallback(msk)) in mptcp_check_data_fin_ack()
336 msk->write_seq == atomic64_read(&msk->snd_una)) { in mptcp_check_data_fin_ack()
339 WRITE_ONCE(msk->snd_data_fin_enable, 0); in mptcp_check_data_fin_ack()
363 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin() local
365 if (READ_ONCE(msk->rcv_data_fin) && in mptcp_pending_data_fin()
368 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); in mptcp_pending_data_fin()
370 if (msk->ack_seq == rcv_data_fin_seq) { in mptcp_pending_data_fin()
393 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin() local
396 if (__mptcp_check_fallback(msk) || !msk->first) in mptcp_check_data_fin()
415 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); in mptcp_check_data_fin()
416 WRITE_ONCE(msk->rcv_data_fin, 0); in mptcp_check_data_fin()
420 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_check_data_fin()
440 mptcp_for_each_subflow(msk, subflow) { in mptcp_check_data_fin()
458 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, in __mptcp_move_skbs_from_subflow() argument
463 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow()
470 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
494 if (__mptcp_check_fallback(msk)) { in __mptcp_move_skbs_from_subflow()
516 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
544 static bool mptcp_ofo_queue(struct mptcp_sock *msk) in mptcp_ofo_queue() argument
546 struct sock *sk = (struct sock *)msk; in mptcp_ofo_queue()
552 p = rb_first(&msk->out_of_order_queue); in mptcp_ofo_queue()
553 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_ofo_queue()
556 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) in mptcp_ofo_queue()
560 rb_erase(&skb->rbnode, &msk->out_of_order_queue); in mptcp_ofo_queue()
563 msk->ack_seq))) { in mptcp_ofo_queue()
571 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { in mptcp_ofo_queue()
572 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in mptcp_ofo_queue()
576 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, in mptcp_ofo_queue()
581 msk->ack_seq = end_seq; in mptcp_ofo_queue()
590 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
592 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk()
603 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
604 mptcp_ofo_queue(msk); in move_skbs_to_msk()
612 schedule_work(&msk->work)) in move_skbs_to_msk()
624 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready() local
633 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_data_ready()
636 move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
651 move_skbs_to_msk(msk, ssk); in mptcp_data_ready()
658 static void __mptcp_flush_join_list(struct mptcp_sock *msk) in __mptcp_flush_join_list() argument
660 if (likely(list_empty(&msk->join_list))) in __mptcp_flush_join_list()
663 spin_lock_bh(&msk->join_list_lock); in __mptcp_flush_join_list()
664 list_splice_tail_init(&msk->join_list, &msk->conn_list); in __mptcp_flush_join_list()
665 spin_unlock_bh(&msk->join_list_lock); in __mptcp_flush_join_list()
697 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_subflow_eof() local
699 if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && in mptcp_subflow_eof()
700 schedule_work(&msk->work)) in mptcp_subflow_eof()
704 static void mptcp_check_for_eof(struct mptcp_sock *msk) in mptcp_check_for_eof() argument
707 struct sock *sk = (struct sock *)msk; in mptcp_check_for_eof()
710 mptcp_for_each_subflow(msk, subflow) in mptcp_check_for_eof()
720 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_check_for_eof()
725 static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) in mptcp_ext_cache_refill() argument
727 const struct sock *sk = (const struct sock *)msk; in mptcp_ext_cache_refill()
729 if (!msk->cached_ext) in mptcp_ext_cache_refill()
730 msk->cached_ext = __skb_ext_alloc(sk->sk_allocation); in mptcp_ext_cache_refill()
732 return !!msk->cached_ext; in mptcp_ext_cache_refill()
735 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) in mptcp_subflow_recv_lookup() argument
738 struct sock *sk = (struct sock *)msk; in mptcp_subflow_recv_lookup()
742 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
761 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, in mptcp_frag_can_collapse_to() argument
766 df->data_seq + df->data_len == msk->write_seq; in mptcp_frag_can_collapse_to()
784 static bool mptcp_is_writeable(struct mptcp_sock *msk) in mptcp_is_writeable() argument
788 if (!sk_stream_is_writeable((struct sock *)msk)) in mptcp_is_writeable()
791 mptcp_for_each_subflow(msk, subflow) { in mptcp_is_writeable()
800 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_clean_una() local
808 if (__mptcp_check_fallback(msk)) in mptcp_clean_una()
809 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_clean_una()
810 snd_una = atomic64_read(&msk->snd_una); in mptcp_clean_una()
812 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { in mptcp_clean_una()
840 if (mptcp_is_writeable(msk)) { in mptcp_clean_una()
867 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, in mptcp_carve_data_frag() argument
875 dfrag->data_seq = msk->write_seq; in mptcp_carve_data_frag()
890 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag() local
906 write_seq = &msk->write_seq; in mptcp_sendmsg_frag()
942 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); in mptcp_sendmsg_frag()
944 dfrag = mptcp_carve_data_frag(msk, pfrag, offset); in mptcp_sendmsg_frag()
991 list_add_tail(&dfrag->list, &msk->rtx_queue); in mptcp_sendmsg_frag()
1016 mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); in mptcp_sendmsg_frag()
1017 msk->cached_ext = NULL; in mptcp_sendmsg_frag()
1039 static void mptcp_nospace(struct mptcp_sock *msk) in mptcp_nospace() argument
1043 clear_bit(MPTCP_SEND_SPACE, &msk->flags); in mptcp_nospace()
1046 mptcp_for_each_subflow(msk, subflow) { in mptcp_nospace()
1079 static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk, in mptcp_subflow_get_send() argument
1089 sock_owned_by_me((struct sock *)msk); in mptcp_subflow_get_send()
1092 if (!mptcp_ext_cache_refill(msk)) in mptcp_subflow_get_send()
1095 if (__mptcp_check_fallback(msk)) { in mptcp_subflow_get_send()
1096 if (!msk->first) in mptcp_subflow_get_send()
1098 *sndbuf = msk->first->sk_sndbuf; in mptcp_subflow_get_send()
1099 return sk_stream_memory_free(msk->first) ? msk->first : NULL; in mptcp_subflow_get_send()
1103 if (msk->last_snd && msk->snd_burst > 0 && in mptcp_subflow_get_send()
1104 sk_stream_memory_free(msk->last_snd) && in mptcp_subflow_get_send()
1105 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) { in mptcp_subflow_get_send()
1106 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1110 return msk->last_snd; in mptcp_subflow_get_send()
1118 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1141 msk, nr_active, send_info[0].ssk, send_info[0].ratio, in mptcp_subflow_get_send()
1149 msk->last_snd = send_info[0].ssk; in mptcp_subflow_get_send()
1150 msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE, in mptcp_subflow_get_send()
1151 sk_stream_wspace(msk->last_snd)); in mptcp_subflow_get_send()
1152 return msk->last_snd; in mptcp_subflow_get_send()
1157 static void ssk_check_wmem(struct mptcp_sock *msk) in ssk_check_wmem() argument
1159 if (unlikely(!mptcp_is_writeable(msk))) in ssk_check_wmem()
1160 mptcp_nospace(msk); in ssk_check_wmem()
1166 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg() local
1196 __mptcp_flush_join_list(msk); in mptcp_sendmsg()
1197 ssk = mptcp_subflow_get_send(msk, &sndbuf); in mptcp_sendmsg()
1214 mptcp_nospace(msk); in mptcp_sendmsg()
1221 ssk = mptcp_subflow_get_send(msk, &sndbuf); in mptcp_sendmsg()
1222 if (list_empty(&msk->conn_list)) { in mptcp_sendmsg()
1252 msk->snd_burst -= ret; in mptcp_sendmsg()
1261 !mptcp_ext_cache_refill(msk)) { in mptcp_sendmsg()
1311 ssk_check_wmem(msk); in mptcp_sendmsg()
1319 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_wait_data() local
1325 test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); in mptcp_wait_data()
1331 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, in __mptcp_recvmsg_mskq() argument
1335 struct sock *sk = (struct sock *)msk; in __mptcp_recvmsg_mskq()
1373 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) in mptcp_rcv_space_adjust() argument
1376 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust()
1385 msk->rcvq_space.copied += copied; in mptcp_rcv_space_adjust()
1388 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); in mptcp_rcv_space_adjust()
1390 rtt_us = msk->rcvq_space.rtt_us; in mptcp_rcv_space_adjust()
1395 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1409 msk->rcvq_space.rtt_us = rtt_us; in mptcp_rcv_space_adjust()
1413 if (msk->rcvq_space.copied <= msk->rcvq_space.space) in mptcp_rcv_space_adjust()
1421 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; in mptcp_rcv_space_adjust()
1423 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1425 do_div(grow, msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1447 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1461 msk->rcvq_space.space = msk->rcvq_space.copied; in mptcp_rcv_space_adjust()
1463 msk->rcvq_space.copied = 0; in mptcp_rcv_space_adjust()
1464 msk->rcvq_space.time = mstamp; in mptcp_rcv_space_adjust()
1467 static bool __mptcp_move_skbs(struct mptcp_sock *msk) in __mptcp_move_skbs() argument
1473 if (((struct sock *)msk)->sk_state == TCP_CLOSE) in __mptcp_move_skbs()
1476 __mptcp_flush_join_list(msk); in __mptcp_move_skbs()
1478 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs()
1484 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
1488 if (mptcp_ofo_queue(msk) || moved > 0) { in __mptcp_move_skbs()
1489 mptcp_check_data_fin((struct sock *)msk); in __mptcp_move_skbs()
1498 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg() local
1511 __mptcp_flush_join_list(msk); in mptcp_recvmsg()
1516 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); in mptcp_recvmsg()
1526 __mptcp_move_skbs(msk)) in mptcp_recvmsg()
1548 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_recvmsg()
1549 mptcp_check_for_eof(msk); in mptcp_recvmsg()
1576 clear_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1581 if (unlikely(__mptcp_move_skbs(msk))) in mptcp_recvmsg()
1582 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1583 } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { in mptcp_recvmsg()
1585 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1589 msk, test_bit(MPTCP_DATA_READY, &msk->flags), in mptcp_recvmsg()
1591 mptcp_rcv_space_adjust(msk, copied); in mptcp_recvmsg()
1599 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_handler() local
1601 if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->write_seq)) { in mptcp_retransmit_handler()
1604 set_bit(MPTCP_WORK_RTX, &msk->flags); in mptcp_retransmit_handler()
1605 if (schedule_work(&msk->work)) in mptcp_retransmit_handler()
1634 static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) in mptcp_subflow_get_retrans() argument
1639 sock_owned_by_me((const struct sock *)msk); in mptcp_subflow_get_retrans()
1641 if (__mptcp_check_fallback(msk)) in mptcp_subflow_get_retrans()
1642 return msk->first; in mptcp_subflow_get_retrans()
1644 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
1696 static void pm_work(struct mptcp_sock *msk) in pm_work() argument
1698 struct mptcp_pm_data *pm = &msk->pm; in pm_work()
1700 spin_lock_bh(&msk->pm.lock); in pm_work()
1702 pr_debug("msk=%p status=%x", msk, pm->status); in pm_work()
1705 mptcp_pm_nl_add_addr_received(msk); in pm_work()
1709 mptcp_pm_nl_rm_addr_received(msk); in pm_work()
1713 mptcp_pm_nl_fully_established(msk); in pm_work()
1717 mptcp_pm_nl_subflow_established(msk); in pm_work()
1720 spin_unlock_bh(&msk->pm.lock); in pm_work()
1723 static void __mptcp_close_subflow(struct mptcp_sock *msk) in __mptcp_close_subflow() argument
1727 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { in __mptcp_close_subflow()
1733 __mptcp_close_ssk((struct sock *)msk, ssk, subflow, 0); in __mptcp_close_subflow()
1739 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); in mptcp_worker() local
1740 struct sock *ssk, *sk = &msk->sk.icsk_inet.sk; in mptcp_worker()
1753 __mptcp_flush_join_list(msk); in mptcp_worker()
1754 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) in mptcp_worker()
1755 __mptcp_close_subflow(msk); in mptcp_worker()
1757 __mptcp_move_skbs(msk); in mptcp_worker()
1759 if (msk->pm.status) in mptcp_worker()
1760 pm_work(msk); in mptcp_worker()
1762 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_worker()
1763 mptcp_check_for_eof(msk); in mptcp_worker()
1767 if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_worker()
1774 if (!mptcp_ext_cache_refill(msk)) in mptcp_worker()
1777 ssk = mptcp_subflow_get_retrans(msk); in mptcp_worker()
1797 if (!mptcp_ext_cache_refill(msk)) in mptcp_worker()
1822 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock() local
1824 spin_lock_init(&msk->join_list_lock); in __mptcp_init_sock()
1826 INIT_LIST_HEAD(&msk->conn_list); in __mptcp_init_sock()
1827 INIT_LIST_HEAD(&msk->join_list); in __mptcp_init_sock()
1828 INIT_LIST_HEAD(&msk->rtx_queue); in __mptcp_init_sock()
1829 __set_bit(MPTCP_SEND_SPACE, &msk->flags); in __mptcp_init_sock()
1830 INIT_WORK(&msk->work, mptcp_worker); in __mptcp_init_sock()
1831 msk->out_of_order_queue = RB_ROOT; in __mptcp_init_sock()
1833 msk->first = NULL; in __mptcp_init_sock()
1836 mptcp_pm_data_init(msk); in __mptcp_init_sock()
1839 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
1872 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit() local
1875 sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); in __mptcp_clear_xmit()
1877 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) in __mptcp_clear_xmit()
1883 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work() local
1885 if (cancel_work_sync(&msk->work)) in mptcp_cancel_work()
1947 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_close() local
1960 if (__mptcp_check_fallback(msk)) { in mptcp_close()
1964 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in mptcp_close()
1965 WRITE_ONCE(msk->snd_data_fin_enable, 1); in mptcp_close()
1967 mptcp_for_each_subflow(msk, subflow) { in mptcp_close()
1983 spin_lock_bh(&msk->join_list_lock); in mptcp_close()
1984 list_splice_tail_init(&msk->join_list, &msk->conn_list); in mptcp_close()
1985 spin_unlock_bh(&msk->join_list_lock); in mptcp_close()
1986 list_splice_init(&msk->conn_list, &conn_list); in mptcp_close()
2004 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
2008 struct ipv6_pinfo *msk6 = inet6_sk(msk); in mptcp_copy_inaddrs()
2010 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
2011 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
2019 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
2020 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
2021 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
2022 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
2023 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
2024 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
2052 struct mptcp_sock *msk; in mptcp_sk_clone() local
2065 msk = mptcp_sk(nsk); in mptcp_sk_clone()
2066 msk->local_key = subflow_req->local_key; in mptcp_sk_clone()
2067 msk->token = subflow_req->token; in mptcp_sk_clone()
2068 msk->subflow = NULL; in mptcp_sk_clone()
2069 WRITE_ONCE(msk->fully_established, false); in mptcp_sk_clone()
2071 msk->write_seq = subflow_req->idsn + 1; in mptcp_sk_clone()
2072 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_sk_clone()
2074 msk->can_ack = true; in mptcp_sk_clone()
2075 msk->remote_key = mp_opt->sndr_key; in mptcp_sk_clone()
2076 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); in mptcp_sk_clone()
2078 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_sk_clone()
2091 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
2095 msk->rcvq_space.copied = 0; in mptcp_rcv_space_init()
2096 msk->rcvq_space.rtt_us = 0; in mptcp_rcv_space_init()
2098 msk->rcvq_space.time = tp->tcp_mstamp; in mptcp_rcv_space_init()
2101 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, in mptcp_rcv_space_init()
2103 if (msk->rcvq_space.space == 0) in mptcp_rcv_space_init()
2104 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; in mptcp_rcv_space_init()
2110 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_accept() local
2114 listener = __mptcp_nmpc_socket(msk); in mptcp_accept()
2120 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); in mptcp_accept()
2125 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); in mptcp_accept()
2147 msk = mptcp_sk(new_mptcp_sock); in mptcp_accept()
2148 msk->first = newsk; in mptcp_accept()
2152 list_add(&subflow->node, &msk->conn_list); in mptcp_accept()
2154 mptcp_rcv_space_init(msk, ssk); in mptcp_accept()
2167 void mptcp_destroy_common(struct mptcp_sock *msk) in mptcp_destroy_common() argument
2169 skb_rbtree_purge(&msk->out_of_order_queue); in mptcp_destroy_common()
2170 mptcp_token_destroy(msk); in mptcp_destroy_common()
2171 mptcp_pm_free_anno_list(msk); in mptcp_destroy_common()
2176 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy() local
2178 if (msk->cached_ext) in mptcp_destroy()
2179 __skb_ext_put(msk->cached_ext); in mptcp_destroy()
2181 mptcp_destroy_common(msk); in mptcp_destroy()
2185 static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname, in mptcp_setsockopt_sol_socket() argument
2188 struct sock *sk = (struct sock *)msk; in mptcp_setsockopt_sol_socket()
2196 ssock = __mptcp_nmpc_socket(msk); in mptcp_setsockopt_sol_socket()
2216 static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, in mptcp_setsockopt_v6() argument
2219 struct sock *sk = (struct sock *)msk; in mptcp_setsockopt_v6()
2226 ssock = __mptcp_nmpc_socket(msk); in mptcp_setsockopt_v6()
2246 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_setsockopt() local
2249 pr_debug("msk=%p", msk); in mptcp_setsockopt()
2252 return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen); in mptcp_setsockopt()
2261 ssk = __mptcp_tcp_fallback(msk); in mptcp_setsockopt()
2267 return mptcp_setsockopt_v6(msk, optname, optval, optlen); in mptcp_setsockopt()
2275 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_getsockopt() local
2278 pr_debug("msk=%p", msk); in mptcp_getsockopt()
2287 ssk = __mptcp_tcp_fallback(msk); in mptcp_getsockopt()
2315 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_release_cb() local
2318 ssk = mptcp_subflow_recv_lookup(msk); in mptcp_release_cb()
2319 if (!ssk || !schedule_work(&msk->work)) in mptcp_release_cb()
2345 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port() local
2348 ssock = __mptcp_nmpc_socket(msk); in mptcp_get_port()
2349 pr_debug("msk=%p, subflow=%p", msk, ssock); in mptcp_get_port()
2359 struct mptcp_sock *msk; in mptcp_finish_connect() local
2365 msk = mptcp_sk(sk); in mptcp_finish_connect()
2377 WRITE_ONCE(msk->remote_key, subflow->remote_key); in mptcp_finish_connect()
2378 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
2379 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); in mptcp_finish_connect()
2380 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_finish_connect()
2381 WRITE_ONCE(msk->can_ack, 1); in mptcp_finish_connect()
2382 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_finish_connect()
2384 mptcp_pm_new_connection(msk, 0); in mptcp_finish_connect()
2386 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
2401 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join() local
2402 struct sock *parent = (void *)msk; in mptcp_finish_join()
2406 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
2412 if (!msk->pm.server_side) in mptcp_finish_join()
2415 if (!mptcp_pm_allow_new_subflow(msk)) in mptcp_finish_join()
2423 spin_lock_bh(&msk->join_list_lock); in mptcp_finish_join()
2426 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
2427 spin_unlock_bh(&msk->join_list_lock); in mptcp_finish_join()
2437 subflow->map_seq = READ_ONCE(msk->ack_seq); in mptcp_finish_join()
2443 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_memory_free() local
2445 return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; in mptcp_memory_free()
2479 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind() local
2484 ssock = __mptcp_nmpc_socket(msk); in mptcp_bind()
2499 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, in mptcp_subflow_early_fallback() argument
2503 __mptcp_do_fallback(msk); in mptcp_subflow_early_fallback()
2509 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_connect() local
2515 if (sock->state != SS_UNCONNECTED && msk->subflow) { in mptcp_stream_connect()
2519 ssock = msk->subflow; in mptcp_stream_connect()
2523 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_connect()
2529 mptcp_token_destroy(msk); in mptcp_stream_connect()
2537 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
2540 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
2561 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen() local
2565 pr_debug("msk=%p", msk); in mptcp_listen()
2568 ssock = __mptcp_nmpc_socket(msk); in mptcp_listen()
2574 mptcp_token_destroy(msk); in mptcp_listen()
2591 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept() local
2595 pr_debug("msk=%p", msk); in mptcp_stream_accept()
2601 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_accept()
2605 clear_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
2611 struct mptcp_sock *msk = mptcp_sk(newsock->sk); in mptcp_stream_accept() local
2617 __mptcp_flush_join_list(msk); in mptcp_stream_accept()
2618 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
2627 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
2636 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) in mptcp_check_readable() argument
2638 return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : in mptcp_check_readable()
2646 struct mptcp_sock *msk; in mptcp_poll() local
2650 msk = mptcp_sk(sk); in mptcp_poll()
2654 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); in mptcp_poll()
2656 return mptcp_check_readable(msk); in mptcp_poll()
2659 mask |= mptcp_check_readable(msk); in mptcp_poll()
2660 if (test_bit(MPTCP_SEND_SPACE, &msk->flags)) in mptcp_poll()
2671 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_shutdown() local
2675 pr_debug("sk=%p, how=%d", msk, how); in mptcp_shutdown()
2694 if (__mptcp_check_fallback(msk)) { in mptcp_shutdown()
2698 mptcp_for_each_subflow(msk, subflow) { in mptcp_shutdown()
2708 __mptcp_flush_join_list(msk); in mptcp_shutdown()
2710 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in mptcp_shutdown()
2711 WRITE_ONCE(msk->snd_data_fin_enable, 1); in mptcp_shutdown()
2713 mptcp_for_each_subflow(msk, subflow) { in mptcp_shutdown()