Lines Matching refs:psock
403 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, in sk_msg_recvmsg() argument
411 msg_rx = sk_psock_peek_msg(psock); in sk_msg_recvmsg()
459 msg_rx = sk_psock_next_msg(psock, msg_rx); in sk_msg_recvmsg()
467 msg_rx = sk_psock_dequeue_msg(psock); in sk_msg_recvmsg()
470 msg_rx = sk_psock_peek_msg(psock); in sk_msg_recvmsg()
479 struct sk_psock *psock; in sk_msg_is_readable() local
483 psock = sk_psock(sk); in sk_msg_is_readable()
484 if (likely(psock)) in sk_msg_is_readable()
485 empty = list_empty(&psock->ingress_msg); in sk_msg_is_readable()
511 struct sk_psock *psock, in sk_psock_skb_ingress_enqueue() argument
534 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress_enqueue()
535 sk_psock_data_ready(sk, psock); in sk_psock_skb_ingress_enqueue()
539 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
541 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress() argument
543 struct sock *sk = psock->sk; in sk_psock_skb_ingress()
552 return sk_psock_skb_ingress_self(psock, skb); in sk_psock_skb_ingress()
564 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress()
574 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress_self() argument
577 struct sock *sk = psock->sk; in sk_psock_skb_ingress_self()
584 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress_self()
590 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_handle_skb() argument
594 if (!sock_writeable(psock->sk)) in sk_psock_handle_skb()
596 return skb_send_sock(psock->sk, skb, off, len); in sk_psock_handle_skb()
598 return sk_psock_skb_ingress(psock, skb); in sk_psock_handle_skb()
601 static void sk_psock_skb_state(struct sk_psock *psock, in sk_psock_skb_state() argument
606 spin_lock_bh(&psock->ingress_lock); in sk_psock_skb_state()
607 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_skb_state()
612 sock_drop(psock->sk, skb); in sk_psock_skb_state()
614 spin_unlock_bh(&psock->ingress_lock); in sk_psock_skb_state()
619 struct sk_psock *psock = container_of(work, struct sk_psock, work); in sk_psock_backlog() local
620 struct sk_psock_work_state *state = &psock->work_state; in sk_psock_backlog()
626 mutex_lock(&psock->work_mutex); in sk_psock_backlog()
628 spin_lock_bh(&psock->ingress_lock); in sk_psock_backlog()
633 spin_unlock_bh(&psock->ingress_lock); in sk_psock_backlog()
638 while ((skb = skb_dequeue(&psock->ingress_skb))) { in sk_psock_backlog()
646 if (!sock_flag(psock->sk, SOCK_DEAD)) in sk_psock_backlog()
647 ret = sk_psock_handle_skb(psock, skb, off, in sk_psock_backlog()
651 sk_psock_skb_state(psock, state, skb, in sk_psock_backlog()
656 sk_psock_report_error(psock, ret ? -ret : EPIPE); in sk_psock_backlog()
657 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_backlog()
658 sock_drop(psock->sk, skb); in sk_psock_backlog()
669 mutex_unlock(&psock->work_mutex); in sk_psock_backlog()
674 struct sk_psock *psock; in sk_psock_init() local
680 psock = ERR_PTR(-EBUSY); in sk_psock_init()
684 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node); in sk_psock_init()
685 if (!psock) { in sk_psock_init()
686 psock = ERR_PTR(-ENOMEM); in sk_psock_init()
691 psock->sk = sk; in sk_psock_init()
692 psock->eval = __SK_NONE; in sk_psock_init()
693 psock->sk_proto = prot; in sk_psock_init()
694 psock->saved_unhash = prot->unhash; in sk_psock_init()
695 psock->saved_close = prot->close; in sk_psock_init()
696 psock->saved_write_space = sk->sk_write_space; in sk_psock_init()
698 INIT_LIST_HEAD(&psock->link); in sk_psock_init()
699 spin_lock_init(&psock->link_lock); in sk_psock_init()
701 INIT_WORK(&psock->work, sk_psock_backlog); in sk_psock_init()
702 mutex_init(&psock->work_mutex); in sk_psock_init()
703 INIT_LIST_HEAD(&psock->ingress_msg); in sk_psock_init()
704 spin_lock_init(&psock->ingress_lock); in sk_psock_init()
705 skb_queue_head_init(&psock->ingress_skb); in sk_psock_init()
707 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_init()
708 refcount_set(&psock->refcnt, 1); in sk_psock_init()
710 rcu_assign_sk_user_data_nocopy(sk, psock); in sk_psock_init()
715 return psock; in sk_psock_init()
719 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock) in sk_psock_link_pop() argument
723 spin_lock_bh(&psock->link_lock); in sk_psock_link_pop()
724 link = list_first_entry_or_null(&psock->link, struct sk_psock_link, in sk_psock_link_pop()
728 spin_unlock_bh(&psock->link_lock); in sk_psock_link_pop()
732 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) in __sk_psock_purge_ingress_msg() argument
736 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
738 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
743 static void __sk_psock_zap_ingress(struct sk_psock *psock) in __sk_psock_zap_ingress() argument
747 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) { in __sk_psock_zap_ingress()
749 sock_drop(psock->sk, skb); in __sk_psock_zap_ingress()
751 kfree_skb(psock->work_state.skb); in __sk_psock_zap_ingress()
755 psock->work_state.skb = NULL; in __sk_psock_zap_ingress()
756 __sk_psock_purge_ingress_msg(psock); in __sk_psock_zap_ingress()
759 static void sk_psock_link_destroy(struct sk_psock *psock) in sk_psock_link_destroy() argument
763 list_for_each_entry_safe(link, tmp, &psock->link, list) { in sk_psock_link_destroy()
769 void sk_psock_stop(struct sk_psock *psock, bool wait) in sk_psock_stop() argument
771 spin_lock_bh(&psock->ingress_lock); in sk_psock_stop()
772 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_stop()
773 sk_psock_cork_free(psock); in sk_psock_stop()
774 __sk_psock_zap_ingress(psock); in sk_psock_stop()
775 spin_unlock_bh(&psock->ingress_lock); in sk_psock_stop()
778 cancel_work_sync(&psock->work); in sk_psock_stop()
781 static void sk_psock_done_strp(struct sk_psock *psock);
785 struct sk_psock *psock = container_of(to_rcu_work(work), in sk_psock_destroy() local
789 sk_psock_done_strp(psock); in sk_psock_destroy()
791 cancel_work_sync(&psock->work); in sk_psock_destroy()
792 mutex_destroy(&psock->work_mutex); in sk_psock_destroy()
794 psock_progs_drop(&psock->progs); in sk_psock_destroy()
796 sk_psock_link_destroy(psock); in sk_psock_destroy()
797 sk_psock_cork_free(psock); in sk_psock_destroy()
799 if (psock->sk_redir) in sk_psock_destroy()
800 sock_put(psock->sk_redir); in sk_psock_destroy()
801 sock_put(psock->sk); in sk_psock_destroy()
802 kfree(psock); in sk_psock_destroy()
805 void sk_psock_drop(struct sock *sk, struct sk_psock *psock) in sk_psock_drop() argument
808 sk_psock_restore_proto(sk, psock); in sk_psock_drop()
810 if (psock->progs.stream_parser) in sk_psock_drop()
811 sk_psock_stop_strp(sk, psock); in sk_psock_drop()
812 else if (psock->progs.stream_verdict || psock->progs.skb_verdict) in sk_psock_drop()
813 sk_psock_stop_verdict(sk, psock); in sk_psock_drop()
816 sk_psock_stop(psock, false); in sk_psock_drop()
818 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy); in sk_psock_drop()
819 queue_rcu_work(system_wq, &psock->rwork); in sk_psock_drop()
836 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, in sk_psock_msg_verdict() argument
843 prog = READ_ONCE(psock->progs.msg_parser); in sk_psock_msg_verdict()
853 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
855 if (psock->sk_redir) in sk_psock_msg_verdict()
856 sock_put(psock->sk_redir); in sk_psock_msg_verdict()
857 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()
858 if (!psock->sk_redir) { in sk_psock_msg_verdict()
862 sock_hold(psock->sk_redir); in sk_psock_msg_verdict()
921 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_tls_strp_read() argument
927 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_tls_strp_read()
929 skb->sk = psock->sk; in sk_psock_tls_strp_read()
936 sk_psock_tls_verdict_apply(skb, psock, ret); in sk_psock_tls_strp_read()
942 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_verdict_apply() argument
951 sk_other = psock->sk; in sk_psock_verdict_apply()
953 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
965 if (skb_queue_empty(&psock->ingress_skb)) { in sk_psock_verdict_apply()
966 err = sk_psock_skb_ingress_self(psock, skb); in sk_psock_verdict_apply()
969 spin_lock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
970 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
971 skb_queue_tail(&psock->ingress_skb, skb); in sk_psock_verdict_apply()
972 schedule_work(&psock->work); in sk_psock_verdict_apply()
975 spin_unlock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
983 err = sk_psock_skb_redirect(psock, skb); in sk_psock_verdict_apply()
988 sock_drop(psock->sk, skb); in sk_psock_verdict_apply()
996 struct sk_psock *psock; in sk_psock_write_space() local
1000 psock = sk_psock(sk); in sk_psock_write_space()
1001 if (likely(psock)) { in sk_psock_write_space()
1002 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_write_space()
1003 schedule_work(&psock->work); in sk_psock_write_space()
1004 write_space = psock->saved_write_space; in sk_psock_write_space()
1014 struct sk_psock *psock; in sk_psock_strp_read() local
1021 psock = sk_psock(sk); in sk_psock_strp_read()
1022 if (unlikely(!psock)) { in sk_psock_strp_read()
1026 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_strp_read()
1035 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_strp_read()
1047 struct sk_psock *psock = container_of(strp, struct sk_psock, strp); in sk_psock_strp_parse() local
1052 prog = READ_ONCE(psock->progs.stream_parser); in sk_psock_strp_parse()
1054 skb->sk = psock->sk; in sk_psock_strp_parse()
1065 struct sk_psock *psock; in sk_psock_strp_data_ready() local
1068 psock = sk_psock(sk); in sk_psock_strp_data_ready()
1069 if (likely(psock)) { in sk_psock_strp_data_ready()
1071 psock->saved_data_ready(sk); in sk_psock_strp_data_ready()
1074 strp_data_ready(&psock->strp); in sk_psock_strp_data_ready()
1081 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_init_strp() argument
1089 return strp_init(&psock->strp, sk, &cb); in sk_psock_init_strp()
1092 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_start_strp() argument
1094 if (psock->saved_data_ready) in sk_psock_start_strp()
1097 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_strp()
1102 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_strp() argument
1104 if (!psock->saved_data_ready) in sk_psock_stop_strp()
1107 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_strp()
1108 psock->saved_data_ready = NULL; in sk_psock_stop_strp()
1109 strp_stop(&psock->strp); in sk_psock_stop_strp()
1112 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1115 if (psock->progs.stream_parser) in sk_psock_done_strp()
1116 strp_done(&psock->strp); in sk_psock_done_strp()
1119 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1128 struct sk_psock *psock; in sk_psock_verdict_recv() local
1141 psock = sk_psock(sk); in sk_psock_verdict_recv()
1142 if (unlikely(!psock)) { in sk_psock_verdict_recv()
1147 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_verdict_recv()
1149 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_verdict_recv()
1158 if (sk_psock_verdict_apply(psock, skb, ret) < 0) in sk_psock_verdict_recv()
1180 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_start_verdict() argument
1182 if (psock->saved_data_ready) in sk_psock_start_verdict()
1185 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_verdict()
1190 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_verdict() argument
1192 if (!psock->saved_data_ready) in sk_psock_stop_verdict()
1195 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_verdict()
1196 psock->saved_data_ready = NULL; in sk_psock_stop_verdict()