412 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,  in sk_msg_recvmsg()  argument
420 	msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
470 			msg_rx = sk_psock_next_msg(psock, msg_rx);  in sk_msg_recvmsg()
478 			msg_rx = sk_psock_dequeue_msg(psock);  in sk_msg_recvmsg()
481 		msg_rx = sk_psock_peek_msg(psock);  in sk_msg_recvmsg()
490 	struct sk_psock *psock;  in sk_msg_is_readable()  local
494 	psock = sk_psock(sk);  in sk_msg_is_readable()
495 	if (likely(psock))  in sk_msg_is_readable()
496 		empty = list_empty(&psock->ingress_msg);  in sk_msg_is_readable()
527 					struct sk_psock *psock,  in sk_psock_skb_ingress_enqueue()  argument
554 	sk_psock_queue_msg(psock, msg);  in sk_psock_skb_ingress_enqueue()
555 	sk_psock_data_ready(sk, psock);  in sk_psock_skb_ingress_enqueue()
559 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
562 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress()  argument
565 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress()
574 		return sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_skb_ingress()
586 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress()
596 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_skb_ingress_self()  argument
600 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress_self()
606 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);  in sk_psock_skb_ingress_self()
612 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_handle_skb()  argument
618 		if (!sock_writeable(psock->sk))  in sk_psock_handle_skb()
620 		return skb_send_sock(psock->sk, skb, off, len);  in sk_psock_handle_skb()
623 	err = sk_psock_skb_ingress(psock, skb, off, len);  in sk_psock_handle_skb()
629 static void sk_psock_skb_state(struct sk_psock *psock,  in sk_psock_skb_state()  argument
633 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
634 	if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_skb_state()
638 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_skb_state()
644 	struct sk_psock *psock = container_of(dwork, struct sk_psock, work);  in sk_psock_backlog()  local
645 	struct sk_psock_work_state *state = &psock->work_state;  in sk_psock_backlog()
651 	mutex_lock(&psock->work_mutex);  in sk_psock_backlog()
657 	while ((skb = skb_peek(&psock->ingress_skb))) {  in sk_psock_backlog()
670 			if (!sock_flag(psock->sk, SOCK_DEAD))  in sk_psock_backlog()
671 				ret = sk_psock_handle_skb(psock, skb, off,  in sk_psock_backlog()
675 					sk_psock_skb_state(psock, state, len, off);  in sk_psock_backlog()
680 					if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_backlog()
681 						schedule_delayed_work(&psock->work, 1);  in sk_psock_backlog()
685 				sk_psock_report_error(psock, ret ? -ret : EPIPE);  in sk_psock_backlog()
686 				sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_backlog()
693 		skb = skb_dequeue(&psock->ingress_skb);  in sk_psock_backlog()
697 	mutex_unlock(&psock->work_mutex);  in sk_psock_backlog()
702 	struct sk_psock *psock;  in sk_psock_init()  local
708 		psock = ERR_PTR(-EINVAL);  in sk_psock_init()
713 		psock = ERR_PTR(-EBUSY);  in sk_psock_init()
717 	psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);  in sk_psock_init()
718 	if (!psock) {  in sk_psock_init()
719 		psock = ERR_PTR(-ENOMEM);  in sk_psock_init()
724 	psock->sk = sk;  in sk_psock_init()
725 	psock->eval = __SK_NONE;  in sk_psock_init()
726 	psock->sk_proto = prot;  in sk_psock_init()
727 	psock->saved_unhash = prot->unhash;  in sk_psock_init()
728 	psock->saved_destroy = prot->destroy;  in sk_psock_init()
729 	psock->saved_close = prot->close;  in sk_psock_init()
730 	psock->saved_write_space = sk->sk_write_space;  in sk_psock_init()
732 	INIT_LIST_HEAD(&psock->link);  in sk_psock_init()
733 	spin_lock_init(&psock->link_lock);  in sk_psock_init()
735 	INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);  in sk_psock_init()
736 	mutex_init(&psock->work_mutex);  in sk_psock_init()
737 	INIT_LIST_HEAD(&psock->ingress_msg);  in sk_psock_init()
738 	spin_lock_init(&psock->ingress_lock);  in sk_psock_init()
739 	skb_queue_head_init(&psock->ingress_skb);  in sk_psock_init()
741 	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_init()
742 	refcount_set(&psock->refcnt, 1);  in sk_psock_init()
744 	__rcu_assign_sk_user_data_with_flags(sk, psock,  in sk_psock_init()
751 	return psock;  in sk_psock_init()
755 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)  in sk_psock_link_pop()  argument
759 	spin_lock_bh(&psock->link_lock);  in sk_psock_link_pop()
760 	link = list_first_entry_or_null(&psock->link, struct sk_psock_link,  in sk_psock_link_pop()
764 	spin_unlock_bh(&psock->link_lock);  in sk_psock_link_pop()
768 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)  in __sk_psock_purge_ingress_msg()  argument
772 	list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {  in __sk_psock_purge_ingress_msg()
774 		sk_msg_free(psock->sk, msg);  in __sk_psock_purge_ingress_msg()
779 static void __sk_psock_zap_ingress(struct sk_psock *psock)  in __sk_psock_zap_ingress()  argument
783 	while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {  in __sk_psock_zap_ingress()
785 		sock_drop(psock->sk, skb);  in __sk_psock_zap_ingress()
787 	__sk_psock_purge_ingress_msg(psock);  in __sk_psock_zap_ingress()
790 static void sk_psock_link_destroy(struct sk_psock *psock)  in sk_psock_link_destroy()  argument
794 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sk_psock_link_destroy()
800 void sk_psock_stop(struct sk_psock *psock)  in sk_psock_stop()  argument
802 	spin_lock_bh(&psock->ingress_lock);  in sk_psock_stop()
803 	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_stop()
804 	sk_psock_cork_free(psock);  in sk_psock_stop()
805 	spin_unlock_bh(&psock->ingress_lock);  in sk_psock_stop()
808 static void sk_psock_done_strp(struct sk_psock *psock);
812 	struct sk_psock *psock = container_of(to_rcu_work(work),  in sk_psock_destroy()  local
816 	sk_psock_done_strp(psock);  in sk_psock_destroy()
818 	cancel_delayed_work_sync(&psock->work);  in sk_psock_destroy()
819 	__sk_psock_zap_ingress(psock);  in sk_psock_destroy()
820 	mutex_destroy(&psock->work_mutex);  in sk_psock_destroy()
822 	psock_progs_drop(&psock->progs);  in sk_psock_destroy()
824 	sk_psock_link_destroy(psock);  in sk_psock_destroy()
825 	sk_psock_cork_free(psock);  in sk_psock_destroy()
827 	if (psock->sk_redir)  in sk_psock_destroy()
828 		sock_put(psock->sk_redir);  in sk_psock_destroy()
829 	sock_put(psock->sk);  in sk_psock_destroy()
830 	kfree(psock);  in sk_psock_destroy()
833 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)  in sk_psock_drop()  argument
836 	sk_psock_restore_proto(sk, psock);  in sk_psock_drop()
838 	if (psock->progs.stream_parser)  in sk_psock_drop()
839 		sk_psock_stop_strp(sk, psock);  in sk_psock_drop()
840 	else if (psock->progs.stream_verdict || psock->progs.skb_verdict)  in sk_psock_drop()
841 		sk_psock_stop_verdict(sk, psock);  in sk_psock_drop()
844 	sk_psock_stop(psock);  in sk_psock_drop()
846 	INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);  in sk_psock_drop()
847 	queue_rcu_work(system_wq, &psock->rwork);  in sk_psock_drop()
864 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,  in sk_psock_msg_verdict()  argument
871 	prog = READ_ONCE(psock->progs.msg_parser);  in sk_psock_msg_verdict()
881 	psock->apply_bytes = msg->apply_bytes;  in sk_psock_msg_verdict()
883 		if (psock->sk_redir) {  in sk_psock_msg_verdict()
884 			sock_put(psock->sk_redir);  in sk_psock_msg_verdict()
885 			psock->sk_redir = NULL;  in sk_psock_msg_verdict()
891 		psock->redir_ingress = sk_msg_to_ingress(msg);  in sk_psock_msg_verdict()
892 		psock->sk_redir = msg->sk_redir;  in sk_psock_msg_verdict()
893 		sock_hold(psock->sk_redir);  in sk_psock_msg_verdict()
953 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_tls_strp_read()  argument
959 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_tls_strp_read()
961 		skb->sk = psock->sk;  in sk_psock_tls_strp_read()
968 	sk_psock_tls_verdict_apply(skb, psock, ret);  in sk_psock_tls_strp_read()
974 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_verdict_apply()  argument
984 		sk_other = psock->sk;  in sk_psock_verdict_apply()
986 		    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_verdict_apply()
997 		if (skb_queue_empty(&psock->ingress_skb)) {  in sk_psock_verdict_apply()
1006 			err = sk_psock_skb_ingress_self(psock, skb, off, len);  in sk_psock_verdict_apply()
1009 			spin_lock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1010 			if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_verdict_apply()
1011 				skb_queue_tail(&psock->ingress_skb, skb);  in sk_psock_verdict_apply()
1012 				schedule_delayed_work(&psock->work, 0);  in sk_psock_verdict_apply()
1015 			spin_unlock_bh(&psock->ingress_lock);  in sk_psock_verdict_apply()
1021 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1022 		err = sk_psock_skb_redirect(psock, skb);  in sk_psock_verdict_apply()
1028 		tcp_eat_skb(psock->sk, skb);  in sk_psock_verdict_apply()
1029 		sock_drop(psock->sk, skb);  in sk_psock_verdict_apply()
1037 	struct sk_psock *psock;  in sk_psock_write_space()  local
1041 	psock = sk_psock(sk);  in sk_psock_write_space()
1042 	if (likely(psock)) {  in sk_psock_write_space()
1043 		if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_write_space()
1044 			schedule_delayed_work(&psock->work, 0);  in sk_psock_write_space()
1045 		write_space = psock->saved_write_space;  in sk_psock_write_space()
1055 	struct sk_psock *psock;  in sk_psock_strp_read()  local
1062 	psock = sk_psock(sk);  in sk_psock_strp_read()
1063 	if (unlikely(!psock)) {  in sk_psock_strp_read()
1067 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_strp_read()
1077 	sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_strp_read()
1089 	struct sk_psock *psock = container_of(strp, struct sk_psock, strp);  in sk_psock_strp_parse()  local
1094 	prog = READ_ONCE(psock->progs.stream_parser);  in sk_psock_strp_parse()
1096 		skb->sk = psock->sk;  in sk_psock_strp_parse()
1107 	struct sk_psock *psock;  in sk_psock_strp_data_ready()  local
1112 	psock = sk_psock(sk);  in sk_psock_strp_data_ready()
1113 	if (likely(psock)) {  in sk_psock_strp_data_ready()
1115 			psock->saved_data_ready(sk);  in sk_psock_strp_data_ready()
1118 			strp_data_ready(&psock->strp);  in sk_psock_strp_data_ready()
1125 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_init_strp()  argument
1135 	ret = strp_init(&psock->strp, sk, &cb);  in sk_psock_init_strp()
1137 		sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);  in sk_psock_init_strp()
1142 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_strp()  argument
1144 	if (psock->saved_data_ready)  in sk_psock_start_strp()
1147 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_strp()
1152 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_strp()  argument
1154 	psock_set_prog(&psock->progs.stream_parser, NULL);  in sk_psock_stop_strp()
1156 	if (!psock->saved_data_ready)  in sk_psock_stop_strp()
1159 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_strp()
1160 	psock->saved_data_ready = NULL;  in sk_psock_stop_strp()
1161 	strp_stop(&psock->strp);  in sk_psock_stop_strp()
1164 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1167 	if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))  in sk_psock_done_strp()
1168 		strp_done(&psock->strp);  in sk_psock_done_strp()
1171 static void sk_psock_done_strp(struct sk_psock *psock)  in sk_psock_done_strp()  argument
1178 	struct sk_psock *psock;  in sk_psock_verdict_recv()  local
1184 	psock = sk_psock(sk);  in sk_psock_verdict_recv()
1185 	if (unlikely(!psock)) {  in sk_psock_verdict_recv()
1191 	prog = READ_ONCE(psock->progs.stream_verdict);  in sk_psock_verdict_recv()
1193 		prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_verdict_recv()
1200 	ret = sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_verdict_recv()
1223 		struct sk_psock *psock;  in sk_psock_verdict_data_ready()  local
1226 		psock = sk_psock(sk);  in sk_psock_verdict_data_ready()
1227 		if (psock)  in sk_psock_verdict_data_ready()
1228 			psock->saved_data_ready(sk);  in sk_psock_verdict_data_ready()
1233 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_verdict()  argument
1235 	if (psock->saved_data_ready)  in sk_psock_start_verdict()
1238 	psock->saved_data_ready = sk->sk_data_ready;  in sk_psock_start_verdict()
1243 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_verdict()  argument
1245 	psock_set_prog(&psock->progs.stream_verdict, NULL);  in sk_psock_stop_verdict()
1246 	psock_set_prog(&psock->progs.skb_verdict, NULL);  in sk_psock_stop_verdict()
1248 	if (!psock->saved_data_ready)  in sk_psock_stop_verdict()
1251 	sk->sk_data_ready = psock->saved_data_ready;  in sk_psock_stop_verdict()
1252 	psock->saved_data_ready = NULL;  in sk_psock_stop_verdict()