128 static void sock_map_add_link(struct sk_psock *psock,  in sock_map_add_link()  argument
134 	spin_lock_bh(&psock->link_lock);  in sock_map_add_link()
135 	list_add_tail(&link->list, &psock->link);  in sock_map_add_link()
136 	spin_unlock_bh(&psock->link_lock);  in sock_map_add_link()
140 			      struct sk_psock *psock, void *link_raw)  in sock_map_del_link()  argument
145 	spin_lock_bh(&psock->link_lock);  in sock_map_del_link()
146 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sock_map_del_link()
151 			if (psock->saved_data_ready && progs->stream_parser)  in sock_map_del_link()
153 			if (psock->saved_data_ready && progs->stream_verdict)  in sock_map_del_link()
155 			if (psock->saved_data_ready && progs->skb_verdict)  in sock_map_del_link()
161 	spin_unlock_bh(&psock->link_lock);  in sock_map_del_link()
165 			sk_psock_stop_strp(sk, psock);  in sock_map_del_link()
167 			sk_psock_stop_verdict(sk, psock);  in sock_map_del_link()
169 		if (psock->psock_update_sk_prot)  in sock_map_del_link()
170 			psock->psock_update_sk_prot(sk, psock, false);  in sock_map_del_link()
177 	struct sk_psock *psock = sk_psock(sk);  in sock_map_unref()  local
179 	if (likely(psock)) {  in sock_map_unref()
180 		sock_map_del_link(sk, psock, link_raw);  in sock_map_unref()
181 		sk_psock_put(sk, psock);  in sock_map_unref()
185 static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)  in sock_map_init_proto()  argument
189 	psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;  in sock_map_init_proto()
190 	return sk->sk_prot->psock_update_sk_prot(sk, psock, false);  in sock_map_init_proto()
195 	struct sk_psock *psock;  in sock_map_psock_get_checked()  local
198 	psock = sk_psock(sk);  in sock_map_psock_get_checked()
199 	if (psock) {  in sock_map_psock_get_checked()
201 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
205 		if (!refcount_inc_not_zero(&psock->refcnt))  in sock_map_psock_get_checked()
206 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
210 	return psock;  in sock_map_psock_get_checked()
220 	struct sk_psock *psock;  in sock_map_link()  local
257 	psock = sock_map_psock_get_checked(sk);  in sock_map_link()
258 	if (IS_ERR(psock)) {  in sock_map_link()
259 		ret = PTR_ERR(psock);  in sock_map_link()
263 	if (psock) {  in sock_map_link()
264 		if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||  in sock_map_link()
265 		    (stream_parser  && READ_ONCE(psock->progs.stream_parser)) ||  in sock_map_link()
266 		    (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||  in sock_map_link()
267 		    (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||  in sock_map_link()
268 		    (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||  in sock_map_link()
269 		    (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {  in sock_map_link()
270 			sk_psock_put(sk, psock);  in sock_map_link()
275 		psock = sk_psock_init(sk, map->numa_node);  in sock_map_link()
276 		if (IS_ERR(psock)) {  in sock_map_link()
277 			ret = PTR_ERR(psock);  in sock_map_link()
283 		psock_set_prog(&psock->progs.msg_parser, msg_parser);  in sock_map_link()
285 		psock_set_prog(&psock->progs.stream_parser, stream_parser);  in sock_map_link()
287 		psock_set_prog(&psock->progs.stream_verdict, stream_verdict);  in sock_map_link()
289 		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);  in sock_map_link()
294 	ret = sock_map_init_proto(sk, psock);  in sock_map_link()
296 		sk_psock_put(sk, psock);  in sock_map_link()
301 	if (stream_parser && stream_verdict && !psock->saved_data_ready) {  in sock_map_link()
302 		ret = sk_psock_init_strp(sk, psock);  in sock_map_link()
305 			sk_psock_put(sk, psock);  in sock_map_link()
308 		sk_psock_start_strp(sk, psock);  in sock_map_link()
309 	} else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {  in sock_map_link()
310 		sk_psock_start_verdict(sk,psock);  in sock_map_link()
311 	} else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {  in sock_map_link()
312 		sk_psock_start_verdict(sk, psock);  in sock_map_link()
469 	struct sk_psock *psock;  in sock_map_update_common()  local
487 	psock = sk_psock(sk);  in sock_map_update_common()
488 	WARN_ON_ONCE(!psock);  in sock_map_update_common()
500 	sock_map_add_link(psock, link, map, &stab->sks[idx]);  in sock_map_update_common()
508 	if (psock)  in sock_map_update_common()
509 		sk_psock_put(sk, psock);  in sock_map_update_common()
984 	struct sk_psock *psock;  in sock_hash_update_common()  local
999 	psock = sk_psock(sk);  in sock_hash_update_common()
1000 	WARN_ON_ONCE(!psock);  in sock_hash_update_common()
1021 	sock_map_add_link(psock, link, map, elem_new);  in sock_hash_update_common()
1035 	sk_psock_put(sk, psock);  in sock_hash_update_common()
1571 static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)  in sock_map_remove_links()  argument
1575 	while ((link = sk_psock_link_pop(psock))) {  in sock_map_remove_links()
1584 	struct sk_psock *psock;  in sock_map_unhash()  local
1587 	psock = sk_psock(sk);  in sock_map_unhash()
1588 	if (unlikely(!psock)) {  in sock_map_unhash()
1592 		saved_unhash = psock->saved_unhash;  in sock_map_unhash()
1593 		sock_map_remove_links(sk, psock);  in sock_map_unhash()
1606 	struct sk_psock *psock;  in sock_map_destroy()  local
1609 	psock = sk_psock_get(sk);  in sock_map_destroy()
1610 	if (unlikely(!psock)) {  in sock_map_destroy()
1614 		saved_destroy = psock->saved_destroy;  in sock_map_destroy()
1615 		sock_map_remove_links(sk, psock);  in sock_map_destroy()
1617 		sk_psock_stop(psock);  in sock_map_destroy()
1618 		sk_psock_put(sk, psock);  in sock_map_destroy()
1630 	struct sk_psock *psock;  in sock_map_close()  local
1634 	psock = sk_psock_get(sk);  in sock_map_close()
1635 	if (unlikely(!psock)) {  in sock_map_close()
1640 		saved_close = psock->saved_close;  in sock_map_close()
1641 		sock_map_remove_links(sk, psock);  in sock_map_close()
1643 		sk_psock_stop(psock);  in sock_map_close()
1645 		cancel_delayed_work_sync(&psock->work);  in sock_map_close()
1646 		sk_psock_put(sk, psock);  in sock_map_close()