132 static void sock_map_add_link(struct sk_psock *psock,  in sock_map_add_link()  argument
138 	spin_lock_bh(&psock->link_lock);  in sock_map_add_link()
139 	list_add_tail(&link->list, &psock->link);  in sock_map_add_link()
140 	spin_unlock_bh(&psock->link_lock);  in sock_map_add_link()
144 			      struct sk_psock *psock, void *link_raw)  in sock_map_del_link()  argument
149 	spin_lock_bh(&psock->link_lock);  in sock_map_del_link()
150 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sock_map_del_link()
155 			if (psock->saved_data_ready && stab->progs.stream_parser)  in sock_map_del_link()
157 			if (psock->saved_data_ready && stab->progs.stream_verdict)  in sock_map_del_link()
159 			if (psock->saved_data_ready && stab->progs.skb_verdict)  in sock_map_del_link()
165 	spin_unlock_bh(&psock->link_lock);  in sock_map_del_link()
169 			sk_psock_stop_strp(sk, psock);  in sock_map_del_link()
171 			sk_psock_stop_verdict(sk, psock);  in sock_map_del_link()
178 	struct sk_psock *psock = sk_psock(sk);  in sock_map_unref()  local
180 	if (likely(psock)) {  in sock_map_unref()
181 		sock_map_del_link(sk, psock, link_raw);  in sock_map_unref()
182 		sk_psock_put(sk, psock);  in sock_map_unref()
186 static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)  in sock_map_init_proto()  argument
190 	psock->psock_update_sk_prot = sk->sk_prot->psock_update_sk_prot;  in sock_map_init_proto()
191 	return sk->sk_prot->psock_update_sk_prot(sk, psock, false);  in sock_map_init_proto()
196 	struct sk_psock *psock;  in sock_map_psock_get_checked()  local
199 	psock = sk_psock(sk);  in sock_map_psock_get_checked()
200 	if (psock) {  in sock_map_psock_get_checked()
202 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
206 		if (!refcount_inc_not_zero(&psock->refcnt))  in sock_map_psock_get_checked()
207 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
211 	return psock;  in sock_map_psock_get_checked()
221 	struct sk_psock *psock;  in sock_map_link()  local
258 	psock = sock_map_psock_get_checked(sk);  in sock_map_link()
259 	if (IS_ERR(psock)) {  in sock_map_link()
260 		ret = PTR_ERR(psock);  in sock_map_link()
264 	if (psock) {  in sock_map_link()
265 		if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||  in sock_map_link()
266 		    (stream_parser  && READ_ONCE(psock->progs.stream_parser)) ||  in sock_map_link()
267 		    (skb_verdict && READ_ONCE(psock->progs.skb_verdict)) ||  in sock_map_link()
268 		    (skb_verdict && READ_ONCE(psock->progs.stream_verdict)) ||  in sock_map_link()
269 		    (stream_verdict && READ_ONCE(psock->progs.skb_verdict)) ||  in sock_map_link()
270 		    (stream_verdict && READ_ONCE(psock->progs.stream_verdict))) {  in sock_map_link()
271 			sk_psock_put(sk, psock);  in sock_map_link()
276 		psock = sk_psock_init(sk, map->numa_node);  in sock_map_link()
277 		if (IS_ERR(psock)) {  in sock_map_link()
278 			ret = PTR_ERR(psock);  in sock_map_link()
284 		psock_set_prog(&psock->progs.msg_parser, msg_parser);  in sock_map_link()
286 	ret = sock_map_init_proto(sk, psock);  in sock_map_link()
291 	if (stream_parser && stream_verdict && !psock->saved_data_ready) {  in sock_map_link()
292 		ret = sk_psock_init_strp(sk, psock);  in sock_map_link()
295 		psock_set_prog(&psock->progs.stream_verdict, stream_verdict);  in sock_map_link()
296 		psock_set_prog(&psock->progs.stream_parser, stream_parser);  in sock_map_link()
297 		sk_psock_start_strp(sk, psock);  in sock_map_link()
298 	} else if (!stream_parser && stream_verdict && !psock->saved_data_ready) {  in sock_map_link()
299 		psock_set_prog(&psock->progs.stream_verdict, stream_verdict);  in sock_map_link()
300 		sk_psock_start_verdict(sk,psock);  in sock_map_link()
301 	} else if (!stream_verdict && skb_verdict && !psock->saved_data_ready) {  in sock_map_link()
302 		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);  in sock_map_link()
303 		sk_psock_start_verdict(sk, psock);  in sock_map_link()
310 	sk_psock_put(sk, psock);  in sock_map_link()
461 	struct sk_psock *psock;  in sock_map_update_common()  local
479 	psock = sk_psock(sk);  in sock_map_update_common()
480 	WARN_ON_ONCE(!psock);  in sock_map_update_common()
492 	sock_map_add_link(psock, link, map, &stab->sks[idx]);  in sock_map_update_common()
500 	if (psock)  in sock_map_update_common()
501 		sk_psock_put(sk, psock);  in sock_map_update_common()
963 	struct sk_psock *psock;  in sock_hash_update_common()  local
978 	psock = sk_psock(sk);  in sock_hash_update_common()
979 	WARN_ON_ONCE(!psock);  in sock_hash_update_common()
1000 	sock_map_add_link(psock, link, map, elem_new);  in sock_hash_update_common()
1014 	sk_psock_put(sk, psock);  in sock_hash_update_common()
1468 static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)  in sock_map_remove_links()  argument
1472 	while ((link = sk_psock_link_pop(psock))) {  in sock_map_remove_links()
1481 	struct sk_psock *psock;  in sock_map_unhash()  local
1484 	psock = sk_psock(sk);  in sock_map_unhash()
1485 	if (unlikely(!psock)) {  in sock_map_unhash()
1492 	saved_unhash = psock->saved_unhash;  in sock_map_unhash()
1493 	sock_map_remove_links(sk, psock);  in sock_map_unhash()
1502 	struct sk_psock *psock;  in sock_map_close()  local
1506 	psock = sk_psock_get(sk);  in sock_map_close()
1507 	if (unlikely(!psock)) {  in sock_map_close()
1513 	saved_close = psock->saved_close;  in sock_map_close()
1514 	sock_map_remove_links(sk, psock);  in sock_map_close()
1516 	sk_psock_stop(psock, true);  in sock_map_close()
1517 	sk_psock_put(sk, psock);  in sock_map_close()