137 static void sock_map_add_link(struct sk_psock *psock,  in sock_map_add_link()  argument
143 	spin_lock_bh(&psock->link_lock);  in sock_map_add_link()
144 	list_add_tail(&link->list, &psock->link);  in sock_map_add_link()
145 	spin_unlock_bh(&psock->link_lock);  in sock_map_add_link()
149 			      struct sk_psock *psock, void *link_raw)  in sock_map_del_link()  argument
154 	spin_lock_bh(&psock->link_lock);  in sock_map_del_link()
155 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sock_map_del_link()
160 			if (psock->parser.enabled && stab->progs.skb_parser)  in sock_map_del_link()
162 			if (psock->parser.enabled && stab->progs.skb_verdict)  in sock_map_del_link()
168 	spin_unlock_bh(&psock->link_lock);  in sock_map_del_link()
172 			sk_psock_stop_strp(sk, psock);  in sock_map_del_link()
174 			sk_psock_stop_verdict(sk, psock);  in sock_map_del_link()
181 	struct sk_psock *psock = sk_psock(sk);  in sock_map_unref()  local
183 	if (likely(psock)) {  in sock_map_unref()
184 		sock_map_del_link(sk, psock, link_raw);  in sock_map_unref()
185 		sk_psock_put(sk, psock);  in sock_map_unref()
189 static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)  in sock_map_init_proto()  argument
195 		prot = tcp_bpf_get_proto(sk, psock);  in sock_map_init_proto()
199 		prot = udp_bpf_get_proto(sk, psock);  in sock_map_init_proto()
209 	sk_psock_update_proto(sk, psock, prot);  in sock_map_init_proto()
215 	struct sk_psock *psock;  in sock_map_psock_get_checked()  local
218 	psock = sk_psock(sk);  in sock_map_psock_get_checked()
219 	if (psock) {  in sock_map_psock_get_checked()
221 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
225 		if (!refcount_inc_not_zero(&psock->refcnt))  in sock_map_psock_get_checked()
226 			psock = ERR_PTR(-EBUSY);  in sock_map_psock_get_checked()
230 	return psock;  in sock_map_psock_get_checked()
237 	struct sk_psock *psock;  in sock_map_link()  local
265 	psock = sock_map_psock_get_checked(sk);  in sock_map_link()
266 	if (IS_ERR(psock)) {  in sock_map_link()
267 		ret = PTR_ERR(psock);  in sock_map_link()
271 	if (psock) {  in sock_map_link()
272 		if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||  in sock_map_link()
273 		    (skb_parser  && READ_ONCE(psock->progs.skb_parser)) ||  in sock_map_link()
274 		    (skb_verdict && READ_ONCE(psock->progs.skb_verdict))) {  in sock_map_link()
275 			sk_psock_put(sk, psock);  in sock_map_link()
280 		psock = sk_psock_init(sk, map->numa_node);  in sock_map_link()
281 		if (IS_ERR(psock)) {  in sock_map_link()
282 			ret = PTR_ERR(psock);  in sock_map_link()
288 		psock_set_prog(&psock->progs.msg_parser, msg_parser);  in sock_map_link()
290 	ret = sock_map_init_proto(sk, psock);  in sock_map_link()
295 	if (skb_parser && skb_verdict && !psock->parser.enabled) {  in sock_map_link()
296 		ret = sk_psock_init_strp(sk, psock);  in sock_map_link()
299 		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);  in sock_map_link()
300 		psock_set_prog(&psock->progs.skb_parser, skb_parser);  in sock_map_link()
301 		sk_psock_start_strp(sk, psock);  in sock_map_link()
302 	} else if (!skb_parser && skb_verdict && !psock->parser.enabled) {  in sock_map_link()
303 		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);  in sock_map_link()
304 		sk_psock_start_verdict(sk,psock);  in sock_map_link()
311 	sk_psock_put(sk, psock);  in sock_map_link()
326 	struct sk_psock *psock;  in sock_map_link_no_progs()  local
329 	psock = sock_map_psock_get_checked(sk);  in sock_map_link_no_progs()
330 	if (IS_ERR(psock))  in sock_map_link_no_progs()
331 		return PTR_ERR(psock);  in sock_map_link_no_progs()
333 	if (!psock) {  in sock_map_link_no_progs()
334 		psock = sk_psock_init(sk, map->numa_node);  in sock_map_link_no_progs()
335 		if (IS_ERR(psock))  in sock_map_link_no_progs()
336 			return PTR_ERR(psock);  in sock_map_link_no_progs()
339 	ret = sock_map_init_proto(sk, psock);  in sock_map_link_no_progs()
341 		sk_psock_put(sk, psock);  in sock_map_link_no_progs()
482 	struct sk_psock *psock;  in sock_map_update_common()  local
507 	psock = sk_psock(sk);  in sock_map_update_common()
508 	WARN_ON_ONCE(!psock);  in sock_map_update_common()
520 	sock_map_add_link(psock, link, map, &stab->sks[idx]);  in sock_map_update_common()
528 	if (psock)  in sock_map_update_common()
529 		sk_psock_put(sk, psock);  in sock_map_update_common()
998 	struct sk_psock *psock;  in sock_hash_update_common()  local
1020 	psock = sk_psock(sk);  in sock_hash_update_common()
1021 	WARN_ON_ONCE(!psock);  in sock_hash_update_common()
1042 	sock_map_add_link(psock, link, map, elem_new);  in sock_hash_update_common()
1056 	sk_psock_put(sk, psock);  in sock_hash_update_common()
1515 static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)  in sock_map_remove_links()  argument
1519 	while ((link = sk_psock_link_pop(psock))) {  in sock_map_remove_links()
1528 	struct sk_psock *psock;  in sock_map_unhash()  local
1531 	psock = sk_psock(sk);  in sock_map_unhash()
1532 	if (unlikely(!psock)) {  in sock_map_unhash()
1539 	saved_unhash = psock->saved_unhash;  in sock_map_unhash()
1540 	sock_map_remove_links(sk, psock);  in sock_map_unhash()
1548 	struct sk_psock *psock;  in sock_map_close()  local
1552 	psock = sk_psock(sk);  in sock_map_close()
1553 	if (unlikely(!psock)) {  in sock_map_close()
1559 	saved_close = psock->saved_close;  in sock_map_close()
1560 	sock_map_remove_links(sk, psock);  in sock_map_close()