43 static u32 sk_ehashfn(const struct sock *sk)  in sk_ehashfn()  argument
46 	if (sk->sk_family == AF_INET6 &&  in sk_ehashfn()
47 	    !ipv6_addr_v4mapped(&sk->sk_v6_daddr))  in sk_ehashfn()
48 		return inet6_ehashfn(sock_net(sk),  in sk_ehashfn()
49 				     &sk->sk_v6_rcv_saddr, sk->sk_num,  in sk_ehashfn()
50 				     &sk->sk_v6_daddr, sk->sk_dport);  in sk_ehashfn()
52 	return inet_ehashfn(sock_net(sk),  in sk_ehashfn()
53 			    sk->sk_rcv_saddr, sk->sk_num,  in sk_ehashfn()
54 			    sk->sk_daddr, sk->sk_dport);  in sk_ehashfn()
92 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,  in inet_bind_hash()  argument
95 	inet_sk(sk)->inet_num = snum;  in inet_bind_hash()
96 	sk_add_bind_node(sk, &tb->owners);  in inet_bind_hash()
97 	inet_csk(sk)->icsk_bind_hash = tb;  in inet_bind_hash()
103 static void __inet_put_port(struct sock *sk)  in __inet_put_port()  argument
105 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_put_port()
106 	const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,  in __inet_put_port()
112 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_put_port()
113 	__sk_del_bind_node(sk);  in __inet_put_port()
114 	inet_csk(sk)->icsk_bind_hash = NULL;  in __inet_put_port()
115 	inet_sk(sk)->inet_num = 0;  in __inet_put_port()
120 void inet_put_port(struct sock *sk)  in inet_put_port()  argument
123 	__inet_put_port(sk);  in inet_put_port()
128 int __inet_inherit_port(const struct sock *sk, struct sock *child)  in __inet_inherit_port()  argument
130 	struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;  in __inet_inherit_port()
132 	const int bhash = inet_bhashfn(sock_net(sk), port,  in __inet_inherit_port()
139 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_inherit_port()
145 		l3mdev = inet_sk_bound_l3mdev(sk);  in __inet_inherit_port()
153 			if (net_eq(ib_net(tb), sock_net(sk)) &&  in __inet_inherit_port()
159 						     sock_net(sk), head, port,  in __inet_inherit_port()
175 inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)  in inet_lhash2_bucket_sk()  argument
180 	if (sk->sk_family == AF_INET6)  in inet_lhash2_bucket_sk()
181 		hash = ipv6_portaddr_hash(sock_net(sk),  in inet_lhash2_bucket_sk()
182 					  &sk->sk_v6_rcv_saddr,  in inet_lhash2_bucket_sk()
183 					  inet_sk(sk)->inet_num);  in inet_lhash2_bucket_sk()
186 		hash = ipv4_portaddr_hash(sock_net(sk),  in inet_lhash2_bucket_sk()
187 					  inet_sk(sk)->inet_rcv_saddr,  in inet_lhash2_bucket_sk()
188 					  inet_sk(sk)->inet_num);  in inet_lhash2_bucket_sk()
192 static void inet_hash2(struct inet_hashinfo *h, struct sock *sk)  in inet_hash2()  argument
199 	ilb2 = inet_lhash2_bucket_sk(h, sk);  in inet_hash2()
202 	if (sk->sk_reuseport && sk->sk_family == AF_INET6)  in inet_hash2()
203 		hlist_add_tail_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,  in inet_hash2()
206 		hlist_add_head_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,  in inet_hash2()
212 static void inet_unhash2(struct inet_hashinfo *h, struct sock *sk)  in inet_unhash2()  argument
217 	    WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk)->icsk_listen_portaddr_node)))  in inet_unhash2()
220 	ilb2 = inet_lhash2_bucket_sk(h, sk);  in inet_unhash2()
223 	hlist_del_init_rcu(&inet_csk(sk)->icsk_listen_portaddr_node);  in inet_unhash2()
228 static inline int compute_score(struct sock *sk, struct net *net,  in compute_score()  argument
234 	if (net_eq(sock_net(sk), net) && sk->sk_num == hnum &&  in compute_score()
235 			!ipv6_only_sock(sk)) {  in compute_score()
236 		if (sk->sk_rcv_saddr != daddr)  in compute_score()
239 		if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))  in compute_score()
242 		score = sk->sk_family == PF_INET ? 2 : 1;  in compute_score()
243 		if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())  in compute_score()
266 	struct sock *sk, *result = NULL;  in inet_lhash2_lookup()  local
271 		sk = (struct sock *)icsk;  in inet_lhash2_lookup()
272 		score = compute_score(sk, net, hnum, daddr,  in inet_lhash2_lookup()
275 			if (sk->sk_reuseport) {  in inet_lhash2_lookup()
278 				result = reuseport_select_sock(sk, phash,  in inet_lhash2_lookup()
283 			result = sk;  in inet_lhash2_lookup()
326 void sock_gen_put(struct sock *sk)  in sock_gen_put()  argument
328 	if (!refcount_dec_and_test(&sk->sk_refcnt))  in sock_gen_put()
331 	if (sk->sk_state == TCP_TIME_WAIT)  in sock_gen_put()
332 		inet_twsk_free(inet_twsk(sk));  in sock_gen_put()
333 	else if (sk->sk_state == TCP_NEW_SYN_RECV)  in sock_gen_put()
334 		reqsk_free(inet_reqsk(sk));  in sock_gen_put()
336 		sk_free(sk);  in sock_gen_put()
342 	sock_gen_put(skb->sk);  in sock_edemux()
354 	struct sock *sk;  in __inet_lookup_established()  local
364 	sk_nulls_for_each_rcu(sk, node, &head->chain) {  in __inet_lookup_established()
365 		if (sk->sk_hash != hash)  in __inet_lookup_established()
367 		if (likely(INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
369 			if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))  in __inet_lookup_established()
371 			if (unlikely(!INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
374 				sock_gen_put(sk);  in __inet_lookup_established()
388 	sk = NULL;  in __inet_lookup_established()
390 	return sk;  in __inet_lookup_established()
396 				    struct sock *sk, __u16 lport,  in __inet_check_established()  argument
400 	struct inet_sock *inet = inet_sk(sk);  in __inet_check_established()
403 	int dif = sk->sk_bound_dev_if;  in __inet_check_established()
404 	struct net *net = sock_net(sk);  in __inet_check_established()
426 				if (twsk_unique(sk, sk2, twp))  in __inet_check_established()
438 	sk->sk_hash = hash;  in __inet_check_established()
439 	WARN_ON(!sk_unhashed(sk));  in __inet_check_established()
440 	__sk_nulls_add_node_rcu(sk, &head->chain);  in __inet_check_established()
446 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_check_established()
461 static u32 inet_sk_port_offset(const struct sock *sk)  in inet_sk_port_offset()  argument
463 	const struct inet_sock *inet = inet_sk(sk);  in inet_sk_port_offset()
473 bool inet_ehash_insert(struct sock *sk, struct sock *osk)  in inet_ehash_insert()  argument
475 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_ehash_insert()
481 	WARN_ON_ONCE(!sk_unhashed(sk));  in inet_ehash_insert()
483 	sk->sk_hash = sk_ehashfn(sk);  in inet_ehash_insert()
484 	head = inet_ehash_bucket(hashinfo, sk->sk_hash);  in inet_ehash_insert()
486 	lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_ehash_insert()
490 		WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);  in inet_ehash_insert()
494 		__sk_nulls_add_node_rcu(sk, list);  in inet_ehash_insert()
499 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk)  in inet_ehash_nolisten()  argument
501 	bool ok = inet_ehash_insert(sk, osk);  in inet_ehash_nolisten()
504 		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in inet_ehash_nolisten()
506 		percpu_counter_inc(sk->sk_prot->orphan_count);  in inet_ehash_nolisten()
507 		inet_sk_set_state(sk, TCP_CLOSE);  in inet_ehash_nolisten()
508 		sock_set_flag(sk, SOCK_DEAD);  in inet_ehash_nolisten()
509 		inet_csk_destroy_sock(sk);  in inet_ehash_nolisten()
515 static int inet_reuseport_add_sock(struct sock *sk,  in inet_reuseport_add_sock()  argument
518 	struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;  in inet_reuseport_add_sock()
520 	kuid_t uid = sock_i_uid(sk);  in inet_reuseport_add_sock()
523 		if (sk2 != sk &&  in inet_reuseport_add_sock()
524 		    sk2->sk_family == sk->sk_family &&  in inet_reuseport_add_sock()
525 		    ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&  in inet_reuseport_add_sock()
526 		    sk2->sk_bound_dev_if == sk->sk_bound_dev_if &&  in inet_reuseport_add_sock()
529 		    inet_rcv_saddr_equal(sk, sk2, false))  in inet_reuseport_add_sock()
530 			return reuseport_add_sock(sk, sk2,  in inet_reuseport_add_sock()
531 						  inet_rcv_saddr_any(sk));  in inet_reuseport_add_sock()
534 	return reuseport_alloc(sk, inet_rcv_saddr_any(sk));  in inet_reuseport_add_sock()
537 int __inet_hash(struct sock *sk, struct sock *osk)  in __inet_hash()  argument
539 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_hash()
543 	if (sk->sk_state != TCP_LISTEN) {  in __inet_hash()
544 		inet_ehash_nolisten(sk, osk);  in __inet_hash()
547 	WARN_ON(!sk_unhashed(sk));  in __inet_hash()
548 	ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];  in __inet_hash()
551 	if (sk->sk_reuseport) {  in __inet_hash()
552 		err = inet_reuseport_add_sock(sk, ilb);  in __inet_hash()
556 	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&  in __inet_hash()
557 		sk->sk_family == AF_INET6)  in __inet_hash()
558 		hlist_add_tail_rcu(&sk->sk_node, &ilb->head);  in __inet_hash()
560 		hlist_add_head_rcu(&sk->sk_node, &ilb->head);  in __inet_hash()
561 	inet_hash2(hashinfo, sk);  in __inet_hash()
563 	sock_set_flag(sk, SOCK_RCU_FREE);  in __inet_hash()
564 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_hash()
572 int inet_hash(struct sock *sk)  in inet_hash()  argument
576 	if (sk->sk_state != TCP_CLOSE) {  in inet_hash()
578 		err = __inet_hash(sk, NULL);  in inet_hash()
586 void inet_unhash(struct sock *sk)  in inet_unhash()  argument
588 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_unhash()
592 	if (sk_unhashed(sk))  in inet_unhash()
595 	if (sk->sk_state == TCP_LISTEN) {  in inet_unhash()
596 		ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];  in inet_unhash()
599 		lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_unhash()
602 	if (sk_unhashed(sk))  in inet_unhash()
605 	if (rcu_access_pointer(sk->sk_reuseport_cb))  in inet_unhash()
606 		reuseport_detach_sock(sk);  in inet_unhash()
608 		inet_unhash2(hashinfo, sk);  in inet_unhash()
609 		 __sk_del_node_init(sk);  in inet_unhash()
612 		__sk_nulls_del_node_init_rcu(sk);  in inet_unhash()
614 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);  in inet_unhash()
621 		struct sock *sk, u32 port_offset,  in __inet_hash_connect()  argument
628 	int port = inet_sk(sk)->inet_num;  in __inet_hash_connect()
629 	struct net *net = sock_net(sk);  in __inet_hash_connect()
639 		tb = inet_csk(sk)->icsk_bind_hash;  in __inet_hash_connect()
641 		if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {  in __inet_hash_connect()
642 			inet_ehash_nolisten(sk, NULL);  in __inet_hash_connect()
648 		ret = check_established(death_row, sk, port, NULL);  in __inet_hash_connect()
653 	l3mdev = inet_sk_bound_l3mdev(sk);  in __inet_hash_connect()
687 				if (!check_established(death_row, sk,  in __inet_hash_connect()
718 	inet_bind_hash(sk, tb, port);  in __inet_hash_connect()
719 	if (sk_unhashed(sk)) {  in __inet_hash_connect()
720 		inet_sk(sk)->inet_sport = htons(port);  in __inet_hash_connect()
721 		inet_ehash_nolisten(sk, (struct sock *)tw);  in __inet_hash_connect()
736 		      struct sock *sk)  in inet_hash_connect()  argument
740 	if (!inet_sk(sk)->inet_num)  in inet_hash_connect()
741 		port_offset = inet_sk_port_offset(sk);  in inet_hash_connect()
742 	return __inet_hash_connect(death_row, sk, port_offset,  in inet_hash_connect()