47 static u32 sk_ehashfn(const struct sock *sk)  in sk_ehashfn()  argument
50 	if (sk->sk_family == AF_INET6 &&  in sk_ehashfn()
51 	    !ipv6_addr_v4mapped(&sk->sk_v6_daddr))  in sk_ehashfn()
52 		return inet6_ehashfn(sock_net(sk),  in sk_ehashfn()
53 				     &sk->sk_v6_rcv_saddr, sk->sk_num,  in sk_ehashfn()
54 				     &sk->sk_v6_daddr, sk->sk_dport);  in sk_ehashfn()
56 	return inet_ehashfn(sock_net(sk),  in sk_ehashfn()
57 			    sk->sk_rcv_saddr, sk->sk_num,  in sk_ehashfn()
58 			    sk->sk_daddr, sk->sk_dport);  in sk_ehashfn()
94 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,  in inet_bind_hash()  argument
97 	inet_sk(sk)->inet_num = snum;  in inet_bind_hash()
98 	sk_add_bind_node(sk, &tb->owners);  in inet_bind_hash()
99 	inet_csk(sk)->icsk_bind_hash = tb;  in inet_bind_hash()
105 static void __inet_put_port(struct sock *sk)  in __inet_put_port()  argument
107 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_put_port()
108 	const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,  in __inet_put_port()
114 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_put_port()
115 	__sk_del_bind_node(sk);  in __inet_put_port()
116 	inet_csk(sk)->icsk_bind_hash = NULL;  in __inet_put_port()
117 	inet_sk(sk)->inet_num = 0;  in __inet_put_port()
122 void inet_put_port(struct sock *sk)  in inet_put_port()  argument
125 	__inet_put_port(sk);  in inet_put_port()
130 int __inet_inherit_port(const struct sock *sk, struct sock *child)  in __inet_inherit_port()  argument
132 	struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;  in __inet_inherit_port()
134 	const int bhash = inet_bhashfn(sock_net(sk), port,  in __inet_inherit_port()
140 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_inherit_port()
152 			if (net_eq(ib_net(tb), sock_net(sk)) &&  in __inet_inherit_port()
158 						     sock_net(sk), head, port);  in __inet_inherit_port()
173 inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)  in inet_lhash2_bucket_sk()  argument
178 	if (sk->sk_family == AF_INET6)  in inet_lhash2_bucket_sk()
179 		hash = ipv6_portaddr_hash(sock_net(sk),  in inet_lhash2_bucket_sk()
180 					  &sk->sk_v6_rcv_saddr,  in inet_lhash2_bucket_sk()
181 					  inet_sk(sk)->inet_num);  in inet_lhash2_bucket_sk()
184 		hash = ipv4_portaddr_hash(sock_net(sk),  in inet_lhash2_bucket_sk()
185 					  inet_sk(sk)->inet_rcv_saddr,  in inet_lhash2_bucket_sk()
186 					  inet_sk(sk)->inet_num);  in inet_lhash2_bucket_sk()
190 static void inet_hash2(struct inet_hashinfo *h, struct sock *sk)  in inet_hash2()  argument
197 	ilb2 = inet_lhash2_bucket_sk(h, sk);  in inet_hash2()
200 	if (sk->sk_reuseport && sk->sk_family == AF_INET6)  in inet_hash2()
201 		hlist_add_tail_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,  in inet_hash2()
204 		hlist_add_head_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,  in inet_hash2()
210 static void inet_unhash2(struct inet_hashinfo *h, struct sock *sk)  in inet_unhash2()  argument
215 	    WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk)->icsk_listen_portaddr_node)))  in inet_unhash2()
218 	ilb2 = inet_lhash2_bucket_sk(h, sk);  in inet_unhash2()
221 	hlist_del_init_rcu(&inet_csk(sk)->icsk_listen_portaddr_node);  in inet_unhash2()
226 static inline int compute_score(struct sock *sk, struct net *net,  in compute_score()  argument
231 	struct inet_sock *inet = inet_sk(sk);  in compute_score()
233 	if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&  in compute_score()
234 			!ipv6_only_sock(sk)) {  in compute_score()
236 		score = sk->sk_family == PF_INET ? 2 : 1;  in compute_score()
242 		if (sk->sk_bound_dev_if || exact_dif) {  in compute_score()
243 			bool dev_match = (sk->sk_bound_dev_if == dif ||  in compute_score()
244 					  sk->sk_bound_dev_if == sdif);  in compute_score()
248 			if (sk->sk_bound_dev_if)  in compute_score()
251 		if (sk->sk_incoming_cpu == raw_smp_processor_id())  in compute_score()
274 	struct sock *sk, *result = NULL;  in inet_lhash2_lookup()  local
279 		sk = (struct sock *)icsk;  in inet_lhash2_lookup()
280 		score = compute_score(sk, net, hnum, daddr,  in inet_lhash2_lookup()
283 			if (sk->sk_reuseport) {  in inet_lhash2_lookup()
286 				result = reuseport_select_sock(sk, phash,  in inet_lhash2_lookup()
291 			result = sk;  in inet_lhash2_lookup()
310 	struct sock *sk, *result = NULL;  in __inet_lookup_listener()  local
346 	sk_for_each_rcu(sk, &ilb->head) {  in __inet_lookup_listener()
347 		score = compute_score(sk, net, hnum, daddr,  in __inet_lookup_listener()
350 			if (sk->sk_reuseport) {  in __inet_lookup_listener()
353 				result = reuseport_select_sock(sk, phash,  in __inet_lookup_listener()
358 			result = sk;  in __inet_lookup_listener()
370 void sock_gen_put(struct sock *sk)  in sock_gen_put()  argument
372 	if (!refcount_dec_and_test(&sk->sk_refcnt))  in sock_gen_put()
375 	if (sk->sk_state == TCP_TIME_WAIT)  in sock_gen_put()
376 		inet_twsk_free(inet_twsk(sk));  in sock_gen_put()
377 	else if (sk->sk_state == TCP_NEW_SYN_RECV)  in sock_gen_put()
378 		reqsk_free(inet_reqsk(sk));  in sock_gen_put()
380 		sk_free(sk);  in sock_gen_put()
386 	sock_gen_put(skb->sk);  in sock_edemux()
398 	struct sock *sk;  in __inet_lookup_established()  local
408 	sk_nulls_for_each_rcu(sk, node, &head->chain) {  in __inet_lookup_established()
409 		if (sk->sk_hash != hash)  in __inet_lookup_established()
411 		if (likely(INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
413 			if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))  in __inet_lookup_established()
415 			if (unlikely(!INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
418 				sock_gen_put(sk);  in __inet_lookup_established()
432 	sk = NULL;  in __inet_lookup_established()
434 	return sk;  in __inet_lookup_established()
440 				    struct sock *sk, __u16 lport,  in __inet_check_established()  argument
444 	struct inet_sock *inet = inet_sk(sk);  in __inet_check_established()
447 	int dif = sk->sk_bound_dev_if;  in __inet_check_established()
448 	struct net *net = sock_net(sk);  in __inet_check_established()
470 				if (twsk_unique(sk, sk2, twp))  in __inet_check_established()
482 	sk->sk_hash = hash;  in __inet_check_established()
483 	WARN_ON(!sk_unhashed(sk));  in __inet_check_established()
484 	__sk_nulls_add_node_rcu(sk, &head->chain);  in __inet_check_established()
490 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_check_established()
505 static u32 inet_sk_port_offset(const struct sock *sk)  in inet_sk_port_offset()  argument
507 	const struct inet_sock *inet = inet_sk(sk);  in inet_sk_port_offset()
517 bool inet_ehash_insert(struct sock *sk, struct sock *osk)  in inet_ehash_insert()  argument
519 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_ehash_insert()
525 	WARN_ON_ONCE(!sk_unhashed(sk));  in inet_ehash_insert()
527 	sk->sk_hash = sk_ehashfn(sk);  in inet_ehash_insert()
528 	head = inet_ehash_bucket(hashinfo, sk->sk_hash);  in inet_ehash_insert()
530 	lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_ehash_insert()
534 		WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);  in inet_ehash_insert()
538 		__sk_nulls_add_node_rcu(sk, list);  in inet_ehash_insert()
543 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk)  in inet_ehash_nolisten()  argument
545 	bool ok = inet_ehash_insert(sk, osk);  in inet_ehash_nolisten()
548 		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in inet_ehash_nolisten()
550 		percpu_counter_inc(sk->sk_prot->orphan_count);  in inet_ehash_nolisten()
551 		inet_sk_set_state(sk, TCP_CLOSE);  in inet_ehash_nolisten()
552 		sock_set_flag(sk, SOCK_DEAD);  in inet_ehash_nolisten()
553 		inet_csk_destroy_sock(sk);  in inet_ehash_nolisten()
559 static int inet_reuseport_add_sock(struct sock *sk,  in inet_reuseport_add_sock()  argument
562 	struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;  in inet_reuseport_add_sock()
564 	kuid_t uid = sock_i_uid(sk);  in inet_reuseport_add_sock()
567 		if (sk2 != sk &&  in inet_reuseport_add_sock()
568 		    sk2->sk_family == sk->sk_family &&  in inet_reuseport_add_sock()
569 		    ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&  in inet_reuseport_add_sock()
570 		    sk2->sk_bound_dev_if == sk->sk_bound_dev_if &&  in inet_reuseport_add_sock()
573 		    inet_rcv_saddr_equal(sk, sk2, false))  in inet_reuseport_add_sock()
574 			return reuseport_add_sock(sk, sk2,  in inet_reuseport_add_sock()
575 						  inet_rcv_saddr_any(sk));  in inet_reuseport_add_sock()
578 	return reuseport_alloc(sk, inet_rcv_saddr_any(sk));  in inet_reuseport_add_sock()
581 int __inet_hash(struct sock *sk, struct sock *osk)  in __inet_hash()  argument
583 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_hash()
587 	if (sk->sk_state != TCP_LISTEN) {  in __inet_hash()
588 		inet_ehash_nolisten(sk, osk);  in __inet_hash()
591 	WARN_ON(!sk_unhashed(sk));  in __inet_hash()
592 	ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];  in __inet_hash()
595 	if (sk->sk_reuseport) {  in __inet_hash()
596 		err = inet_reuseport_add_sock(sk, ilb);  in __inet_hash()
600 	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&  in __inet_hash()
601 		sk->sk_family == AF_INET6)  in __inet_hash()
602 		hlist_add_tail_rcu(&sk->sk_node, &ilb->head);  in __inet_hash()
604 		hlist_add_head_rcu(&sk->sk_node, &ilb->head);  in __inet_hash()
605 	inet_hash2(hashinfo, sk);  in __inet_hash()
607 	sock_set_flag(sk, SOCK_RCU_FREE);  in __inet_hash()
608 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_hash()
616 int inet_hash(struct sock *sk)  in inet_hash()  argument
620 	if (sk->sk_state != TCP_CLOSE) {  in inet_hash()
622 		err = __inet_hash(sk, NULL);  in inet_hash()
630 void inet_unhash(struct sock *sk)  in inet_unhash()  argument
632 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_unhash()
636 	if (sk_unhashed(sk))  in inet_unhash()
639 	if (sk->sk_state == TCP_LISTEN) {  in inet_unhash()
640 		ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];  in inet_unhash()
643 		lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_unhash()
646 	if (sk_unhashed(sk))  in inet_unhash()
649 	if (rcu_access_pointer(sk->sk_reuseport_cb))  in inet_unhash()
650 		reuseport_detach_sock(sk);  in inet_unhash()
652 		inet_unhash2(hashinfo, sk);  in inet_unhash()
653 		 __sk_del_node_init(sk);  in inet_unhash()
656 		__sk_nulls_del_node_init_rcu(sk);  in inet_unhash()
658 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);  in inet_unhash()
665 		struct sock *sk, u32 port_offset,  in __inet_hash_connect()  argument
672 	int port = inet_sk(sk)->inet_num;  in __inet_hash_connect()
673 	struct net *net = sock_net(sk);  in __inet_hash_connect()
682 		tb = inet_csk(sk)->icsk_bind_hash;  in __inet_hash_connect()
684 		if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {  in __inet_hash_connect()
685 			inet_ehash_nolisten(sk, NULL);  in __inet_hash_connect()
691 		ret = check_established(death_row, sk, port, NULL);  in __inet_hash_connect()
727 				if (!check_established(death_row, sk,  in __inet_hash_connect()
758 	inet_bind_hash(sk, tb, port);  in __inet_hash_connect()
759 	if (sk_unhashed(sk)) {  in __inet_hash_connect()
760 		inet_sk(sk)->inet_sport = htons(port);  in __inet_hash_connect()
761 		inet_ehash_nolisten(sk, (struct sock *)tw);  in __inet_hash_connect()
776 		      struct sock *sk)  in inet_hash_connect()  argument
780 	if (!inet_sk(sk)->inet_num)  in inet_hash_connect()
781 		port_offset = inet_sk_port_offset(sk);  in inet_hash_connect()
782 	return __inet_hash_connect(death_row, sk, port_offset,  in inet_hash_connect()