364 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_rcv_wake()  local
367 		clear_bit(NETLINK_S_CONGESTED, &nlk->state);  in netlink_rcv_wake()
368 	if (!test_bit(NETLINK_S_CONGESTED, &nlk->state))  in netlink_rcv_wake()
369 		wake_up_interruptible(&nlk->wait);  in netlink_rcv_wake()
396 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_sock_destruct()  local
398 	if (nlk->cb_running) {  in netlink_sock_destruct()
399 		if (nlk->cb.done)  in netlink_sock_destruct()
400 			nlk->cb.done(&nlk->cb);  in netlink_sock_destruct()
401 		module_put(nlk->cb.module);  in netlink_sock_destruct()
402 		kfree_skb(nlk->cb.skb);  in netlink_sock_destruct()
419 	struct netlink_sock *nlk = container_of(work, struct netlink_sock,  in netlink_sock_destruct_work()  local
422 	sk_free(&nlk->sk);  in netlink_sock_destruct_work()
496 	const struct netlink_sock *nlk = ptr;  in netlink_compare()  local
498 	return nlk->portid != x->portid ||  in netlink_compare()
499 	       !net_eq(sock_net(&nlk->sk), read_pnet(&x->pnet));  in netlink_compare()
643 	struct netlink_sock *nlk;  in __netlink_create()  local
653 	nlk = nlk_sk(sk);  in __netlink_create()
655 		nlk->cb_mutex = cb_mutex;  in __netlink_create()
657 		nlk->cb_mutex = &nlk->cb_def_mutex;  in __netlink_create()
658 		mutex_init(nlk->cb_mutex);  in __netlink_create()
659 		lockdep_set_class_and_name(nlk->cb_mutex,  in __netlink_create()
663 	init_waitqueue_head(&nlk->wait);  in __netlink_create()
675 	struct netlink_sock *nlk;  in netlink_create()  local
718 	nlk = nlk_sk(sock->sk);  in netlink_create()
719 	nlk->module = module;  in netlink_create()
720 	nlk->netlink_bind = bind;  in netlink_create()
721 	nlk->netlink_unbind = unbind;  in netlink_create()
722 	nlk->netlink_release = release;  in netlink_create()
733 	struct netlink_sock *nlk = container_of(head, struct netlink_sock, rcu);  in deferred_put_nlk_sk()  local
734 	struct sock *sk = &nlk->sk;  in deferred_put_nlk_sk()
736 	kfree(nlk->groups);  in deferred_put_nlk_sk()
737 	nlk->groups = NULL;  in deferred_put_nlk_sk()
742 	if (nlk->cb_running && nlk->cb.done) {  in deferred_put_nlk_sk()
743 		INIT_WORK(&nlk->work, netlink_sock_destruct_work);  in deferred_put_nlk_sk()
744 		schedule_work(&nlk->work);  in deferred_put_nlk_sk()
754 	struct netlink_sock *nlk;  in netlink_release()  local
761 	nlk = nlk_sk(sk);  in netlink_release()
767 	if (nlk->netlink_release)  in netlink_release()
768 		nlk->netlink_release(sk, nlk->groups);  in netlink_release()
773 	if (nlk->netlink_unbind) {  in netlink_release()
776 		for (i = 0; i < nlk->ngroups; i++)  in netlink_release()
777 			if (test_bit(i, nlk->groups))  in netlink_release()
778 				nlk->netlink_unbind(sock_net(sk), i + 1);  in netlink_release()
785 	wake_up_interruptible_all(&nlk->wait);  in netlink_release()
789 	if (nlk->portid && nlk->bound) {  in netlink_release()
793 						.portid = nlk->portid,  in netlink_release()
799 	module_put(nlk->module);  in netlink_release()
831 	call_rcu(&nlk->rcu, deferred_put_nlk_sk);  in netlink_release()
948 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_update_subscriptions()  local
950 	if (nlk->subscriptions && !subscriptions)  in netlink_update_subscriptions()
952 	else if (!nlk->subscriptions && subscriptions)  in netlink_update_subscriptions()
954 	nlk->subscriptions = subscriptions;  in netlink_update_subscriptions()
959 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_realloc_groups()  local
972 	if (nlk->ngroups >= groups)  in netlink_realloc_groups()
975 	new_groups = krealloc(nlk->groups, NLGRPSZ(groups), GFP_ATOMIC);  in netlink_realloc_groups()
980 	memset((char *)new_groups + NLGRPSZ(nlk->ngroups), 0,  in netlink_realloc_groups()
981 	       NLGRPSZ(groups) - NLGRPSZ(nlk->ngroups));  in netlink_realloc_groups()
983 	nlk->groups = new_groups;  in netlink_realloc_groups()
984 	nlk->ngroups = groups;  in netlink_realloc_groups()
993 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_undo_bind()  local
996 	if (!nlk->netlink_unbind)  in netlink_undo_bind()
1001 			nlk->netlink_unbind(sock_net(sk), undo + 1);  in netlink_undo_bind()
1009 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_bind()  local
1031 	if (nlk->ngroups < BITS_PER_LONG)  in netlink_bind()
1032 		groups &= (1UL << nlk->ngroups) - 1;  in netlink_bind()
1035 	bound = READ_ONCE(nlk->bound);  in netlink_bind()
1040 		if (nladdr->nl_pid != nlk->portid)  in netlink_bind()
1044 	if (nlk->netlink_bind && groups) {  in netlink_bind()
1051 			err = nlk->netlink_bind(net, group + 1);  in netlink_bind()
1073 	if (!groups && (nlk->groups == NULL || !(u32)nlk->groups[0]))  in netlink_bind()
1078 	netlink_update_subscriptions(sk, nlk->subscriptions +  in netlink_bind()
1080 					 hweight32(nlk->groups[0]));  in netlink_bind()
1081 	nlk->groups[0] = (nlk->groups[0] & ~0xffffffffUL) | groups;  in netlink_bind()
1097 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_connect()  local
1107 		WRITE_ONCE(nlk->dst_portid, 0);  in netlink_connect()
1108 		WRITE_ONCE(nlk->dst_group, 0);  in netlink_connect()
1125 	if (!READ_ONCE(nlk->bound))  in netlink_connect()
1132 		WRITE_ONCE(nlk->dst_portid, nladdr->nl_pid);  in netlink_connect()
1133 		WRITE_ONCE(nlk->dst_group, ffs(nladdr->nl_groups));  in netlink_connect()
1143 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_getname()  local
1151 		nladdr->nl_pid = READ_ONCE(nlk->dst_portid);  in netlink_getname()
1152 		nladdr->nl_groups = netlink_group_mask(READ_ONCE(nlk->dst_group));  in netlink_getname()
1155 		nladdr->nl_pid = READ_ONCE(nlk->portid);  in netlink_getname()
1157 		nladdr->nl_groups = nlk->groups ? nlk->groups[0] : 0;  in netlink_getname()
1174 	struct netlink_sock *nlk;  in netlink_getsockbyportid()  local
1181 	nlk = nlk_sk(sock);  in netlink_getsockbyportid()
1184 	    READ_ONCE(nlk->dst_portid) != nlk_sk(ssk)->portid) {  in netlink_getsockbyportid()
1245 	struct netlink_sock *nlk;  in netlink_attachskb()  local
1247 	nlk = nlk_sk(sk);  in netlink_attachskb()
1250 	     test_bit(NETLINK_S_CONGESTED, &nlk->state))) {  in netlink_attachskb()
1261 		add_wait_queue(&nlk->wait, &wait);  in netlink_attachskb()
1264 		     test_bit(NETLINK_S_CONGESTED, &nlk->state)) &&  in netlink_attachskb()
1269 		remove_wait_queue(&nlk->wait, &wait);  in netlink_attachskb()
1334 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_unicast_kernel()  local
1337 	if (nlk->netlink_rcv != NULL) {  in netlink_unicast_kernel()
1342 		nlk->netlink_rcv(skb);  in netlink_unicast_kernel()
1414 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_broadcast_deliver()  local
1417 	    !test_bit(NETLINK_S_CONGESTED, &nlk->state)) {  in netlink_broadcast_deliver()
1443 	struct netlink_sock *nlk = nlk_sk(sk);  in do_one_broadcast()  local
1449 	if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||  in do_one_broadcast()
1450 	    !test_bit(p->group - 1, nlk->groups))  in do_one_broadcast()
1590 	struct netlink_sock *nlk = nlk_sk(sk);  in do_one_set_err()  local
1599 	if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||  in do_one_set_err()
1600 	    !test_bit(p->group - 1, nlk->groups))  in do_one_set_err()
1648 static void netlink_update_socket_mc(struct netlink_sock *nlk,  in netlink_update_socket_mc()  argument
1654 	old = test_bit(group - 1, nlk->groups);  in netlink_update_socket_mc()
1655 	subscriptions = nlk->subscriptions - old + new;  in netlink_update_socket_mc()
1656 	__assign_bit(group - 1, nlk->groups, new);  in netlink_update_socket_mc()
1657 	netlink_update_subscriptions(&nlk->sk, subscriptions);  in netlink_update_socket_mc()
1658 	netlink_update_listeners(&nlk->sk);  in netlink_update_socket_mc()
1665 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_setsockopt()  local
1689 		if (!val || val - 1 >= nlk->ngroups)  in netlink_setsockopt()
1691 		if (optname == NETLINK_ADD_MEMBERSHIP && nlk->netlink_bind) {  in netlink_setsockopt()
1692 			err = nlk->netlink_bind(sock_net(sk), val);  in netlink_setsockopt()
1697 		netlink_update_socket_mc(nlk, val,  in netlink_setsockopt()
1700 		if (optname == NETLINK_DROP_MEMBERSHIP && nlk->netlink_unbind)  in netlink_setsockopt()
1701 			nlk->netlink_unbind(sock_net(sk), val);  in netlink_setsockopt()
1709 		assign_bit(NETLINK_F_RECV_NO_ENOBUFS, &nlk->flags, val);  in netlink_setsockopt()
1711 			clear_bit(NETLINK_S_CONGESTED, &nlk->state);  in netlink_setsockopt()
1712 			wake_up_interruptible(&nlk->wait);  in netlink_setsockopt()
1733 		assign_bit(nr, &nlk->flags, val);  in netlink_setsockopt()
1741 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_getsockopt()  local
1767 		for (pos = 0; pos * 8 < nlk->ngroups; pos += sizeof(u32)) {  in netlink_getsockopt()
1773 			if (put_user((u32)(nlk->groups[idx] >> shift),  in netlink_getsockopt()
1779 		if (put_user(ALIGN(BITS_TO_BYTES(nlk->ngroups), sizeof(u32)), optlen))  in netlink_getsockopt()
1801 	val = test_bit(flag, &nlk->flags);  in netlink_getsockopt()
1831 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_sendmsg()  local
1867 		dst_portid = READ_ONCE(nlk->dst_portid);  in netlink_sendmsg()
1868 		dst_group = READ_ONCE(nlk->dst_group);  in netlink_sendmsg()
1872 	if (!READ_ONCE(nlk->bound)) {  in netlink_sendmsg()
1889 	NETLINK_CB(skb).portid	= nlk->portid;  in netlink_sendmsg()
1922 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_recvmsg()  local
1956 	max_recvmsg_len = max(READ_ONCE(nlk->max_recvmsg_len), len);  in netlink_recvmsg()
1959 	WRITE_ONCE(nlk->max_recvmsg_len, max_recvmsg_len);  in netlink_recvmsg()
1990 	if (READ_ONCE(nlk->cb_running) &&  in netlink_recvmsg()
2022 	struct netlink_sock *nlk;  in __netlink_kernel_create()  local
2056 	nlk = nlk_sk(sk);  in __netlink_kernel_create()
2057 	set_bit(NETLINK_F_KERNEL_SOCKET, &nlk->flags);  in __netlink_kernel_create()
2178 static int netlink_dump_done(struct netlink_sock *nlk, struct sk_buff *skb,  in netlink_dump_done()  argument
2184 	nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, sizeof(nlk->dump_done_errno),  in netlink_dump_done()
2190 	memcpy(nlmsg_data(nlh), &nlk->dump_done_errno, sizeof(nlk->dump_done_errno));  in netlink_dump_done()
2192 	if (extack->_msg && test_bit(NETLINK_F_EXT_ACK, &nlk->flags)) {  in netlink_dump_done()
2203 	struct netlink_sock *nlk = nlk_sk(sk);  in netlink_dump()  local
2213 	mutex_lock(nlk->cb_mutex);  in netlink_dump()
2214 	if (!nlk->cb_running) {  in netlink_dump()
2227 	cb = &nlk->cb;  in netlink_dump()
2230 	max_recvmsg_len = READ_ONCE(nlk->max_recvmsg_len);  in netlink_dump()
2264 	if (nlk->dump_done_errno > 0) {  in netlink_dump()
2266 		nlk->dump_done_errno = cb->dump(skb, cb);  in netlink_dump()
2270 	if (nlk->dump_done_errno > 0 ||  in netlink_dump()
2271 	    skb_tailroom(skb) < nlmsg_total_size(sizeof(nlk->dump_done_errno))) {  in netlink_dump()
2272 		mutex_unlock(nlk->cb_mutex);  in netlink_dump()
2281 	if (netlink_dump_done(nlk, skb, cb, &extack))  in netlink_dump()
2290 		if (netlink_dump_done(nlk, skb_shinfo(skb)->frag_list, cb, &extack))  in netlink_dump()
2303 	WRITE_ONCE(nlk->cb_running, false);  in netlink_dump()
2306 	mutex_unlock(nlk->cb_mutex);  in netlink_dump()
2312 	mutex_unlock(nlk->cb_mutex);  in netlink_dump()
2322 	struct netlink_sock *nlk;  in __netlink_dump_start()  local
2334 	nlk = nlk_sk(sk);  in __netlink_dump_start()
2335 	mutex_lock(nlk->cb_mutex);  in __netlink_dump_start()
2337 	if (nlk->cb_running) {  in __netlink_dump_start()
2347 	cb = &nlk->cb;  in __netlink_dump_start()
2367 	WRITE_ONCE(nlk->cb_running, true);  in __netlink_dump_start()
2368 	nlk->dump_done_errno = INT_MAX;  in __netlink_dump_start()
2370 	mutex_unlock(nlk->cb_mutex);  in __netlink_dump_start()
2388 	mutex_unlock(nlk->cb_mutex);  in __netlink_dump_start()
2396 netlink_ack_tlv_len(struct netlink_sock *nlk, int err,  in netlink_ack_tlv_len()  argument
2401 	if (!extack || !test_bit(NETLINK_F_EXT_ACK, &nlk->flags))  in netlink_ack_tlv_len()
2465 	struct netlink_sock *nlk = nlk_sk(NETLINK_CB(in_skb).sk);  in netlink_ack()  local
2473 	if (err && !test_bit(NETLINK_F_CAP_ACK, &nlk->flags))  in netlink_ack()
2478 	tlvlen = netlink_ack_tlv_len(nlk, err, extack);  in netlink_ack()
2627 	struct netlink_sock *nlk;  in __netlink_seq_next()  local
2631 			nlk = rhashtable_walk_next(&iter->hti);  in __netlink_seq_next()
2633 			if (IS_ERR(nlk)) {  in __netlink_seq_next()
2634 				if (PTR_ERR(nlk) == -EAGAIN)  in __netlink_seq_next()
2637 				return nlk;  in __netlink_seq_next()
2640 			if (nlk)  in __netlink_seq_next()
2649 	} while (sock_net(&nlk->sk) != seq_file_net(seq));  in __netlink_seq_next()
2651 	return nlk;  in __netlink_seq_next()
2696 		struct netlink_sock *nlk = nlk_sk(s);  in netlink_native_seq_show()  local
2701 			   nlk->portid,  in netlink_native_seq_show()
2702 			   nlk->groups ? (u32)nlk->groups[0] : 0,  in netlink_native_seq_show()
2705 			   READ_ONCE(nlk->cb_running),  in netlink_native_seq_show()
2867 	const struct netlink_sock *nlk = data;  in netlink_hash()  local
2870 	netlink_compare_arg_init(&arg, sock_net(&nlk->sk), nlk->portid);  in netlink_hash()