19 static void remove_nexthop(struct net *net, struct nexthop *nh,
61 static void nexthop_free_mpath(struct nexthop *nh)  in nexthop_free_mpath()  argument
66 	nhg = rcu_dereference_raw(nh->nh_grp);  in nexthop_free_mpath()
68 		WARN_ON(nhg->nh_entries[i].nh);  in nexthop_free_mpath()
73 static void nexthop_free_single(struct nexthop *nh)  in nexthop_free_single()  argument
77 	nhi = rcu_dereference_raw(nh->nh_info);  in nexthop_free_single()
80 		fib_nh_release(nh->net, &nhi->fib_nh);  in nexthop_free_single()
91 	struct nexthop *nh = container_of(head, struct nexthop, rcu);  in nexthop_free_rcu()  local
93 	if (nh->is_group)  in nexthop_free_rcu()
94 		nexthop_free_mpath(nh);  in nexthop_free_rcu()
96 		nexthop_free_single(nh);  in nexthop_free_rcu()
98 	kfree(nh);  in nexthop_free_rcu()
104 	struct nexthop *nh;  in nexthop_alloc()  local
106 	nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);  in nexthop_alloc()
107 	if (nh) {  in nexthop_alloc()
108 		INIT_LIST_HEAD(&nh->fi_list);  in nexthop_alloc()
109 		INIT_LIST_HEAD(&nh->f6i_list);  in nexthop_alloc()
110 		INIT_LIST_HEAD(&nh->grp_list);  in nexthop_alloc()
112 	return nh;  in nexthop_alloc()
142 		struct nexthop *nh;  in nexthop_find_by_id()  local
149 		nh = rb_entry(parent, struct nexthop, rb_node);  in nexthop_find_by_id()
150 		if (id < nh->id)  in nexthop_find_by_id()
152 		else if (id > nh->id)  in nexthop_find_by_id()
155 			return nh;  in nexthop_find_by_id()
197 		p->id = nhg->nh_entries[i].nh->id;  in nla_put_nh_group()
208 static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,  in nh_fill_node()  argument
223 	nhm->nh_flags = nh->nh_flags;  in nh_fill_node()
224 	nhm->nh_protocol = nh->protocol;  in nh_fill_node()
228 	if (nla_put_u32(skb, NHA_ID, nh->id))  in nh_fill_node()
231 	if (nh->is_group) {  in nh_fill_node()
232 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_fill_node()
239 	nhi = rtnl_dereference(nh->nh_info);  in nh_fill_node()
283 static size_t nh_nlmsg_size_grp(struct nexthop *nh)  in nh_nlmsg_size_grp()  argument
285 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_nlmsg_size_grp()
292 static size_t nh_nlmsg_size_single(struct nexthop *nh)  in nh_nlmsg_size_single()  argument
294 	struct nh_info *nhi = rtnl_dereference(nh->nh_info);  in nh_nlmsg_size_single()
323 static size_t nh_nlmsg_size(struct nexthop *nh)  in nh_nlmsg_size()  argument
327 	if (nh->is_group)  in nh_nlmsg_size()
328 		sz += nh_nlmsg_size_grp(nh);  in nh_nlmsg_size()
330 		sz += nh_nlmsg_size_single(nh);  in nh_nlmsg_size()
335 static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)  in nexthop_notify()  argument
342 	skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());  in nexthop_notify()
346 	err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags);  in nexthop_notify()
362 static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,  in valid_group_nh()  argument
365 	if (nh->is_group) {  in valid_group_nh()
366 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in valid_group_nh()
377 		struct nh_info *nhi = rtnl_dereference(nh->nh_info);  in valid_group_nh()
425 		struct nexthop *nh;  in nh_check_attr_group()  local
427 		nh = nexthop_find_by_id(net, nhg[i].id);  in nh_check_attr_group()
428 		if (!nh) {  in nh_check_attr_group()
432 		if (!valid_group_nh(nh, len, extack))  in nh_check_attr_group()
447 static bool ipv6_good_nh(const struct fib6_nh *nh)  in ipv6_good_nh()  argument
454 	n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);  in ipv6_good_nh()
463 static bool ipv4_good_nh(const struct fib_nh *nh)  in ipv4_good_nh()  argument
470 	n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,  in ipv4_good_nh()
471 				      (__force u32)nh->fib_nh_gw4);  in ipv4_good_nh()
480 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)  in nexthop_select_path()  argument
486 	if (!nh->is_group)  in nexthop_select_path()
487 		return nh;  in nexthop_select_path()
489 	nhg = rcu_dereference(nh->nh_grp);  in nexthop_select_path()
500 		nhi = rcu_dereference(nhge->nh->nh_info);  in nexthop_select_path()
504 				return nhge->nh;  in nexthop_select_path()
508 				return nhge->nh;  in nexthop_select_path()
513 			rc = nhge->nh;  in nexthop_select_path()
520 int nexthop_for_each_fib6_nh(struct nexthop *nh,  in nexthop_for_each_fib6_nh()  argument
521 			     int (*cb)(struct fib6_nh *nh, void *arg),  in nexthop_for_each_fib6_nh()  argument
527 	if (nh->is_group) {  in nexthop_for_each_fib6_nh()
531 		nhg = rcu_dereference_rtnl(nh->nh_grp);  in nexthop_for_each_fib6_nh()
535 			nhi = rcu_dereference_rtnl(nhge->nh->nh_info);  in nexthop_for_each_fib6_nh()
541 		nhi = rcu_dereference_rtnl(nh->nh_info);  in nexthop_for_each_fib6_nh()
561 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,  in fib6_check_nexthop()  argument
575 	if (nh->is_group) {  in fib6_check_nexthop()
578 		nhg = rtnl_dereference(nh->nh_grp);  in fib6_check_nexthop()
582 		nhi = rtnl_dereference(nh->nh_info);  in fib6_check_nexthop()
613 static int nexthop_check_scope(struct nexthop *nh, u8 scope,  in nexthop_check_scope()  argument
618 	nhi = rtnl_dereference(nh->nh_info);  in nexthop_check_scope()
637 int fib_check_nexthop(struct nexthop *nh, u8 scope,  in fib_check_nexthop()  argument
642 	if (nh->is_group) {  in fib_check_nexthop()
651 		nhg = rtnl_dereference(nh->nh_grp);  in fib_check_nexthop()
653 		err = nexthop_check_scope(nhg->nh_entries[0].nh, scope, extack);  in fib_check_nexthop()
655 		err = nexthop_check_scope(nh, scope, extack);  in fib_check_nexthop()
699 	struct nexthop *nh = nhge->nh;  in remove_nh_grp_entry()  local
704 	WARN_ON(!nh);  in remove_nh_grp_entry()
709 			nhges[i-1].nh = nhges[i].nh;  in remove_nh_grp_entry()
712 			list_add(&nhges[i-1].nh_list, &nhges[i-1].nh->grp_list);  in remove_nh_grp_entry()
713 		} else if (nhg->nh_entries[i].nh == nh) {  in remove_nh_grp_entry()
722 	nhg->nh_entries[nhg->num_nh].nh = NULL;  in remove_nh_grp_entry()
726 	nexthop_put(nh);  in remove_nh_grp_entry()
732 static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,  in remove_nexthop_from_groups()  argument
737 	list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) {  in remove_nexthop_from_groups()
750 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)  in remove_nexthop_group()  argument
752 	struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);  in remove_nexthop_group()
758 		if (WARN_ON(!nhge->nh))  in remove_nexthop_group()
762 		nexthop_put(nhge->nh);  in remove_nexthop_group()
763 		nhge->nh = NULL;  in remove_nexthop_group()
769 static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)  in __remove_nexthop_fib()  argument
775 	list_for_each_entry(fi, &nh->fi_list, nh_list) {  in __remove_nexthop_fib()
783 	list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {  in __remove_nexthop_fib()
790 static void __remove_nexthop(struct net *net, struct nexthop *nh,  in __remove_nexthop()  argument
793 	__remove_nexthop_fib(net, nh);  in __remove_nexthop()
795 	if (nh->is_group) {  in __remove_nexthop()
796 		remove_nexthop_group(nh, nlinfo);  in __remove_nexthop()
800 		nhi = rtnl_dereference(nh->nh_info);  in __remove_nexthop()
804 		remove_nexthop_from_groups(net, nh, nlinfo);  in __remove_nexthop()
808 static void remove_nexthop(struct net *net, struct nexthop *nh,  in remove_nexthop()  argument
812 	rb_erase(&nh->rb_node, &net->nexthop.rb_root);  in remove_nexthop()
815 		nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);  in remove_nexthop()
817 	__remove_nexthop(net, nh, nlinfo);  in remove_nexthop()
820 	nexthop_put(nh);  in remove_nexthop()
826 static void nh_rt_cache_flush(struct net *net, struct nexthop *nh)  in nh_rt_cache_flush()  argument
830 	if (!list_empty(&nh->fi_list))  in nh_rt_cache_flush()
833 	list_for_each_entry(f6i, &nh->f6i_list, nh_list)  in nh_rt_cache_flush()
892 static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,  in __nexthop_replace_notify()  argument
897 	if (!list_empty(&nh->fi_list)) {  in __nexthop_replace_notify()
904 		list_for_each_entry(fi, &nh->fi_list, nh_list)  in __nexthop_replace_notify()
909 		list_for_each_entry(fi, &nh->fi_list, nh_list)  in __nexthop_replace_notify()
913 	list_for_each_entry(f6i, &nh->f6i_list, nh_list)  in __nexthop_replace_notify()
921 static void nexthop_replace_notify(struct net *net, struct nexthop *nh,  in nexthop_replace_notify()  argument
926 	__nexthop_replace_notify(net, nh, info);  in nexthop_replace_notify()
928 	list_for_each_entry(nhge, &nh->grp_list, nh_list)  in nexthop_replace_notify()
1004 		struct nexthop *nh;  in insert_nexthop()  local
1012 		nh = rb_entry(parent, struct nexthop, rb_node);  in insert_nexthop()
1013 		if (new_id < nh->id) {  in insert_nexthop()
1015 		} else if (new_id > nh->id) {  in insert_nexthop()
1018 			rc = replace_nexthop(net, nh, new_nh, extack);  in insert_nexthop()
1020 				new_nh = nh; /* send notification with old nh */  in insert_nexthop()
1073 	struct nexthop *nh;  in flush_all_nexthops()  local
1076 		nh = rb_entry(node, struct nexthop, rb_node);  in flush_all_nexthops()
1077 		remove_nexthop(net, nh, NULL);  in flush_all_nexthops()
1088 	struct nexthop *nh;  in nexthop_create_group()  local
1091 	nh = nexthop_alloc();  in nexthop_create_group()
1092 	if (!nh)  in nexthop_create_group()
1095 	nh->is_group = 1;  in nexthop_create_group()
1099 		kfree(nh);  in nexthop_create_group()
1115 		nhg->nh_entries[i].nh = nhe;  in nexthop_create_group()
1118 		nhg->nh_entries[i].nh_parent = nh;  in nexthop_create_group()
1126 	rcu_assign_pointer(nh->nh_grp, nhg);  in nexthop_create_group()
1128 	return nh;  in nexthop_create_group()
1132 		nexthop_put(nhg->nh_entries[i].nh);  in nexthop_create_group()
1135 	kfree(nh);  in nexthop_create_group()
1140 static int nh_create_ipv4(struct net *net, struct nexthop *nh,  in nh_create_ipv4()  argument
1165 		nh->nh_flags = fib_nh->fib_nh_flags;  in nh_create_ipv4()
1175 static int nh_create_ipv6(struct net *net,  struct nexthop *nh,  in nh_create_ipv6()  argument
1199 		nh->nh_flags = fib6_nh->fib_nh_flags;  in nh_create_ipv6()
1208 	struct nexthop *nh;  in nexthop_create()  local
1211 	nh = nexthop_alloc();  in nexthop_create()
1212 	if (!nh)  in nexthop_create()
1217 		kfree(nh);  in nexthop_create()
1221 	nh->nh_flags = cfg->nh_flags;  in nexthop_create()
1222 	nh->net = net;  in nexthop_create()
1224 	nhi->nh_parent = nh;  in nexthop_create()
1235 		err = nh_create_ipv4(net, nh, nhi, cfg, extack);  in nexthop_create()
1238 		err = nh_create_ipv6(net, nh, nhi, cfg, extack);  in nexthop_create()
1244 		kfree(nh);  in nexthop_create()
1251 	rcu_assign_pointer(nh->nh_info, nhi);  in nexthop_create()
1253 	return nh;  in nexthop_create()
1260 	struct nexthop *nh;  in nexthop_add()  local
1277 		nh = nexthop_create_group(net, cfg);  in nexthop_add()
1279 		nh = nexthop_create(net, cfg, extack);  in nexthop_add()
1281 	if (IS_ERR(nh))  in nexthop_add()
1282 		return nh;  in nexthop_add()
1284 	refcount_set(&nh->refcnt, 1);  in nexthop_add()
1285 	nh->id = cfg->nh_id;  in nexthop_add()
1286 	nh->protocol = cfg->nh_protocol;  in nexthop_add()
1287 	nh->net = net;  in nexthop_add()
1289 	err = insert_nexthop(net, nh, cfg, extack);  in nexthop_add()
1291 		__remove_nexthop(net, nh, NULL);  in nexthop_add()
1292 		nexthop_put(nh);  in nexthop_add()
1293 		nh = ERR_PTR(err);  in nexthop_add()
1296 	return nh;  in nexthop_add()
1471 	struct nexthop *nh;  in rtm_new_nexthop()  local
1476 		nh = nexthop_add(net, &cfg, extack);  in rtm_new_nexthop()
1477 		if (IS_ERR(nh))  in rtm_new_nexthop()
1478 			err = PTR_ERR(nh);  in rtm_new_nexthop()
1539 	struct nexthop *nh;  in rtm_del_nexthop()  local
1547 	nh = nexthop_find_by_id(net, id);  in rtm_del_nexthop()
1548 	if (!nh)  in rtm_del_nexthop()
1551 	remove_nexthop(net, nh, &nlinfo);  in rtm_del_nexthop()
1562 	struct nexthop *nh;  in rtm_get_nexthop()  local
1576 	nh = nexthop_find_by_id(net, id);  in rtm_get_nexthop()
1577 	if (!nh)  in rtm_get_nexthop()
1580 	err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,  in rtm_get_nexthop()
1595 static bool nh_dump_filtered(struct nexthop *nh, int dev_idx, int master_idx,  in nh_dump_filtered()  argument
1601 	if (group_filter && !nh->is_group)  in nh_dump_filtered()
1607 	if (nh->is_group)  in nh_dump_filtered()
1610 	nhi = rtnl_dereference(nh->nh_info);  in nh_dump_filtered()
1705 		struct nexthop *nh;  in rtm_dump_nexthop()  local
1710 		nh = rb_entry(node, struct nexthop, rb_node);  in rtm_dump_nexthop()
1711 		if (nh_dump_filtered(nh, dev_filter_idx, master_idx,  in rtm_dump_nexthop()
1715 		err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP,  in rtm_dump_nexthop()