101 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
432 	if ((!match->fib6_nsiblings && !match->nh) || have_oif_match)  in fib6_select_path()
439 	    (!match->nh || nexthop_is_multipath(match->nh)))  in fib6_select_path()
442 	if (unlikely(match->nh)) {  in fib6_select_path()
452 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()  local
455 		nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);  in fib6_select_path()
458 		if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)  in fib6_select_path()
466 	res->nh = match->fib6_nh;  in fib6_select_path()
473 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()  argument
478 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in __rt6_device_match()
481 	dev = nh->fib_nh_dev;  in __rt6_device_match()
499 	struct fib6_nh		*nh;  member
502 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()  argument
506 	arg->nh = nh;  in __rt6_nh_dev_match()
507 	return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,  in __rt6_nh_dev_match()
512 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()  argument
524 	if (nexthop_is_blackhole(nh))  in rt6_nh_dev_match()
527 	if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))  in rt6_nh_dev_match()
528 		return arg.nh;  in rt6_nh_dev_match()
538 	struct fib6_nh *nh;  in rt6_device_match()  local
541 		if (unlikely(f6i->nh)) {  in rt6_device_match()
542 			nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
543 			if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
546 			nh = f6i->fib6_nh;  in rt6_device_match()
548 		if (!(nh->fib_nh_flags & RTNH_F_DEAD))  in rt6_device_match()
555 		if (unlikely(spf6i->nh)) {  in rt6_device_match()
556 			nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,  in rt6_device_match()
558 			if (nh)  in rt6_device_match()
561 			nh = spf6i->fib6_nh;  in rt6_device_match()
562 			if (__rt6_device_match(net, nh, saddr, oif, flags))  in rt6_device_match()
573 		nh = res->f6i->fib6_nh;  in rt6_device_match()
577 	if (unlikely(f6i->nh)) {  in rt6_device_match()
578 		nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
579 		if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
582 		nh = f6i->fib6_nh;  in rt6_device_match()
585 	if (nh->fib_nh_flags & RTNH_F_DEAD) {  in rt6_device_match()
587 		nh = res->f6i->fib6_nh;  in rt6_device_match()
590 	res->nh = nh;  in rt6_device_match()
598 	res->nh = nh;  in rt6_device_match()
715 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()  argument
720 	if (!oif || nh->fib_nh_dev->ifindex == oif)  in rt6_score_route()
729 	    !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {  in rt6_score_route()
730 		int n = rt6_check_neigh(nh);  in rt6_score_route()
737 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()  argument
744 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in find_match()
747 	if (ip6_ignore_linkdown(nh->fib_nh_dev) &&  in find_match()
748 	    nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in find_match()
752 	m = rt6_score_route(nh, fib6_flags, oif, strict);  in find_match()
761 		rt6_probe(nh);  in find_match()
779 	struct fib6_nh	*nh;  member
782 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()  argument
786 	arg->nh = nh;  in rt6_nh_find_match()
787 	return find_match(nh, arg->flags, arg->oif, arg->strict,  in rt6_nh_find_match()
802 		struct fib6_nh *nh;  in __find_rr_leaf()  local
812 		if (unlikely(f6i->nh)) {  in __find_rr_leaf()
821 			if (nexthop_is_blackhole(f6i->nh)) {  in __find_rr_leaf()
825 				res->nh = nexthop_fib6_nh(f6i->nh);  in __find_rr_leaf()
828 			if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,  in __find_rr_leaf()
831 				nh = arg.nh;  in __find_rr_leaf()
834 			nh = f6i->fib6_nh;  in __find_rr_leaf()
835 			if (find_match(nh, f6i->fib6_flags, oif, strict,  in __find_rr_leaf()
841 			res->nh = nh;  in __find_rr_leaf()
920 		res->nh = res->f6i->fib6_nh;  in rt6_select()
929 	       res->nh->fib_nh_gw_family;  in rt6_is_gw_or_nonexthop()
1015 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_rt_get_dev_rcu()
1111 	if (res->nh->fib_nh_lws) {  in ip6_rt_init_dst()
1112 		rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);  in ip6_rt_init_dst()
1130 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()  local
1131 	const struct net_device *dev = nh->fib_nh_dev;  in ip6_rt_copy_init()
1139 	if (nh->fib_nh_gw_family) {  in ip6_rt_copy_init()
1140 		rt->rt6i_gateway = nh->fib_nh_gw6;  in ip6_rt_copy_init()
1186 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_create_rt_rcu()
1397 	pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1411 	p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_make_pcpu_route()
1561 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()  local
1567 		struct net_device *dev = nh->fib_nh_dev;  in fib6_mtu()
1578 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in fib6_mtu()
1589 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()  argument
1595 		bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_get_excptn_bucket()
1598 		bucket = rcu_dereference(nh->rt6i_exception_bucket);  in fib6_nh_get_excptn_bucket()
1619 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()  argument
1625 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_excptn_bucket_set_flushed()
1631 	rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in fib6_nh_excptn_bucket_set_flushed()
1642 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()  local
1647 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in rt6_insert_exception()
1656 		rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in rt6_insert_exception()
1714 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()  argument
1723 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1729 		fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1744 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()  argument
1748 	fib6_nh_flush_exceptions(nh, f6i);  in rt6_nh_flush_exceptions()
1755 	if (f6i->nh)  in rt6_flush_exceptions()
1756 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,  in rt6_flush_exceptions()
1790 	bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);  in rt6_find_cached_rt()
1808 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()  argument
1816 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_remove_exception()
1820 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_remove_exception()
1851 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()  argument
1856 	err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);  in rt6_nh_remove_exception_rt()
1871 	if (from->nh) {  in rt6_remove_exception_rt()
1879 		rc = nexthop_for_each_fib6_nh(from->nh,  in rt6_remove_exception_rt()
1892 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()  argument
1899 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in fib6_nh_update_exception()
1922 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()  argument
1926 	if (arg->dev != nh->fib_nh_dev ||  in fib6_nh_find_match()
1927 	    (arg->gw && !nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1928 	    (!arg->gw && nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1929 	    (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))  in fib6_nh_find_match()
1932 	arg->match = nh;  in fib6_nh_find_match()
1949 	if (from->nh) {  in rt6_update_exception_stamp_rt()
1955 		nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);  in rt6_update_exception_stamp_rt()
1991 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()  argument
1997 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in rt6_exceptions_update_pmtu()
2019 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()  argument
2027 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_exceptions_clean_tohost()
2031 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_exceptions_clean_tohost()
2096 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()  argument
2105 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_age_exceptions()
2110 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_age_exceptions()
2130 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()  argument
2134 	fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);  in rt6_nh_age_exceptions()
2142 	if (f6i->nh) {  in rt6_age_exceptions()
2148 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,  in rt6_age_exceptions()
2215 			    !res.nh->fib_nh_gw_family)) {  in ip6_pol_route()
2738 		if (res.f6i->nh) {  in __ip6_rt_update_pmtu()
2744 			nexthop_for_each_fib6_nh(res.f6i->nh,  in __ip6_rt_update_pmtu()
2753 			res.nh = arg.match;  in __ip6_rt_update_pmtu()
2755 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2840 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()  local
2842 	if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||  in ip6_redirect_nh_match()
2843 	    fl6->flowi6_oif != nh->fib_nh_dev->ifindex)  in ip6_redirect_nh_match()
2851 	if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {  in ip6_redirect_nh_match()
2872 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()  argument
2876 	arg->res->nh = nh;  in fib6_nh_redirect_match()
2929 		if (unlikely(rt->nh)) {  in __ip6_route_redirect()
2930 			if (nexthop_is_blackhole(rt->nh))  in __ip6_route_redirect()
2933 			if (nexthop_for_each_fib6_nh(rt->nh,  in __ip6_route_redirect()
2938 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
2959 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3093 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()  local
3109 		struct net_device *dev = nh->fib_nh_dev;  in ip6_mtu_from_fib6()
3119 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in ip6_mtu_from_fib6()
3229 	    (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {  in ip6_route_check_nh_onlink()
3256 		    res.nh->fib_nh_gw_family ||  in ip6_route_check_nh()
3257 		    (dev && dev != res.nh->fib_nh_dev))  in ip6_route_check_nh()
3269 		    res.nh->fib_nh_gw_family)  in ip6_route_check_nh()
3281 		if (dev != res.nh->fib_nh_dev)  in ip6_route_check_nh()
3284 		*_dev = dev = res.nh->fib_nh_dev;  in ip6_route_check_nh()
3545 	struct nexthop *nh = NULL;  in ip6_route_info_create()  local
3584 		nh = nexthop_find_by_id(net, cfg->fc_nh_id);  in ip6_route_info_create()
3585 		if (!nh) {  in ip6_route_info_create()
3589 		err = fib6_check_nexthop(nh, cfg, extack);  in ip6_route_info_create()
3610 	rt = fib6_info_alloc(gfp_flags, !nh);  in ip6_route_info_create()
3650 	if (nh) {  in ip6_route_info_create()
3651 		if (!nexthop_get(nh)) {  in ip6_route_info_create()
3659 		rt->nh = nh;  in ip6_route_info_create()
3660 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3815 			     struct fib6_nh *nh)  in ip6_del_cached_rt()  argument
3819 		.nh = nh,  in ip6_del_cached_rt()
3835 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()  argument
3840 	rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);  in fib6_nh_del_cached_rt()
3851 	return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);  in ip6_del_cached_rt_nh()
3877 			struct fib6_nh *nh;  in ip6_route_del()  local
3879 			if (rt->nh && cfg->fc_nh_id &&  in ip6_route_del()
3880 			    rt->nh->id != cfg->fc_nh_id)  in ip6_route_del()
3886 				if (rt->nh) {  in ip6_route_del()
3891 					nh = rt->fib6_nh;  in ip6_route_del()
3892 					rc = ip6_del_cached_rt(cfg, rt, nh);  in ip6_route_del()
3907 			if (rt->nh) {  in ip6_route_del()
3917 			nh = rt->fib6_nh;  in ip6_route_del()
3919 			    (!nh->fib_nh_dev ||  in ip6_route_del()
3920 			     nh->fib_nh_dev->ifindex != cfg->fc_ifindex))  in ip6_route_del()
3923 			    !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))  in ip6_route_del()
4035 	if (res.f6i->nh) {  in rt6_do_redirect()
4041 		nexthop_for_each_fib6_nh(res.f6i->nh,  in rt6_do_redirect()
4049 		res.nh = arg.match;  in rt6_do_redirect()
4051 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4106 		if (rt->nh)  in rt6_get_route_info()
4171 		struct fib6_nh *nh;  in rt6_get_dflt_router()  local
4174 		if (rt->nh)  in rt6_get_dflt_router()
4177 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4178 		if (dev == nh->fib_nh_dev &&  in rt6_get_dflt_router()
4180 		    ipv6_addr_equal(&nh->fib_nh_gw6, addr))  in rt6_get_dflt_router()
4432 	if (!rt->nh &&  in fib6_remove_prefsrc()
4461 	struct fib6_nh *nh;  in fib6_clean_tohost()  local
4464 	if (rt->nh)  in fib6_clean_tohost()
4467 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4469 	    nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))  in fib6_clean_tohost()
4476 	fib6_nh_exceptions_clean_tohost(nh, gateway);  in fib6_clean_tohost()
4593 	if (rt != net->ipv6.fib6_null_entry && !rt->nh &&  in fib6_ifup()
4679 	if (rt == net->ipv6.fib6_null_entry || rt->nh)  in fib6_ifdown()
4745 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()  argument
4755 	if (nh->fib_nh_dev == arg->dev) {  in fib6_nh_mtu_change()
4764 		rt6_exceptions_update_pmtu(idev, nh, arg->mtu);  in fib6_nh_mtu_change()
4790 	if (f6i->nh) {  in rt6_mtu_change_route()
4792 		return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,  in rt6_mtu_change_route()
4983 	struct rt6_nh *nh;  in ip6_route_info_append()  local
4986 	list_for_each_entry(nh, rt6_nh_list, next) {  in ip6_route_info_append()
4988 		if (rt6_duplicate_nexthop(nh->fib6_info, rt))  in ip6_route_info_append()
4992 	nh = kzalloc(sizeof(*nh), GFP_KERNEL);  in ip6_route_info_append()
4993 	if (!nh)  in ip6_route_info_append()
4995 	nh->fib6_info = rt;  in ip6_route_info_append()
4996 	memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));  in ip6_route_info_append()
4997 	list_add_tail(&nh->next, rt6_nh_list);  in ip6_route_info_append()
5033 	struct rt6_nh *nh, *nh_safe;  in ip6_route_multipath_add()  local
5118 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5119 		err = __ip6_ins_rt(nh->fib6_info, info, extack);  in ip6_route_multipath_add()
5120 		fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5124 			rt_last = nh->fib6_info;  in ip6_route_multipath_add()
5128 				rt_notif = nh->fib6_info;  in ip6_route_multipath_add()
5132 		nh->fib6_info = NULL;  in ip6_route_multipath_add()
5137 			err_nh = nh;  in ip6_route_multipath_add()
5175 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5176 		if (err_nh == nh)  in ip6_route_multipath_add()
5178 		ip6_route_del(&nh->r_cfg, extack);  in ip6_route_multipath_add()
5182 	list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5183 		if (nh->fib6_info)  in ip6_route_multipath_add()
5184 			fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5185 		list_del(&nh->next);  in ip6_route_multipath_add()
5186 		kfree(nh);  in ip6_route_multipath_add()
5274 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()  argument
5282 	if (nh->fib_nh_lws) {  in rt6_nh_nlmsg_size()
5284 		*nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nh_nlmsg_size()
5296 	if (f6i->nh) {  in rt6_nlmsg_size()
5298 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,  in rt6_nlmsg_size()
5301 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()  local
5308 				    + lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5312 		nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5331 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,  in rt6_fill_node_nexthop()  argument
5334 	if (nexthop_is_multipath(nh)) {  in rt6_fill_node_nexthop()
5341 		if (nexthop_mpath_fill_node(skb, nh, AF_INET6))  in rt6_fill_node_nexthop()
5348 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5490 	} else if (rt->nh) {  in rt6_fill_node()
5491 		if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))  in rt6_fill_node()
5494 		if (nexthop_is_blackhole(rt->nh))  in rt6_fill_node()
5497 		if (rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)  in rt6_fill_node()
5529 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()  argument
5533 	if (nh->fib_nh_dev == dev)  in fib6_info_nh_uses_dev()
5542 	if (f6i->nh) {  in fib6_info_uses_dev()
5545 		return !!nexthop_for_each_fib6_nh(f6i->nh,  in fib6_info_uses_dev()
5574 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()  argument
5582 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in rt6_nh_dump_exceptions()
5677 		if (rt->nh) {  in rt6_dump_route()
5678 			err = nexthop_for_each_fib6_nh(rt->nh,  in rt6_dump_route()