103 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
435 	if (!match->nh && (!match->fib6_nsiblings || have_oif_match))  in fib6_select_path()
438 	if (match->nh && have_oif_match && res->nh)  in fib6_select_path()
445 	    (!match->nh || nexthop_is_multipath(match->nh)))  in fib6_select_path()
448 	if (unlikely(match->nh)) {  in fib6_select_path()
458 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()  local
461 		nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);  in fib6_select_path()
464 		if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)  in fib6_select_path()
472 	res->nh = match->fib6_nh;  in fib6_select_path()
479 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()  argument
484 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in __rt6_device_match()
487 	dev = nh->fib_nh_dev;  in __rt6_device_match()
505 	struct fib6_nh		*nh;  member
508 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()  argument
512 	arg->nh = nh;  in __rt6_nh_dev_match()
513 	return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,  in __rt6_nh_dev_match()
518 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()  argument
530 	if (nexthop_is_blackhole(nh))  in rt6_nh_dev_match()
533 	if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))  in rt6_nh_dev_match()
534 		return arg.nh;  in rt6_nh_dev_match()
544 	struct fib6_nh *nh;  in rt6_device_match()  local
547 		if (unlikely(f6i->nh)) {  in rt6_device_match()
548 			nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
549 			if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
552 			nh = f6i->fib6_nh;  in rt6_device_match()
554 		if (!(nh->fib_nh_flags & RTNH_F_DEAD))  in rt6_device_match()
561 		if (unlikely(spf6i->nh)) {  in rt6_device_match()
562 			nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,  in rt6_device_match()
564 			if (nh)  in rt6_device_match()
567 			nh = spf6i->fib6_nh;  in rt6_device_match()
568 			if (__rt6_device_match(net, nh, saddr, oif, flags))  in rt6_device_match()
579 		nh = res->f6i->fib6_nh;  in rt6_device_match()
583 	if (unlikely(f6i->nh)) {  in rt6_device_match()
584 		nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
585 		if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
588 		nh = f6i->fib6_nh;  in rt6_device_match()
591 	if (nh->fib_nh_flags & RTNH_F_DEAD) {  in rt6_device_match()
593 		nh = res->f6i->fib6_nh;  in rt6_device_match()
596 	res->nh = nh;  in rt6_device_match()
604 	res->nh = nh;  in rt6_device_match()
721 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()  argument
726 	if (!oif || nh->fib_nh_dev->ifindex == oif)  in rt6_score_route()
735 	    !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {  in rt6_score_route()
736 		int n = rt6_check_neigh(nh);  in rt6_score_route()
743 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()  argument
750 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in find_match()
753 	if (ip6_ignore_linkdown(nh->fib_nh_dev) &&  in find_match()
754 	    nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in find_match()
758 	m = rt6_score_route(nh, fib6_flags, oif, strict);  in find_match()
767 		rt6_probe(nh);  in find_match()
785 	struct fib6_nh	*nh;  member
788 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()  argument
792 	arg->nh = nh;  in rt6_nh_find_match()
793 	return find_match(nh, arg->flags, arg->oif, arg->strict,  in rt6_nh_find_match()
808 		struct fib6_nh *nh;  in __find_rr_leaf()  local
818 		if (unlikely(f6i->nh)) {  in __find_rr_leaf()
827 			if (nexthop_is_blackhole(f6i->nh)) {  in __find_rr_leaf()
831 				res->nh = nexthop_fib6_nh(f6i->nh);  in __find_rr_leaf()
834 			if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,  in __find_rr_leaf()
837 				nh = arg.nh;  in __find_rr_leaf()
840 			nh = f6i->fib6_nh;  in __find_rr_leaf()
841 			if (find_match(nh, f6i->fib6_flags, oif, strict,  in __find_rr_leaf()
847 			res->nh = nh;  in __find_rr_leaf()
926 		res->nh = res->f6i->fib6_nh;  in rt6_select()
935 	       res->nh->fib_nh_gw_family;  in rt6_is_gw_or_nonexthop()
1021 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_rt_get_dev_rcu()
1115 	if (res->nh->fib_nh_lws) {  in ip6_rt_init_dst()
1116 		rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);  in ip6_rt_init_dst()
1134 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()  local
1135 	const struct net_device *dev = nh->fib_nh_dev;  in ip6_rt_copy_init()
1143 	if (nh->fib_nh_gw_family) {  in ip6_rt_copy_init()
1144 		rt->rt6i_gateway = nh->fib_nh_gw6;  in ip6_rt_copy_init()
1190 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_create_rt_rcu()
1393 	if (f6i->nh)  in ip6_rt_pcpu_alloc()
1409 	pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1414 		p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1436 	p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_make_pcpu_route()
1586 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()  local
1592 		struct net_device *dev = nh->fib_nh_dev;  in fib6_mtu()
1603 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in fib6_mtu()
1614 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()  argument
1620 		bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_get_excptn_bucket()
1623 		bucket = rcu_dereference(nh->rt6i_exception_bucket);  in fib6_nh_get_excptn_bucket()
1644 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()  argument
1650 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_excptn_bucket_set_flushed()
1656 	rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in fib6_nh_excptn_bucket_set_flushed()
1667 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()  local
1672 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in rt6_insert_exception()
1681 		rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in rt6_insert_exception()
1739 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()  argument
1748 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1754 		fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1769 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()  argument
1773 	fib6_nh_flush_exceptions(nh, f6i);  in rt6_nh_flush_exceptions()
1780 	if (f6i->nh)  in rt6_flush_exceptions()
1781 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,  in rt6_flush_exceptions()
1815 	bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);  in rt6_find_cached_rt()
1833 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()  argument
1841 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_remove_exception()
1845 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_remove_exception()
1876 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()  argument
1881 	err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);  in rt6_nh_remove_exception_rt()
1896 	if (from->nh) {  in rt6_remove_exception_rt()
1904 		rc = nexthop_for_each_fib6_nh(from->nh,  in rt6_remove_exception_rt()
1917 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()  argument
1924 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in fib6_nh_update_exception()
1947 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()  argument
1951 	if (arg->dev != nh->fib_nh_dev ||  in fib6_nh_find_match()
1952 	    (arg->gw && !nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1953 	    (!arg->gw && nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1954 	    (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))  in fib6_nh_find_match()
1957 	arg->match = nh;  in fib6_nh_find_match()
1974 	if (from->nh) {  in rt6_update_exception_stamp_rt()
1980 		nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);  in rt6_update_exception_stamp_rt()
2016 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()  argument
2022 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in rt6_exceptions_update_pmtu()
2044 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()  argument
2052 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_exceptions_clean_tohost()
2056 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_exceptions_clean_tohost()
2121 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()  argument
2130 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_age_exceptions()
2135 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_age_exceptions()
2155 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()  argument
2159 	fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);  in rt6_nh_age_exceptions()
2167 	if (f6i->nh) {  in rt6_age_exceptions()
2173 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,  in rt6_age_exceptions()
2240 			    !res.nh->fib_nh_gw_family)) {  in ip6_pol_route()
2770 		if (res.f6i->nh) {  in __ip6_rt_update_pmtu()
2776 			nexthop_for_each_fib6_nh(res.f6i->nh,  in __ip6_rt_update_pmtu()
2785 			res.nh = arg.match;  in __ip6_rt_update_pmtu()
2787 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2874 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()  local
2876 	if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||  in ip6_redirect_nh_match()
2877 	    fl6->flowi6_oif != nh->fib_nh_dev->ifindex)  in ip6_redirect_nh_match()
2885 	if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {  in ip6_redirect_nh_match()
2906 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()  argument
2910 	arg->res->nh = nh;  in fib6_nh_redirect_match()
2963 		if (unlikely(rt->nh)) {  in __ip6_route_redirect()
2964 			if (nexthop_is_blackhole(rt->nh))  in __ip6_route_redirect()
2967 			if (nexthop_for_each_fib6_nh(rt->nh,  in __ip6_route_redirect()
2972 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
2993 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3127 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()  local
3143 		struct net_device *dev = nh->fib_nh_dev;  in ip6_mtu_from_fib6()
3153 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in ip6_mtu_from_fib6()
3265 	    (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {  in ip6_route_check_nh_onlink()
3292 		    res.nh->fib_nh_gw_family ||  in ip6_route_check_nh()
3293 		    (dev && dev != res.nh->fib_nh_dev))  in ip6_route_check_nh()
3305 		    res.nh->fib_nh_gw_family)  in ip6_route_check_nh()
3317 		if (dev != res.nh->fib_nh_dev)  in ip6_route_check_nh()
3320 		*_dev = dev = res.nh->fib_nh_dev;  in ip6_route_check_nh()
3586 	struct nexthop *nh = NULL;  in ip6_route_info_create()  local
3625 		nh = nexthop_find_by_id(net, cfg->fc_nh_id);  in ip6_route_info_create()
3626 		if (!nh) {  in ip6_route_info_create()
3630 		err = fib6_check_nexthop(nh, cfg, extack);  in ip6_route_info_create()
3651 	rt = fib6_info_alloc(gfp_flags, !nh);  in ip6_route_info_create()
3689 	if (nh) {  in ip6_route_info_create()
3694 		if (!nexthop_get(nh)) {  in ip6_route_info_create()
3698 		rt->nh = nh;  in ip6_route_info_create()
3699 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3878 			     struct fib6_nh *nh)  in ip6_del_cached_rt()  argument
3882 		.nh = nh,  in ip6_del_cached_rt()
3898 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()  argument
3903 	rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);  in fib6_nh_del_cached_rt()
3914 	return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);  in ip6_del_cached_rt_nh()
3940 			struct fib6_nh *nh;  in ip6_route_del()  local
3942 			if (rt->nh && cfg->fc_nh_id &&  in ip6_route_del()
3943 			    rt->nh->id != cfg->fc_nh_id)  in ip6_route_del()
3949 				if (rt->nh) {  in ip6_route_del()
3954 					nh = rt->fib6_nh;  in ip6_route_del()
3955 					rc = ip6_del_cached_rt(cfg, rt, nh);  in ip6_route_del()
3970 			if (rt->nh) {  in ip6_route_del()
3980 			nh = rt->fib6_nh;  in ip6_route_del()
3982 			    (!nh->fib_nh_dev ||  in ip6_route_del()
3983 			     nh->fib_nh_dev->ifindex != cfg->fc_ifindex))  in ip6_route_del()
3986 			    !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))  in ip6_route_del()
4098 	if (res.f6i->nh) {  in rt6_do_redirect()
4104 		nexthop_for_each_fib6_nh(res.f6i->nh,  in rt6_do_redirect()
4112 		res.nh = arg.match;  in rt6_do_redirect()
4114 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4169 		if (rt->nh)  in rt6_get_route_info()
4234 		struct fib6_nh *nh;  in rt6_get_dflt_router()  local
4237 		if (rt->nh)  in rt6_get_dflt_router()
4240 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4241 		if (dev == nh->fib_nh_dev &&  in rt6_get_dflt_router()
4243 		    ipv6_addr_equal(&nh->fib_nh_gw6, addr))  in rt6_get_dflt_router()
4483 	if (!rt->nh &&  in fib6_remove_prefsrc()
4512 	struct fib6_nh *nh;  in fib6_clean_tohost()  local
4515 	if (rt->nh)  in fib6_clean_tohost()
4518 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4520 	    nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))  in fib6_clean_tohost()
4527 	fib6_nh_exceptions_clean_tohost(nh, gateway);  in fib6_clean_tohost()
4644 	if (rt != net->ipv6.fib6_null_entry && !rt->nh &&  in fib6_ifup()
4730 	if (rt == net->ipv6.fib6_null_entry || rt->nh)  in fib6_ifdown()
4796 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()  argument
4806 	if (nh->fib_nh_dev == arg->dev) {  in fib6_nh_mtu_change()
4815 		rt6_exceptions_update_pmtu(idev, nh, arg->mtu);  in fib6_nh_mtu_change()
4841 	if (f6i->nh) {  in rt6_mtu_change_route()
4843 		return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,  in rt6_mtu_change_route()
5034 	struct rt6_nh *nh;  in ip6_route_info_append()  local
5037 	list_for_each_entry(nh, rt6_nh_list, next) {  in ip6_route_info_append()
5039 		if (rt6_duplicate_nexthop(nh->fib6_info, rt))  in ip6_route_info_append()
5043 	nh = kzalloc(sizeof(*nh), GFP_KERNEL);  in ip6_route_info_append()
5044 	if (!nh)  in ip6_route_info_append()
5046 	nh->fib6_info = rt;  in ip6_route_info_append()
5047 	memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));  in ip6_route_info_append()
5048 	list_add_tail(&nh->next, rt6_nh_list);  in ip6_route_info_append()
5109 	struct rt6_nh *nh, *nh_safe;  in ip6_route_multipath_add()  local
5194 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5195 		err = __ip6_ins_rt(nh->fib6_info, info, extack);  in ip6_route_multipath_add()
5196 		fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5200 			rt_last = nh->fib6_info;  in ip6_route_multipath_add()
5204 				rt_notif = nh->fib6_info;  in ip6_route_multipath_add()
5208 		nh->fib6_info = NULL;  in ip6_route_multipath_add()
5213 			err_nh = nh;  in ip6_route_multipath_add()
5266 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5267 		if (err_nh == nh)  in ip6_route_multipath_add()
5269 		ip6_route_del(&nh->r_cfg, extack);  in ip6_route_multipath_add()
5273 	list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5274 		if (nh->fib6_info)  in ip6_route_multipath_add()
5275 			fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5276 		list_del(&nh->next);  in ip6_route_multipath_add()
5277 		kfree(nh);  in ip6_route_multipath_add()
5366 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()  argument
5374 	if (nh->fib_nh_lws) {  in rt6_nh_nlmsg_size()
5376 		*nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nh_nlmsg_size()
5388 	if (f6i->nh) {  in rt6_nlmsg_size()
5390 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,  in rt6_nlmsg_size()
5393 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()  local
5400 				    + lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5404 		nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5423 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,  in rt6_fill_node_nexthop()  argument
5426 	if (nexthop_is_multipath(nh)) {  in rt6_fill_node_nexthop()
5433 		if (nexthop_mpath_fill_node(skb, nh, AF_INET6))  in rt6_fill_node_nexthop()
5440 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5582 	} else if (rt->nh) {  in rt6_fill_node()
5583 		if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))  in rt6_fill_node()
5586 		if (nexthop_is_blackhole(rt->nh))  in rt6_fill_node()
5590 		    rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)  in rt6_fill_node()
5629 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()  argument
5633 	if (nh->fib_nh_dev == dev)  in fib6_info_nh_uses_dev()
5642 	if (f6i->nh) {  in fib6_info_uses_dev()
5645 		return !!nexthop_for_each_fib6_nh(f6i->nh,  in fib6_info_uses_dev()
5674 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()  argument
5682 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in rt6_nh_dump_exceptions()
5777 		if (rt->nh) {  in rt6_dump_route()
5778 			err = nexthop_for_each_fib6_nh(rt->nh,  in rt6_dump_route()