106 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
422 	if (!match->nh && (!match->fib6_nsiblings || have_oif_match))  in fib6_select_path()
425 	if (match->nh && have_oif_match && res->nh)  in fib6_select_path()
432 	    (!match->nh || nexthop_is_multipath(match->nh)))  in fib6_select_path()
435 	if (unlikely(match->nh)) {  in fib6_select_path()
445 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()  local
448 		nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);  in fib6_select_path()
451 		if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)  in fib6_select_path()
459 	res->nh = match->fib6_nh;  in fib6_select_path()
466 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()  argument
471 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in __rt6_device_match()
474 	dev = nh->fib_nh_dev;  in __rt6_device_match()
492 	struct fib6_nh		*nh;  member
495 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()  argument
499 	arg->nh = nh;  in __rt6_nh_dev_match()
500 	return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,  in __rt6_nh_dev_match()
505 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()  argument
517 	if (nexthop_is_blackhole(nh))  in rt6_nh_dev_match()
520 	if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))  in rt6_nh_dev_match()
521 		return arg.nh;  in rt6_nh_dev_match()
531 	struct fib6_nh *nh;  in rt6_device_match()  local
534 		if (unlikely(f6i->nh)) {  in rt6_device_match()
535 			nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
536 			if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
539 			nh = f6i->fib6_nh;  in rt6_device_match()
541 		if (!(nh->fib_nh_flags & RTNH_F_DEAD))  in rt6_device_match()
548 		if (unlikely(spf6i->nh)) {  in rt6_device_match()
549 			nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,  in rt6_device_match()
551 			if (nh)  in rt6_device_match()
554 			nh = spf6i->fib6_nh;  in rt6_device_match()
555 			if (__rt6_device_match(net, nh, saddr, oif, flags))  in rt6_device_match()
566 		nh = res->f6i->fib6_nh;  in rt6_device_match()
570 	if (unlikely(f6i->nh)) {  in rt6_device_match()
571 		nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
572 		if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
575 		nh = f6i->fib6_nh;  in rt6_device_match()
578 	if (nh->fib_nh_flags & RTNH_F_DEAD) {  in rt6_device_match()
580 		nh = res->f6i->fib6_nh;  in rt6_device_match()
583 	res->nh = nh;  in rt6_device_match()
591 	res->nh = nh;  in rt6_device_match()
709 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()  argument
714 	if (!oif || nh->fib_nh_dev->ifindex == oif)  in rt6_score_route()
723 	    !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {  in rt6_score_route()
724 		int n = rt6_check_neigh(nh);  in rt6_score_route()
731 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()  argument
738 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in find_match()
741 	if (ip6_ignore_linkdown(nh->fib_nh_dev) &&  in find_match()
742 	    nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in find_match()
746 	m = rt6_score_route(nh, fib6_flags, oif, strict);  in find_match()
755 		rt6_probe(nh);  in find_match()
773 	struct fib6_nh	*nh;  member
776 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()  argument
780 	arg->nh = nh;  in rt6_nh_find_match()
781 	return find_match(nh, arg->flags, arg->oif, arg->strict,  in rt6_nh_find_match()
796 		struct fib6_nh *nh;  in __find_rr_leaf()  local
806 		if (unlikely(f6i->nh)) {  in __find_rr_leaf()
815 			if (nexthop_is_blackhole(f6i->nh)) {  in __find_rr_leaf()
819 				res->nh = nexthop_fib6_nh(f6i->nh);  in __find_rr_leaf()
822 			if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,  in __find_rr_leaf()
825 				nh = arg.nh;  in __find_rr_leaf()
828 			nh = f6i->fib6_nh;  in __find_rr_leaf()
829 			if (find_match(nh, f6i->fib6_flags, oif, strict,  in __find_rr_leaf()
835 			res->nh = nh;  in __find_rr_leaf()
914 		res->nh = res->f6i->fib6_nh;  in rt6_select()
923 	       res->nh->fib_nh_gw_family;  in rt6_is_gw_or_nonexthop()
1009 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_rt_get_dev_rcu()
1103 	if (res->nh->fib_nh_lws) {  in ip6_rt_init_dst()
1104 		rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);  in ip6_rt_init_dst()
1122 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()  local
1123 	const struct net_device *dev = nh->fib_nh_dev;  in ip6_rt_copy_init()
1131 	if (nh->fib_nh_gw_family) {  in ip6_rt_copy_init()
1132 		rt->rt6i_gateway = nh->fib_nh_gw6;  in ip6_rt_copy_init()
1178 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_create_rt_rcu()
1378 	if (f6i->nh)  in ip6_rt_pcpu_alloc()
1394 	pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1399 		p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1421 	p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_make_pcpu_route()
1578 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()  local
1584 		struct net_device *dev = nh->fib_nh_dev;  in fib6_mtu()
1595 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in fib6_mtu()
1606 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()  argument
1612 		bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_get_excptn_bucket()
1615 		bucket = rcu_dereference(nh->rt6i_exception_bucket);  in fib6_nh_get_excptn_bucket()
1636 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()  argument
1642 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_excptn_bucket_set_flushed()
1648 	rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in fib6_nh_excptn_bucket_set_flushed()
1659 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()  local
1665 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in rt6_insert_exception()
1674 		rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in rt6_insert_exception()
1734 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()  argument
1743 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1749 		fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1764 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()  argument
1768 	fib6_nh_flush_exceptions(nh, f6i);  in rt6_nh_flush_exceptions()
1775 	if (f6i->nh)  in rt6_flush_exceptions()
1776 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,  in rt6_flush_exceptions()
1810 	bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);  in rt6_find_cached_rt()
1828 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()  argument
1836 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_remove_exception()
1840 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_remove_exception()
1871 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()  argument
1876 	err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);  in rt6_nh_remove_exception_rt()
1891 	if (from->nh) {  in rt6_remove_exception_rt()
1899 		rc = nexthop_for_each_fib6_nh(from->nh,  in rt6_remove_exception_rt()
1912 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()  argument
1919 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in fib6_nh_update_exception()
1942 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()  argument
1946 	if (arg->dev != nh->fib_nh_dev ||  in fib6_nh_find_match()
1947 	    (arg->gw && !nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1948 	    (!arg->gw && nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1949 	    (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))  in fib6_nh_find_match()
1952 	arg->match = nh;  in fib6_nh_find_match()
1969 	if (from->nh) {  in rt6_update_exception_stamp_rt()
1975 		nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);  in rt6_update_exception_stamp_rt()
2011 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()  argument
2017 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in rt6_exceptions_update_pmtu()
2039 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()  argument
2047 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_exceptions_clean_tohost()
2051 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_exceptions_clean_tohost()
2113 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()  argument
2122 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_age_exceptions()
2127 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_age_exceptions()
2147 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()  argument
2151 	fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);  in rt6_nh_age_exceptions()
2159 	if (f6i->nh) {  in rt6_age_exceptions()
2165 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,  in rt6_age_exceptions()
2229 			    !res.nh->fib_nh_gw_family)) {  in ip6_pol_route()
2887 		if (res.f6i->nh) {  in __ip6_rt_update_pmtu()
2893 			nexthop_for_each_fib6_nh(res.f6i->nh,  in __ip6_rt_update_pmtu()
2902 			res.nh = arg.match;  in __ip6_rt_update_pmtu()
2904 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2991 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()  local
2993 	if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||  in ip6_redirect_nh_match()
2994 	    fl6->flowi6_oif != nh->fib_nh_dev->ifindex)  in ip6_redirect_nh_match()
3002 	if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {  in ip6_redirect_nh_match()
3023 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()  argument
3027 	arg->res->nh = nh;  in fib6_nh_redirect_match()
3074 		if (unlikely(rt->nh)) {  in __ip6_route_redirect()
3075 			if (nexthop_is_blackhole(rt->nh))  in __ip6_route_redirect()
3077 			/* on match, res->nh is filled in and potentially ret */  in __ip6_route_redirect()
3078 			if (nexthop_for_each_fib6_nh(rt->nh,  in __ip6_route_redirect()
3083 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3104 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3221 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()  local
3237 		struct net_device *dev = nh->fib_nh_dev;  in ip6_mtu_from_fib6()
3247 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in ip6_mtu_from_fib6()
3359 	    (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {  in ip6_route_check_nh_onlink()
3386 		    res.nh->fib_nh_gw_family ||  in ip6_route_check_nh()
3387 		    (dev && dev != res.nh->fib_nh_dev))  in ip6_route_check_nh()
3399 		    res.nh->fib_nh_gw_family)  in ip6_route_check_nh()
3411 		if (dev != res.nh->fib_nh_dev)  in ip6_route_check_nh()
3414 		*_dev = dev = res.nh->fib_nh_dev;  in ip6_route_check_nh()
3684 	struct nexthop *nh = NULL;  in ip6_route_info_create()  local
3723 		nh = nexthop_find_by_id(net, cfg->fc_nh_id);  in ip6_route_info_create()
3724 		if (!nh) {  in ip6_route_info_create()
3728 		err = fib6_check_nexthop(nh, cfg, extack);  in ip6_route_info_create()
3749 	rt = fib6_info_alloc(gfp_flags, !nh);  in ip6_route_info_create()
3787 	if (nh) {  in ip6_route_info_create()
3792 		if (!nexthop_get(nh)) {  in ip6_route_info_create()
3796 		rt->nh = nh;  in ip6_route_info_create()
3797 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3980 			     struct fib6_nh *nh)  in ip6_del_cached_rt()  argument
3984 		.nh = nh,  in ip6_del_cached_rt()
4000 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()  argument
4005 	rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);  in fib6_nh_del_cached_rt()
4016 	return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);  in ip6_del_cached_rt_nh()
4042 			struct fib6_nh *nh;  in ip6_route_del()  local
4044 			if (rt->nh && cfg->fc_nh_id &&  in ip6_route_del()
4045 			    rt->nh->id != cfg->fc_nh_id)  in ip6_route_del()
4051 				if (rt->nh) {  in ip6_route_del()
4056 					nh = rt->fib6_nh;  in ip6_route_del()
4057 					rc = ip6_del_cached_rt(cfg, rt, nh);  in ip6_route_del()
4072 			if (rt->nh) {  in ip6_route_del()
4082 			nh = rt->fib6_nh;  in ip6_route_del()
4084 			    (!nh->fib_nh_dev ||  in ip6_route_del()
4085 			     nh->fib_nh_dev->ifindex != cfg->fc_ifindex))  in ip6_route_del()
4088 			    !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))  in ip6_route_del()
4200 	if (res.f6i->nh) {  in rt6_do_redirect()
4206 		nexthop_for_each_fib6_nh(res.f6i->nh,  in rt6_do_redirect()
4214 		res.nh = arg.match;  in rt6_do_redirect()
4216 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4271 		if (rt->nh)  in rt6_get_route_info()
4336 		struct fib6_nh *nh;  in rt6_get_dflt_router()  local
4339 		if (rt->nh)  in rt6_get_dflt_router()
4342 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4343 		if (dev == nh->fib_nh_dev &&  in rt6_get_dflt_router()
4345 		    ipv6_addr_equal(&nh->fib_nh_gw6, addr))  in rt6_get_dflt_router()
4597 	if (!rt->nh &&  in fib6_remove_prefsrc()
4626 	struct fib6_nh *nh;  in fib6_clean_tohost()  local
4629 	if (rt->nh)  in fib6_clean_tohost()
4632 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4634 	    nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))  in fib6_clean_tohost()
4641 	fib6_nh_exceptions_clean_tohost(nh, gateway);  in fib6_clean_tohost()
4758 	if (rt != net->ipv6.fib6_null_entry && !rt->nh &&  in fib6_ifup()
4844 	if (rt == net->ipv6.fib6_null_entry || rt->nh)  in fib6_ifdown()
4910 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()  argument
4920 	if (nh->fib_nh_dev == arg->dev) {  in fib6_nh_mtu_change()
4929 		rt6_exceptions_update_pmtu(idev, nh, arg->mtu);  in fib6_nh_mtu_change()
4955 	if (f6i->nh) {  in rt6_mtu_change_route()
4957 		return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,  in rt6_mtu_change_route()
5154 	struct rt6_nh *nh;  in ip6_route_info_append()  local
5157 	list_for_each_entry(nh, rt6_nh_list, next) {  in ip6_route_info_append()
5159 		if (rt6_duplicate_nexthop(nh->fib6_info, rt))  in ip6_route_info_append()
5163 	nh = kzalloc(sizeof(*nh), GFP_KERNEL);  in ip6_route_info_append()
5164 	if (!nh)  in ip6_route_info_append()
5166 	nh->fib6_info = rt;  in ip6_route_info_append()
5167 	memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));  in ip6_route_info_append()
5168 	list_add_tail(&nh->next, rt6_nh_list);  in ip6_route_info_append()
5242 	struct rt6_nh *nh, *nh_safe;  in ip6_route_multipath_add()  local
5335 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5336 		err = __ip6_ins_rt(nh->fib6_info, info, extack);  in ip6_route_multipath_add()
5337 		fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5341 			rt_last = nh->fib6_info;  in ip6_route_multipath_add()
5345 				rt_notif = nh->fib6_info;  in ip6_route_multipath_add()
5348 		/* nh->fib6_info is used or freed at this point, reset to NULL*/  in ip6_route_multipath_add()
5349 		nh->fib6_info = NULL;  in ip6_route_multipath_add()
5354 			err_nh = nh;  in ip6_route_multipath_add()
5409 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5410 		if (err_nh == nh)  in ip6_route_multipath_add()
5412 		ip6_route_del(&nh->r_cfg, extack);  in ip6_route_multipath_add()
5416 	list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5417 		if (nh->fib6_info)  in ip6_route_multipath_add()
5418 			fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5419 		list_del(&nh->next);  in ip6_route_multipath_add()
5420 		kfree(nh);  in ip6_route_multipath_add()
5516 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()  argument
5524 	if (nh->fib_nh_lws) {  in rt6_nh_nlmsg_size()
5526 		*nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nh_nlmsg_size()
5538 	if (f6i->nh) {  in rt6_nlmsg_size()
5540 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,  in rt6_nlmsg_size()
5543 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()  local
5550 				    + lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5554 		nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5573 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,  in rt6_fill_node_nexthop()  argument
5576 	if (nexthop_is_multipath(nh)) {  in rt6_fill_node_nexthop()
5583 		if (nexthop_mpath_fill_node(skb, nh, AF_INET6))  in rt6_fill_node_nexthop()
5590 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5737 	} else if (rt->nh) {  in rt6_fill_node()
5738 		if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))  in rt6_fill_node()
5741 		if (nexthop_is_blackhole(rt->nh))  in rt6_fill_node()
5745 		    rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)  in rt6_fill_node()
5786 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()  argument
5790 	if (nh->fib_nh_dev == dev)  in fib6_info_nh_uses_dev()
5799 	if (f6i->nh) {  in fib6_info_uses_dev()
5802 		return !!nexthop_for_each_fib6_nh(f6i->nh,  in fib6_info_uses_dev()
5831 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()  argument
5839 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in rt6_nh_dump_exceptions()
5934 		if (rt->nh) {  in rt6_dump_route()
5935 			err = nexthop_for_each_fib6_nh(rt->nh,  in rt6_dump_route()