106 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
420 	if (!match->nh && (!match->fib6_nsiblings || have_oif_match))  in fib6_select_path()
423 	if (match->nh && have_oif_match && res->nh)  in fib6_select_path()
433 	    (!match->nh || nexthop_is_multipath(match->nh)))  in fib6_select_path()
436 	if (unlikely(match->nh)) {  in fib6_select_path()
446 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()  local
449 		nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);  in fib6_select_path()
452 		if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)  in fib6_select_path()
460 	res->nh = match->fib6_nh;  in fib6_select_path()
467 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()  argument
472 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in __rt6_device_match()
475 	dev = nh->fib_nh_dev;  in __rt6_device_match()
493 	struct fib6_nh		*nh;  member
496 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()  argument
500 	arg->nh = nh;  in __rt6_nh_dev_match()
501 	return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,  in __rt6_nh_dev_match()
506 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()  argument
518 	if (nexthop_is_blackhole(nh))  in rt6_nh_dev_match()
521 	if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))  in rt6_nh_dev_match()
522 		return arg.nh;  in rt6_nh_dev_match()
532 	struct fib6_nh *nh;  in rt6_device_match()  local
535 		if (unlikely(f6i->nh)) {  in rt6_device_match()
536 			nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
537 			if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
540 			nh = f6i->fib6_nh;  in rt6_device_match()
542 		if (!(nh->fib_nh_flags & RTNH_F_DEAD))  in rt6_device_match()
549 		if (unlikely(spf6i->nh)) {  in rt6_device_match()
550 			nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,  in rt6_device_match()
552 			if (nh)  in rt6_device_match()
555 			nh = spf6i->fib6_nh;  in rt6_device_match()
556 			if (__rt6_device_match(net, nh, saddr, oif, flags))  in rt6_device_match()
567 		nh = res->f6i->fib6_nh;  in rt6_device_match()
571 	if (unlikely(f6i->nh)) {  in rt6_device_match()
572 		nh = nexthop_fib6_nh(f6i->nh);  in rt6_device_match()
573 		if (nexthop_is_blackhole(f6i->nh))  in rt6_device_match()
576 		nh = f6i->fib6_nh;  in rt6_device_match()
579 	if (nh->fib_nh_flags & RTNH_F_DEAD) {  in rt6_device_match()
581 		nh = res->f6i->fib6_nh;  in rt6_device_match()
584 	res->nh = nh;  in rt6_device_match()
592 	res->nh = nh;  in rt6_device_match()
710 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()  argument
715 	if (!oif || nh->fib_nh_dev->ifindex == oif)  in rt6_score_route()
724 	    !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {  in rt6_score_route()
725 		int n = rt6_check_neigh(nh);  in rt6_score_route()
732 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()  argument
739 	if (nh->fib_nh_flags & RTNH_F_DEAD)  in find_match()
742 	if (ip6_ignore_linkdown(nh->fib_nh_dev) &&  in find_match()
743 	    nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in find_match()
747 	m = rt6_score_route(nh, fib6_flags, oif, strict);  in find_match()
756 		rt6_probe(nh);  in find_match()
774 	struct fib6_nh	*nh;  member
777 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()  argument
781 	arg->nh = nh;  in rt6_nh_find_match()
782 	return find_match(nh, arg->flags, arg->oif, arg->strict,  in rt6_nh_find_match()
797 		struct fib6_nh *nh;  in __find_rr_leaf()  local
807 		if (unlikely(f6i->nh)) {  in __find_rr_leaf()
816 			if (nexthop_is_blackhole(f6i->nh)) {  in __find_rr_leaf()
820 				res->nh = nexthop_fib6_nh(f6i->nh);  in __find_rr_leaf()
823 			if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,  in __find_rr_leaf()
826 				nh = arg.nh;  in __find_rr_leaf()
829 			nh = f6i->fib6_nh;  in __find_rr_leaf()
830 			if (find_match(nh, f6i->fib6_flags, oif, strict,  in __find_rr_leaf()
836 			res->nh = nh;  in __find_rr_leaf()
915 		res->nh = res->f6i->fib6_nh;  in rt6_select()
924 	       res->nh->fib_nh_gw_family;  in rt6_is_gw_or_nonexthop()
1010 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_rt_get_dev_rcu()
1104 	if (res->nh->fib_nh_lws) {  in ip6_rt_init_dst()
1105 		rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);  in ip6_rt_init_dst()
1123 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()  local
1124 	const struct net_device *dev = nh->fib_nh_dev;  in ip6_rt_copy_init()
1132 	if (nh->fib_nh_gw_family) {  in ip6_rt_copy_init()
1133 		rt->rt6i_gateway = nh->fib_nh_gw6;  in ip6_rt_copy_init()
1179 	struct net_device *dev = res->nh->fib_nh_dev;  in ip6_create_rt_rcu()
1379 	if (f6i->nh)  in ip6_rt_pcpu_alloc()
1395 	pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1400 		p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_get_pcpu_route()
1422 	p = this_cpu_ptr(res->nh->rt6i_pcpu);  in rt6_make_pcpu_route()
1579 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()  local
1585 		struct net_device *dev = nh->fib_nh_dev;  in fib6_mtu()
1596 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in fib6_mtu()
1607 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()  argument
1613 		bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_get_excptn_bucket()
1616 		bucket = rcu_dereference(nh->rt6i_exception_bucket);  in fib6_nh_get_excptn_bucket()
1637 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()  argument
1643 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in fib6_nh_excptn_bucket_set_flushed()
1649 	rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in fib6_nh_excptn_bucket_set_flushed()
1660 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()  local
1666 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,  in rt6_insert_exception()
1675 		rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);  in rt6_insert_exception()
1735 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()  argument
1744 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1750 		fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);  in fib6_nh_flush_exceptions()
1765 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()  argument
1769 	fib6_nh_flush_exceptions(nh, f6i);  in rt6_nh_flush_exceptions()
1776 	if (f6i->nh)  in rt6_flush_exceptions()
1777 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,  in rt6_flush_exceptions()
1811 	bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);  in rt6_find_cached_rt()
1829 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()  argument
1837 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_remove_exception()
1841 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_remove_exception()
1872 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()  argument
1877 	err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);  in rt6_nh_remove_exception_rt()
1892 	if (from->nh) {  in rt6_remove_exception_rt()
1900 		rc = nexthop_for_each_fib6_nh(from->nh,  in rt6_remove_exception_rt()
1913 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()  argument
1920 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in fib6_nh_update_exception()
1943 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()  argument
1947 	if (arg->dev != nh->fib_nh_dev ||  in fib6_nh_find_match()
1948 	    (arg->gw && !nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1949 	    (!arg->gw && nh->fib_nh_gw_family) ||  in fib6_nh_find_match()
1950 	    (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))  in fib6_nh_find_match()
1953 	arg->match = nh;  in fib6_nh_find_match()
1970 	if (from->nh) {  in rt6_update_exception_stamp_rt()
1976 		nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);  in rt6_update_exception_stamp_rt()
2012 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()  argument
2018 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in rt6_exceptions_update_pmtu()
2040 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()  argument
2048 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_exceptions_clean_tohost()
2052 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_exceptions_clean_tohost()
2114 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()  argument
2123 	if (!rcu_access_pointer(nh->rt6i_exception_bucket))  in fib6_nh_age_exceptions()
2128 	bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);  in fib6_nh_age_exceptions()
2148 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()  argument
2152 	fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);  in rt6_nh_age_exceptions()
2160 	if (f6i->nh) {  in rt6_age_exceptions()
2166 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,  in rt6_age_exceptions()
2230 			    !res.nh->fib_nh_gw_family)) {  in ip6_pol_route()
2888 		if (res.f6i->nh) {  in __ip6_rt_update_pmtu()
2894 			nexthop_for_each_fib6_nh(res.f6i->nh,  in __ip6_rt_update_pmtu()
2903 			res.nh = arg.match;  in __ip6_rt_update_pmtu()
2905 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2993 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()  local
2995 	if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||  in ip6_redirect_nh_match()
2996 	    fl6->flowi6_oif != nh->fib_nh_dev->ifindex)  in ip6_redirect_nh_match()
3004 	if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {  in ip6_redirect_nh_match()
3025 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()  argument
3029 	arg->res->nh = nh;  in fib6_nh_redirect_match()
3076 		if (unlikely(rt->nh)) {  in __ip6_route_redirect()
3077 			if (nexthop_is_blackhole(rt->nh))  in __ip6_route_redirect()
3080 			if (nexthop_for_each_fib6_nh(rt->nh,  in __ip6_route_redirect()
3085 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3106 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3223 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()  local
3239 		struct net_device *dev = nh->fib_nh_dev;  in ip6_mtu_from_fib6()
3249 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);  in ip6_mtu_from_fib6()
3354 	    (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {  in ip6_route_check_nh_onlink()
3382 		    res.nh->fib_nh_gw_family ||  in ip6_route_check_nh()
3383 		    (dev && dev != res.nh->fib_nh_dev))  in ip6_route_check_nh()
3395 		    res.nh->fib_nh_gw_family)  in ip6_route_check_nh()
3407 		if (dev != res.nh->fib_nh_dev)  in ip6_route_check_nh()
3410 		*_dev = dev = res.nh->fib_nh_dev;  in ip6_route_check_nh()
3684 	struct nexthop *nh = NULL;  in ip6_route_info_create()  local
3723 		nh = nexthop_find_by_id(net, cfg->fc_nh_id);  in ip6_route_info_create()
3724 		if (!nh) {  in ip6_route_info_create()
3728 		err = fib6_check_nexthop(nh, cfg, extack);  in ip6_route_info_create()
3749 	rt = fib6_info_alloc(gfp_flags, !nh);  in ip6_route_info_create()
3787 	if (nh) {  in ip6_route_info_create()
3792 		if (!nexthop_get(nh)) {  in ip6_route_info_create()
3796 		rt->nh = nh;  in ip6_route_info_create()
3797 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3980 			     struct fib6_nh *nh)  in ip6_del_cached_rt()  argument
3984 		.nh = nh,  in ip6_del_cached_rt()
4000 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()  argument
4005 	rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);  in fib6_nh_del_cached_rt()
4016 	return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);  in ip6_del_cached_rt_nh()
4042 			struct fib6_nh *nh;  in ip6_route_del()  local
4044 			if (rt->nh && cfg->fc_nh_id &&  in ip6_route_del()
4045 			    rt->nh->id != cfg->fc_nh_id)  in ip6_route_del()
4051 				if (rt->nh) {  in ip6_route_del()
4056 					nh = rt->fib6_nh;  in ip6_route_del()
4057 					rc = ip6_del_cached_rt(cfg, rt, nh);  in ip6_route_del()
4072 			if (rt->nh) {  in ip6_route_del()
4082 			nh = rt->fib6_nh;  in ip6_route_del()
4084 			    (!nh->fib_nh_dev ||  in ip6_route_del()
4085 			     nh->fib_nh_dev->ifindex != cfg->fc_ifindex))  in ip6_route_del()
4088 			    !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))  in ip6_route_del()
4200 	if (res.f6i->nh) {  in rt6_do_redirect()
4206 		nexthop_for_each_fib6_nh(res.f6i->nh,  in rt6_do_redirect()
4214 		res.nh = arg.match;  in rt6_do_redirect()
4216 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4271 		if (rt->nh)  in rt6_get_route_info()
4336 		struct fib6_nh *nh;  in rt6_get_dflt_router()  local
4339 		if (rt->nh)  in rt6_get_dflt_router()
4342 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4343 		if (dev == nh->fib_nh_dev &&  in rt6_get_dflt_router()
4345 		    ipv6_addr_equal(&nh->fib_nh_gw6, addr))  in rt6_get_dflt_router()
4596 	if (!rt->nh &&  in fib6_remove_prefsrc()
4624 	struct fib6_nh *nh;  in fib6_clean_tohost()  local
4627 	if (rt->nh)  in fib6_clean_tohost()
4630 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4632 	    nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))  in fib6_clean_tohost()
4639 	fib6_nh_exceptions_clean_tohost(nh, gateway);  in fib6_clean_tohost()
4756 	if (rt != net->ipv6.fib6_null_entry && !rt->nh &&  in fib6_ifup()
4842 	if (rt == net->ipv6.fib6_null_entry || rt->nh)  in fib6_ifdown()
4908 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()  argument
4918 	if (nh->fib_nh_dev == arg->dev) {  in fib6_nh_mtu_change()
4927 		rt6_exceptions_update_pmtu(idev, nh, arg->mtu);  in fib6_nh_mtu_change()
4953 	if (f6i->nh) {  in rt6_mtu_change_route()
4955 		return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,  in rt6_mtu_change_route()
5152 	struct rt6_nh *nh;  in ip6_route_info_append()  local
5155 	list_for_each_entry(nh, rt6_nh_list, next) {  in ip6_route_info_append()
5157 		if (rt6_duplicate_nexthop(nh->fib6_info, rt))  in ip6_route_info_append()
5161 	nh = kzalloc(sizeof(*nh), GFP_KERNEL);  in ip6_route_info_append()
5162 	if (!nh)  in ip6_route_info_append()
5164 	nh->fib6_info = rt;  in ip6_route_info_append()
5165 	memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));  in ip6_route_info_append()
5166 	list_add_tail(&nh->next, rt6_nh_list);  in ip6_route_info_append()
5240 	struct rt6_nh *nh, *nh_safe;  in ip6_route_multipath_add()  local
5333 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5334 		err = __ip6_ins_rt(nh->fib6_info, info, extack);  in ip6_route_multipath_add()
5335 		fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5339 			rt_last = nh->fib6_info;  in ip6_route_multipath_add()
5343 				rt_notif = nh->fib6_info;  in ip6_route_multipath_add()
5347 		nh->fib6_info = NULL;  in ip6_route_multipath_add()
5352 			err_nh = nh;  in ip6_route_multipath_add()
5407 	list_for_each_entry(nh, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5408 		if (err_nh == nh)  in ip6_route_multipath_add()
5410 		ip6_route_del(&nh->r_cfg, extack);  in ip6_route_multipath_add()
5414 	list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {  in ip6_route_multipath_add()
5415 		if (nh->fib6_info)  in ip6_route_multipath_add()
5416 			fib6_info_release(nh->fib6_info);  in ip6_route_multipath_add()
5417 		list_del(&nh->next);  in ip6_route_multipath_add()
5418 		kfree(nh);  in ip6_route_multipath_add()
5514 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()  argument
5522 	if (nh->fib_nh_lws) {  in rt6_nh_nlmsg_size()
5524 		*nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nh_nlmsg_size()
5536 	if (f6i->nh) {  in rt6_nlmsg_size()
5538 		nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,  in rt6_nlmsg_size()
5542 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()  local
5546 			rt6_nh_nlmsg_size(nh, &nexthop_len);  in rt6_nlmsg_size()
5553 		nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);  in rt6_nlmsg_size()
5572 static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,  in rt6_fill_node_nexthop()  argument
5575 	if (nexthop_is_multipath(nh)) {  in rt6_fill_node_nexthop()
5582 		if (nexthop_mpath_fill_node(skb, nh, AF_INET6))  in rt6_fill_node_nexthop()
5589 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5736 	} else if (rt->nh) {  in rt6_fill_node()
5737 		if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))  in rt6_fill_node()
5740 		if (nexthop_is_blackhole(rt->nh))  in rt6_fill_node()
5744 		    rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)  in rt6_fill_node()
5785 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()  argument
5789 	if (nh->fib_nh_dev == dev)  in fib6_info_nh_uses_dev()
5798 	if (f6i->nh) {  in fib6_info_uses_dev()
5801 		return !!nexthop_for_each_fib6_nh(f6i->nh,  in fib6_info_uses_dev()
5830 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()  argument
5838 	bucket = fib6_nh_get_excptn_bucket(nh, NULL);  in rt6_nh_dump_exceptions()
5933 		if (rt->nh) {  in rt6_dump_route()
5934 			err = nexthop_for_each_fib6_nh(rt->nh,  in rt6_dump_route()