103 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
453 	if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))  in fib6_select_path()
458 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()
472 	res->nh = match->fib6_nh;  in fib6_select_path()
479 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()
505 	struct fib6_nh		*nh;
508 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()
518 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()
544 	struct fib6_nh *nh;  in rt6_device_match()
552 			nh = f6i->fib6_nh;  in rt6_device_match()
567 			nh = spf6i->fib6_nh;  in rt6_device_match()
579 		nh = res->f6i->fib6_nh;  in rt6_device_match()
588 		nh = f6i->fib6_nh;  in rt6_device_match()
593 		nh = res->f6i->fib6_nh;  in rt6_device_match()
626 static void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
643 	if (!fib6_nh->fib_nh_gw_family)  in rt6_probe()
646 	nh_gw = &fib6_nh->fib_nh_gw6;  in rt6_probe()
647 	dev = fib6_nh->fib_nh_dev;  in rt6_probe()
649 	last_probe = READ_ONCE(fib6_nh->last_probe);  in rt6_probe()
670 	if (!work || cmpxchg(&fib6_nh->last_probe,  in rt6_probe()
685 static inline void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
693 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)  in rt6_check_neigh()  argument
699 	neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,  in rt6_check_neigh()
700 					  &fib6_nh->fib_nh_gw6);  in rt6_check_neigh()
721 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()
743 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()
785 	struct fib6_nh	*nh;
788 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()
808 		struct fib6_nh *nh;  in __find_rr_leaf()
840 			nh = f6i->fib6_nh;  in __find_rr_leaf()
926 		res->nh = res->f6i->fib6_nh;  in rt6_select()
1134 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()
1586 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()
1614 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()
1644 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()
1667 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()
1739 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()
1769 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()
1784 		fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);  in rt6_flush_exceptions()
1833 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()
1876 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()
1910 	return fib6_nh_remove_exception(from->fib6_nh,  in rt6_remove_exception_rt()
1917 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()
1943 	struct fib6_nh		*match;
1947 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()
1966 	struct fib6_nh *fib6_nh;  in rt6_update_exception_stamp_rt()  local
1984 		fib6_nh = arg.match;  in rt6_update_exception_stamp_rt()
1986 		fib6_nh = from->fib6_nh;  in rt6_update_exception_stamp_rt()
1988 	fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);  in rt6_update_exception_stamp_rt()
2016 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()
2044 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()
2121 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()
2155 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()
2176 		fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);  in rt6_age_exceptions()
2787 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2874 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()
2906 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()
2972 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
2993 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3127 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()
3416 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,  in fib6_nh_init()  argument
3425 	fib6_nh->fib_nh_family = AF_INET6;  in fib6_nh_init()
3427 	fib6_nh->last_probe = jiffies;  in fib6_nh_init()
3430 		fib6_nh->fib_nh_gw6 = cfg->fc_gateway;  in fib6_nh_init()
3431 		fib6_nh->fib_nh_gw_family = AF_INET6;  in fib6_nh_init()
3458 		fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;  in fib6_nh_init()
3461 	fib6_nh->fib_nh_weight = 1;  in fib6_nh_init()
3490 		fib6_nh->fib_nh_gw6 = cfg->fc_gateway;  in fib6_nh_init()
3491 		fib6_nh->fib_nh_gw_family = AF_INET6;  in fib6_nh_init()
3512 		fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_nh_init()
3514 	err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap,  in fib6_nh_init()
3520 	fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);  in fib6_nh_init()
3521 	if (!fib6_nh->rt6i_pcpu) {  in fib6_nh_init()
3526 	fib6_nh->fib_nh_dev = dev;  in fib6_nh_init()
3527 	fib6_nh->fib_nh_oif = dev->ifindex;  in fib6_nh_init()
3534 		lwtstate_put(fib6_nh->fib_nh_lws);  in fib6_nh_init()
3535 		fib6_nh->fib_nh_lws = NULL;  in fib6_nh_init()
3543 void fib6_nh_release(struct fib6_nh *fib6_nh)  in fib6_nh_release()  argument
3549 	fib6_nh_flush_exceptions(fib6_nh, NULL);  in fib6_nh_release()
3550 	bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);  in fib6_nh_release()
3552 		rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);  in fib6_nh_release()
3558 	if (fib6_nh->rt6i_pcpu) {  in fib6_nh_release()
3565 			ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);  in fib6_nh_release()
3574 		free_percpu(fib6_nh->rt6i_pcpu);  in fib6_nh_release()
3577 	fib_nh_common_release(&fib6_nh->nh_common);  in fib6_nh_release()
3588 	struct fib6_nh *fib6_nh;  in ip6_route_info_create()  local
3699 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3701 		err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);  in ip6_route_info_create()
3705 		fib6_nh = rt->fib6_nh;  in ip6_route_info_create()
3711 		if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,  in ip6_route_info_create()
3717 		struct net_device *dev = fib6_nh->fib_nh_dev;  in ip6_route_info_create()
3878 			     struct fib6_nh *nh)  in ip6_del_cached_rt()
3898 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()
3940 			struct fib6_nh *nh;  in ip6_route_del()
3954 					nh = rt->fib6_nh;  in ip6_route_del()
3980 			nh = rt->fib6_nh;  in ip6_route_del()
4114 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4171 		if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)  in rt6_get_route_info()
4174 		    !rt->fib6_nh->fib_nh_gw_family)  in rt6_get_route_info()
4176 		if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))  in rt6_get_route_info()
4234 		struct fib6_nh *nh;  in rt6_get_dflt_router()
4240 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4484 	    ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) &&  in fib6_remove_prefsrc()
4512 	struct fib6_nh *nh;  in fib6_clean_tohost()
4518 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4568 	if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||  in rt6_is_dead()
4569 	    (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in rt6_is_dead()
4570 	     ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))  in rt6_is_dead()
4582 		total += rt->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4586 			total += iter->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4597 		*weight += rt->fib6_nh->fib_nh_weight;  in rt6_upper_bound_set()
4601 	atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);  in rt6_upper_bound_set()
4645 	    rt->fib6_nh->fib_nh_dev == arg->dev) {  in fib6_ifup()
4646 		rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;  in fib6_ifup()
4675 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4678 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4699 	if (rt->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4700 	    rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4703 		if (iter->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4704 		    iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4716 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4717 		rt->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4719 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4720 			iter->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4735 		return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4740 			return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4756 		if (rt->fib6_nh->fib_nh_dev != dev ||  in fib6_ifdown()
4759 		rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_ifdown()
4796 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()
4847 	return fib6_nh_mtu_change(f6i->fib6_nh, arg);  in rt6_mtu_change_route()
5164 		rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;  in ip6_route_multipath_add()
5366 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()
5393 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()
5438 		struct fib6_nh *fib6_nh;  in rt6_fill_node_nexthop()  local
5440 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5441 		if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,  in rt6_fill_node_nexthop()
5569 		if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,  in rt6_fill_node()
5570 				    rt->fib6_nh->fib_nh_weight, AF_INET6) < 0)  in rt6_fill_node()
5575 			if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,  in rt6_fill_node()
5576 					    sibling->fib6_nh->fib_nh_weight,  in rt6_fill_node()
5595 		if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,  in rt6_fill_node()
5629 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()
5650 	if (f6i->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5658 			if (sibling->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5674 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()
5782 			err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);  in rt6_dump_route()
6077 		net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;  in ip6_route_dev_notify()
6414 	init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;  in ip6_route_init_special_entries()