101 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
447 	if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))  in fib6_select_path()
452 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()
466 	res->nh = match->fib6_nh;  in fib6_select_path()
473 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()
499 	struct fib6_nh		*nh;
502 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()
512 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()
538 	struct fib6_nh *nh;  in rt6_device_match()
546 			nh = f6i->fib6_nh;  in rt6_device_match()
561 			nh = spf6i->fib6_nh;  in rt6_device_match()
573 		nh = res->f6i->fib6_nh;  in rt6_device_match()
582 		nh = f6i->fib6_nh;  in rt6_device_match()
587 		nh = res->f6i->fib6_nh;  in rt6_device_match()
620 static void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
637 	if (!fib6_nh->fib_nh_gw_family)  in rt6_probe()
640 	nh_gw = &fib6_nh->fib_nh_gw6;  in rt6_probe()
641 	dev = fib6_nh->fib_nh_dev;  in rt6_probe()
643 	last_probe = READ_ONCE(fib6_nh->last_probe);  in rt6_probe()
664 	if (!work || cmpxchg(&fib6_nh->last_probe,  in rt6_probe()
679 static inline void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
687 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)  in rt6_check_neigh()  argument
693 	neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,  in rt6_check_neigh()
694 					  &fib6_nh->fib_nh_gw6);  in rt6_check_neigh()
715 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()
737 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()
779 	struct fib6_nh	*nh;
782 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()
802 		struct fib6_nh *nh;  in __find_rr_leaf()
834 			nh = f6i->fib6_nh;  in __find_rr_leaf()
920 		res->nh = res->f6i->fib6_nh;  in rt6_select()
1130 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()
1561 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()
1589 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()
1619 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()
1642 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()
1714 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()
1744 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()
1759 		fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);  in rt6_flush_exceptions()
1808 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()
1851 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()
1885 	return fib6_nh_remove_exception(from->fib6_nh,  in rt6_remove_exception_rt()
1892 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()
1918 	struct fib6_nh		*match;
1922 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()
1941 	struct fib6_nh *fib6_nh;  in rt6_update_exception_stamp_rt()  local
1959 		fib6_nh = arg.match;  in rt6_update_exception_stamp_rt()
1961 		fib6_nh = from->fib6_nh;  in rt6_update_exception_stamp_rt()
1963 	fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);  in rt6_update_exception_stamp_rt()
1991 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()
2019 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()
2096 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()
2130 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()
2151 		fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);  in rt6_age_exceptions()
2755 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2840 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()
2872 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()
2938 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
2959 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3093 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()
3380 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,  in fib6_nh_init()  argument
3389 	fib6_nh->fib_nh_family = AF_INET6;  in fib6_nh_init()
3391 	fib6_nh->last_probe = jiffies;  in fib6_nh_init()
3417 		fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;  in fib6_nh_init()
3420 	fib6_nh->fib_nh_weight = 1;  in fib6_nh_init()
3449 		fib6_nh->fib_nh_gw6 = cfg->fc_gateway;  in fib6_nh_init()
3450 		fib6_nh->fib_nh_gw_family = AF_INET6;  in fib6_nh_init()
3471 		fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_nh_init()
3473 	err = fib_nh_common_init(&fib6_nh->nh_common, cfg->fc_encap,  in fib6_nh_init()
3479 	fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);  in fib6_nh_init()
3480 	if (!fib6_nh->rt6i_pcpu) {  in fib6_nh_init()
3485 	fib6_nh->fib_nh_dev = dev;  in fib6_nh_init()
3486 	fib6_nh->fib_nh_oif = dev->ifindex;  in fib6_nh_init()
3493 		lwtstate_put(fib6_nh->fib_nh_lws);  in fib6_nh_init()
3494 		fib6_nh->fib_nh_lws = NULL;  in fib6_nh_init()
3502 void fib6_nh_release(struct fib6_nh *fib6_nh)  in fib6_nh_release()  argument
3508 	fib6_nh_flush_exceptions(fib6_nh, NULL);  in fib6_nh_release()
3509 	bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);  in fib6_nh_release()
3511 		rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);  in fib6_nh_release()
3517 	if (fib6_nh->rt6i_pcpu) {  in fib6_nh_release()
3524 			ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);  in fib6_nh_release()
3533 		free_percpu(fib6_nh->rt6i_pcpu);  in fib6_nh_release()
3536 	fib_nh_common_release(&fib6_nh->nh_common);  in fib6_nh_release()
3547 	struct fib6_nh *fib6_nh;  in ip6_route_info_create()  local
3660 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3662 		err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);  in ip6_route_info_create()
3666 		fib6_nh = rt->fib6_nh;  in ip6_route_info_create()
3672 		if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,  in ip6_route_info_create()
3678 		struct net_device *dev = fib6_nh->fib_nh_dev;  in ip6_route_info_create()
3815 			     struct fib6_nh *nh)  in ip6_del_cached_rt()
3835 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()
3877 			struct fib6_nh *nh;  in ip6_route_del()
3891 					nh = rt->fib6_nh;  in ip6_route_del()
3917 			nh = rt->fib6_nh;  in ip6_route_del()
4051 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4108 		if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)  in rt6_get_route_info()
4111 		    !rt->fib6_nh->fib_nh_gw_family)  in rt6_get_route_info()
4113 		if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))  in rt6_get_route_info()
4171 		struct fib6_nh *nh;  in rt6_get_dflt_router()
4177 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4433 	    ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) &&  in fib6_remove_prefsrc()
4461 	struct fib6_nh *nh;  in fib6_clean_tohost()
4467 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4517 	if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||  in rt6_is_dead()
4518 	    (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in rt6_is_dead()
4519 	     ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))  in rt6_is_dead()
4531 		total += rt->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4535 			total += iter->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4546 		*weight += rt->fib6_nh->fib_nh_weight;  in rt6_upper_bound_set()
4550 	atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);  in rt6_upper_bound_set()
4594 	    rt->fib6_nh->fib_nh_dev == arg->dev) {  in fib6_ifup()
4595 		rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;  in fib6_ifup()
4624 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4627 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4648 	if (rt->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4649 	    rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4652 		if (iter->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4653 		    iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4665 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4666 		rt->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4668 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4669 			iter->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4684 		return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4689 			return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4705 		if (rt->fib6_nh->fib_nh_dev != dev ||  in fib6_ifdown()
4708 		rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_ifdown()
4745 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()
4796 	return fib6_nh_mtu_change(f6i->fib6_nh, arg);  in rt6_mtu_change_route()
5088 		rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;  in ip6_route_multipath_add()
5274 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()
5301 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()
5346 		struct fib6_nh *fib6_nh;  in rt6_fill_node_nexthop()  local
5348 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5349 		if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,  in rt6_fill_node_nexthop()
5477 		if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,  in rt6_fill_node()
5478 				    rt->fib6_nh->fib_nh_weight, AF_INET6) < 0)  in rt6_fill_node()
5483 			if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,  in rt6_fill_node()
5484 					    sibling->fib6_nh->fib_nh_weight,  in rt6_fill_node()
5502 		if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,  in rt6_fill_node()
5529 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()
5550 	if (f6i->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5558 			if (sibling->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5574 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()
5682 			err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);  in rt6_dump_route()
5977 		net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;  in ip6_route_dev_notify()
6312 	init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;  in ip6_route_init_special_entries()