65 #define for_nexthops(fi) {						\  argument
67 	for (nhsel = 0, nh = (fi)->fib_nh;				\
68 	     nhsel < fib_info_num_path((fi));				\
71 #define change_nexthops(fi) {						\  argument
73 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
74 	     nhsel < fib_info_num_path((fi));				\
81 #define for_nexthops(fi) {						\  argument
82 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
85 #define change_nexthops(fi) {						\  argument
87 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
92 #define endfor_nexthops(fi) }  argument
231 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
233 	if (fi->nh) {  in free_fib_info_rcu()
234 		nexthop_put(fi->nh);  in free_fib_info_rcu()
236 		change_nexthops(fi) {  in free_fib_info_rcu()
237 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
238 		} endfor_nexthops(fi);  in free_fib_info_rcu()
241 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
243 	kfree(fi);  in free_fib_info_rcu()
246 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
248 	if (fi->fib_dead == 0) {  in free_fib_info()
249 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
254 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
258 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
261 	if (fi && refcount_dec_and_test(&fi->fib_treeref)) {  in fib_release_info()
262 		hlist_del(&fi->fib_hash);  in fib_release_info()
263 		if (fi->fib_prefsrc)  in fib_release_info()
264 			hlist_del(&fi->fib_lhash);  in fib_release_info()
265 		if (fi->nh) {  in fib_release_info()
266 			list_del(&fi->nh_list);  in fib_release_info()
268 			change_nexthops(fi) {  in fib_release_info()
272 			} endfor_nexthops(fi)  in fib_release_info()
274 		fi->fib_dead = 1;  in fib_release_info()
275 		fib_info_put(fi);  in fib_release_info()
280 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
284 	if (fi->nh || ofi->nh)  in nh_comp()
285 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
290 	for_nexthops(fi) {  in nh_comp()
313 	} endfor_nexthops(fi);  in nh_comp()
345 static inline unsigned int fib_info_hashfn(struct fib_info *fi)  in fib_info_hashfn()  argument
349 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hashfn()
350 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hashfn()
351 				fi->fib_priority);  in fib_info_hashfn()
353 	if (fi->nh) {  in fib_info_hashfn()
354 		val ^= fib_devindex_hashfn(fi->nh->id);  in fib_info_hashfn()
356 		for_nexthops(fi) {  in fib_info_hashfn()
358 		} endfor_nexthops(fi)  in fib_info_hashfn()
369 	struct fib_info *fi;  in fib_find_info_nh()  local
379 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
380 		if (!net_eq(fi->fib_net, net))  in fib_find_info_nh()
382 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
384 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
385 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
386 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
387 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
388 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
389 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
390 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
391 			return fi;  in fib_find_info_nh()
400 	struct fib_info *fi;  in fib_find_info()  local
406 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
407 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
409 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
411 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
412 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
413 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
414 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
415 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
416 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
418 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
419 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
420 			return fi;  in fib_find_info()
453 size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
461 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
466 	if (fi->nh)  in fib_nlmsg_size()
481 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
482 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
514 	fri.fi = fa->fa_info;  in rtmsg_fib()
538 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
542 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
566 		*last_resort = fi;  in fib_detect_death()
666 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
670 	struct net *net = fi->fib_net;  in fib_get_nhs()
675 	change_nexthops(fi) {  in fib_get_nhs()
732 	} endfor_nexthops(fi);  in fib_get_nhs()
735 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
765 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
770 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
774 	for_nexthops(fi) {  in fib_rebalance()
783 	} endfor_nexthops(fi);  in fib_rebalance()
786 	change_nexthops(fi) {  in fib_rebalance()
801 	} endfor_nexthops(fi);  in fib_rebalance()
805 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
814 #define fib_rebalance(fi) do { } while (0)  argument
840 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
848 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
852 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
858 		struct fib_nh *nh = fib_info_nh(fi, 0);  in fib_nh_match()
893 	for_nexthops(fi) {  in fib_nh_match()
949 	} endfor_nexthops(fi);  in fib_nh_match()
954 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
976 			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);  in fib_metrics_match()
983 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1252 		struct fib_info *fi;  in fib_info_hash_move()  local
1254 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
1258 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
1260 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
1268 		struct fib_info *fi;  in fib_info_hash_move()  local
1270 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
1274 			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);  in fib_info_hash_move()
1276 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
1307 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1308 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1318 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1349 	struct fib_info *fi = NULL;  in fib_create_info()  local
1372 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1373 			if (fi) {  in fib_create_info()
1374 				refcount_inc(&fi->fib_treeref);  in fib_create_info()
1375 				return fi;  in fib_create_info()
1417 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1418 	if (!fi)  in fib_create_info()
1420 	fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,  in fib_create_info()
1422 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1423 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1424 		kfree(fi);  in fib_create_info()
1429 	fi->fib_net = net;  in fib_create_info()
1430 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1431 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1432 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1433 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1434 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1435 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1436 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1438 	fi->fib_nhs = nhs;  in fib_create_info()
1445 			fi->nh = nh;  in fib_create_info()
1448 		change_nexthops(fi) {  in fib_create_info()
1449 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1450 		} endfor_nexthops(fi)  in fib_create_info()
1453 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1456 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1488 	if (fi->nh) {  in fib_create_info()
1489 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1493 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1514 		change_nexthops(fi) {  in fib_create_info()
1522 		} endfor_nexthops(fi)  in fib_create_info()
1523 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1524 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1527 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1532 	if (!fi->nh) {  in fib_create_info()
1533 		change_nexthops(fi) {  in fib_create_info()
1535 						  fi->fib_scope);  in fib_create_info()
1537 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1538 		} endfor_nexthops(fi)  in fib_create_info()
1540 		fib_rebalance(fi);  in fib_create_info()
1544 	ofi = fib_find_info(fi);  in fib_create_info()
1546 		fi->fib_dead = 1;  in fib_create_info()
1547 		free_fib_info(fi);  in fib_create_info()
1552 	refcount_set(&fi->fib_treeref, 1);  in fib_create_info()
1553 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1555 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1556 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1557 	if (fi->fib_prefsrc) {  in fib_create_info()
1560 		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];  in fib_create_info()
1561 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1563 	if (fi->nh) {  in fib_create_info()
1564 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1566 		change_nexthops(fi) {  in fib_create_info()
1575 		} endfor_nexthops(fi)  in fib_create_info()
1578 	return fi;  in fib_create_info()
1584 	if (fi) {  in fib_create_info()
1585 		fi->fib_dead = 1;  in fib_create_info()
1586 		free_fib_info(fi);  in fib_create_info()
1697 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1705 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1706 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1711 	for_nexthops(fi) {  in fib_add_multipath()
1719 	} endfor_nexthops(fi);  in fib_add_multipath()
1730 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1739 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1740 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1761 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1762 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1763 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1768 	if (fi->fib_priority &&  in fib_dump_info()
1769 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1771 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1774 	if (fi->fib_prefsrc &&  in fib_dump_info()
1775 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1778 	if (fi->nh) {  in fib_dump_info()
1779 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1781 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1783 		if (!fi->fib_net->ipv4.sysctl_nexthop_compat_mode)  in fib_dump_info()
1788 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1806 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1839 	struct fib_info *fi;  in fib_sync_down_addr()  local
1844 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1845 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1846 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1848 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1849 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1956 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1959 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1960 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
1962 		prev_fi = fi;  in fib_sync_down_dev()
1964 		change_nexthops(fi) {  in fib_sync_down_dev()
1985 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1989 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1990 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1994 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1997 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
2003 		fib_rebalance(fi);  in fib_sync_down_dev()
2012 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2018 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2052 		if (!fi) {  in fib_select_default()
2053 			if (next_fi != res->fi)  in fib_select_default()
2056 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2058 			fib_result_assign(res, fi);  in fib_select_default()
2062 		fi = next_fi;  in fib_select_default()
2066 	if (order <= 0 || !fi) {  in fib_select_default()
2072 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2074 		fib_result_assign(res, fi);  in fib_select_default()
2116 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2119 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2120 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_up()
2123 		prev_fi = fi;  in fib_sync_up()
2125 		change_nexthops(fi) {  in fib_sync_up()
2139 		} endfor_nexthops(fi)  in fib_sync_up()
2142 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2146 		fib_rebalance(fi);  in fib_sync_up()
2181 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2182 	struct net *net = fi->fib_net;  in fib_select_multipath()
2185 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2190 	change_nexthops(fi) {  in fib_select_multipath()
2207 	} endfor_nexthops(fi);  in fib_select_multipath()
2218 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()