65 #define for_nexthops(fi) {						\  argument
67 	for (nhsel = 0, nh = (fi)->fib_nh;				\
68 	     nhsel < fib_info_num_path((fi));				\
71 #define change_nexthops(fi) {						\  argument
73 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
74 	     nhsel < fib_info_num_path((fi));				\
81 #define for_nexthops(fi) {						\  argument
82 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
85 #define change_nexthops(fi) {						\  argument
87 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
92 #define endfor_nexthops(fi) }  argument
233 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
235 	if (fi->nh) {  in free_fib_info_rcu()
236 		nexthop_put(fi->nh);  in free_fib_info_rcu()
238 		change_nexthops(fi) {  in free_fib_info_rcu()
239 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
240 		} endfor_nexthops(fi);  in free_fib_info_rcu()
243 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
245 	kfree(fi);  in free_fib_info_rcu()
248 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
250 	if (fi->fib_dead == 0) {  in free_fib_info()
251 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
256 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
260 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
263 	if (fi && --fi->fib_treeref == 0) {  in fib_release_info()
264 		hlist_del(&fi->fib_hash);  in fib_release_info()
265 		if (fi->fib_prefsrc)  in fib_release_info()
266 			hlist_del(&fi->fib_lhash);  in fib_release_info()
267 		if (fi->nh) {  in fib_release_info()
268 			list_del(&fi->nh_list);  in fib_release_info()
270 			change_nexthops(fi) {  in fib_release_info()
274 			} endfor_nexthops(fi)  in fib_release_info()
276 		fi->fib_dead = 1;  in fib_release_info()
277 		fib_info_put(fi);  in fib_release_info()
282 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
286 	if (fi->nh || ofi->nh)  in nh_comp()
287 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
292 	for_nexthops(fi) {  in nh_comp()
315 	} endfor_nexthops(fi);  in nh_comp()
347 static inline unsigned int fib_info_hashfn(struct fib_info *fi)  in fib_info_hashfn()  argument
351 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hashfn()
352 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hashfn()
353 				fi->fib_priority);  in fib_info_hashfn()
355 	if (fi->nh) {  in fib_info_hashfn()
356 		val ^= fib_devindex_hashfn(fi->nh->id);  in fib_info_hashfn()
358 		for_nexthops(fi) {  in fib_info_hashfn()
360 		} endfor_nexthops(fi)  in fib_info_hashfn()
371 	struct fib_info *fi;  in fib_find_info_nh()  local
381 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
382 		if (!net_eq(fi->fib_net, net))  in fib_find_info_nh()
384 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
386 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
387 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
388 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
389 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
390 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
391 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
392 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
393 			return fi;  in fib_find_info_nh()
402 	struct fib_info *fi;  in fib_find_info()  local
408 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
409 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
411 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
413 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
414 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
415 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
416 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
417 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
418 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
420 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
421 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
422 			return fi;  in fib_find_info()
455 static inline size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
463 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
468 	if (fi->nh)  in fib_nlmsg_size()
483 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
484 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
516 	fri.fi = fa->fa_info;  in rtmsg_fib()
539 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
543 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
567 		*last_resort = fi;  in fib_detect_death()
667 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
671 	struct net *net = fi->fib_net;  in fib_get_nhs()
676 	change_nexthops(fi) {  in fib_get_nhs()
733 	} endfor_nexthops(fi);  in fib_get_nhs()
736 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
766 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
771 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
775 	for_nexthops(fi) {  in fib_rebalance()
784 	} endfor_nexthops(fi);  in fib_rebalance()
787 	change_nexthops(fi) {  in fib_rebalance()
802 	} endfor_nexthops(fi);  in fib_rebalance()
806 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
815 #define fib_rebalance(fi) do { } while (0)  argument
841 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
849 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
853 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
859 		struct fib_nh *nh = fib_info_nh(fi, 0);  in fib_nh_match()
894 	for_nexthops(fi) {  in fib_nh_match()
950 	} endfor_nexthops(fi);  in fib_nh_match()
955 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
977 			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);  in fib_metrics_match()
984 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1253 		struct fib_info *fi;  in fib_info_hash_move()  local
1255 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
1259 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
1261 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
1269 		struct fib_info *fi;  in fib_info_hash_move()  local
1271 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
1275 			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);  in fib_info_hash_move()
1277 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
1308 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1309 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1319 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1350 	struct fib_info *fi = NULL;  in fib_create_info()  local
1373 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1374 			if (fi) {  in fib_create_info()
1375 				fi->fib_treeref++;  in fib_create_info()
1376 				return fi;  in fib_create_info()
1418 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1419 	if (!fi)  in fib_create_info()
1421 	fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,  in fib_create_info()
1423 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1424 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1425 		kfree(fi);  in fib_create_info()
1430 	fi->fib_net = net;  in fib_create_info()
1431 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1432 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1433 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1434 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1435 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1436 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1437 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1439 	fi->fib_nhs = nhs;  in fib_create_info()
1446 			fi->nh = nh;  in fib_create_info()
1449 		change_nexthops(fi) {  in fib_create_info()
1450 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1451 		} endfor_nexthops(fi)  in fib_create_info()
1454 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1457 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1489 	if (fi->nh) {  in fib_create_info()
1490 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1494 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1515 		change_nexthops(fi) {  in fib_create_info()
1523 		} endfor_nexthops(fi)  in fib_create_info()
1524 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1525 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1528 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1533 	if (!fi->nh) {  in fib_create_info()
1534 		change_nexthops(fi) {  in fib_create_info()
1536 						  fi->fib_scope);  in fib_create_info()
1538 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1539 		} endfor_nexthops(fi)  in fib_create_info()
1541 		fib_rebalance(fi);  in fib_create_info()
1545 	ofi = fib_find_info(fi);  in fib_create_info()
1547 		fi->fib_dead = 1;  in fib_create_info()
1548 		free_fib_info(fi);  in fib_create_info()
1553 	fi->fib_treeref++;  in fib_create_info()
1554 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1556 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1557 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1558 	if (fi->fib_prefsrc) {  in fib_create_info()
1561 		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];  in fib_create_info()
1562 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1564 	if (fi->nh) {  in fib_create_info()
1565 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1567 		change_nexthops(fi) {  in fib_create_info()
1576 		} endfor_nexthops(fi)  in fib_create_info()
1579 	return fi;  in fib_create_info()
1585 	if (fi) {  in fib_create_info()
1586 		fi->fib_dead = 1;  in fib_create_info()
1587 		free_fib_info(fi);  in fib_create_info()
1696 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1704 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1705 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1710 	for_nexthops(fi) {  in fib_add_multipath()
1719 	} endfor_nexthops(fi);  in fib_add_multipath()
1730 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1739 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1740 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1761 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1762 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1763 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1768 	if (fi->fib_priority &&  in fib_dump_info()
1769 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1771 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1774 	if (fi->fib_prefsrc &&  in fib_dump_info()
1775 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1778 	if (fi->nh) {  in fib_dump_info()
1779 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1781 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1783 		if (!fi->fib_net->ipv4.sysctl_nexthop_compat_mode)  in fib_dump_info()
1788 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1806 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1837 	struct fib_info *fi;  in fib_sync_down_addr()  local
1842 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1843 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1844 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1846 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1847 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1953 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1956 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1957 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
1959 		prev_fi = fi;  in fib_sync_down_dev()
1961 		change_nexthops(fi) {  in fib_sync_down_dev()
1982 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1986 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1987 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1991 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1994 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
2000 		fib_rebalance(fi);  in fib_sync_down_dev()
2009 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2015 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2049 		if (!fi) {  in fib_select_default()
2050 			if (next_fi != res->fi)  in fib_select_default()
2053 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2055 			fib_result_assign(res, fi);  in fib_select_default()
2059 		fi = next_fi;  in fib_select_default()
2063 	if (order <= 0 || !fi) {  in fib_select_default()
2069 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2071 		fib_result_assign(res, fi);  in fib_select_default()
2113 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2116 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2117 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_up()
2120 		prev_fi = fi;  in fib_sync_up()
2122 		change_nexthops(fi) {  in fib_sync_up()
2136 		} endfor_nexthops(fi)  in fib_sync_up()
2139 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2143 		fib_rebalance(fi);  in fib_sync_up()
2178 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2179 	struct net *net = fi->fib_net;  in fib_select_multipath()
2182 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2187 	change_nexthops(fi) {  in fib_select_multipath()
2204 	} endfor_nexthops(fi);  in fib_select_multipath()
2215 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()