68 #define for_nexthops(fi) {						\  argument
70 	for (nhsel = 0, nh = (fi)->fib_nh;				\
71 	     nhsel < fib_info_num_path((fi));				\
74 #define change_nexthops(fi) {						\  argument
76 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
77 	     nhsel < fib_info_num_path((fi));				\
84 #define for_nexthops(fi) {						\  argument
85 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
88 #define change_nexthops(fi) {						\  argument
90 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
95 #define endfor_nexthops(fi) }  argument
234 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
236 	if (fi->nh) {  in free_fib_info_rcu()
237 		nexthop_put(fi->nh);  in free_fib_info_rcu()
239 		change_nexthops(fi) {  in free_fib_info_rcu()
240 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
241 		} endfor_nexthops(fi);  in free_fib_info_rcu()
244 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
246 	kfree(fi);  in free_fib_info_rcu()
249 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
251 	if (fi->fib_dead == 0) {  in free_fib_info()
252 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
256 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
260 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
263 	if (fi && refcount_dec_and_test(&fi->fib_treeref)) {  in fib_release_info()
264 		hlist_del(&fi->fib_hash);  in fib_release_info()
269 		if (fi->fib_prefsrc)  in fib_release_info()
270 			hlist_del(&fi->fib_lhash);  in fib_release_info()
271 		if (fi->nh) {  in fib_release_info()
272 			list_del(&fi->nh_list);  in fib_release_info()
274 			change_nexthops(fi) {  in fib_release_info()
278 			} endfor_nexthops(fi)  in fib_release_info()
280 		fi->fib_dead = 1;  in fib_release_info()
281 		fib_info_put(fi);  in fib_release_info()
286 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
290 	if (fi->nh || ofi->nh)  in nh_comp()
291 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
296 	for_nexthops(fi) {  in nh_comp()
319 	} endfor_nexthops(fi);  in nh_comp()
355 static inline unsigned int fib_info_hashfn(struct fib_info *fi)  in fib_info_hashfn()  argument
359 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hashfn()
360 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hashfn()
361 				fi->fib_priority);  in fib_info_hashfn()
363 	if (fi->nh) {  in fib_info_hashfn()
364 		val ^= fib_devindex_hashfn(fi->nh->id);  in fib_info_hashfn()
366 		for_nexthops(fi) {  in fib_info_hashfn()
368 		} endfor_nexthops(fi)  in fib_info_hashfn()
379 	struct fib_info *fi;  in fib_find_info_nh()  local
389 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
390 		if (!net_eq(fi->fib_net, net))  in fib_find_info_nh()
392 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
394 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
395 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
396 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
397 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
398 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
399 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
400 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
401 			return fi;  in fib_find_info_nh()
410 	struct fib_info *fi;  in fib_find_info()  local
416 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
417 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
419 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
421 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
422 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
423 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
424 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
425 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
426 		    nfi->fib_tb_id == fi->fib_tb_id &&  in fib_find_info()
427 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
429 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
430 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
431 			return fi;  in fib_find_info()
463 size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
471 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
476 	if (fi->nh)  in fib_nlmsg_size()
491 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
492 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
524 	fri.fi = fa->fa_info;  in rtmsg_fib()
548 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
552 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
576 		*last_resort = fi;  in fib_detect_death()
689 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
693 	struct net *net = fi->fib_net;  in fib_get_nhs()
698 	change_nexthops(fi) {  in fib_get_nhs()
767 	} endfor_nexthops(fi);  in fib_get_nhs()
770 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
800 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
805 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
809 	for_nexthops(fi) {  in fib_rebalance()
818 	} endfor_nexthops(fi);  in fib_rebalance()
821 	change_nexthops(fi) {  in fib_rebalance()
836 	} endfor_nexthops(fi);  in fib_rebalance()
840 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
849 #define fib_rebalance(fi) do { } while (0)  argument
875 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
883 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
887 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
892 	if (fi->nh) {  in fib_nh_match()
901 		nh = fib_info_nh(fi, 0);  in fib_nh_match()
935 	for_nexthops(fi) {  in fib_nh_match()
1003 	} endfor_nexthops(fi);  in fib_nh_match()
1008 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
1030 			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);  in fib_metrics_match()
1037 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1287 		struct fib_info *fi;  in fib_info_hash_move()  local
1289 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
1293 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
1295 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
1304 		struct fib_info *fi;  in fib_info_hash_move()  local
1306 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
1309 			ldest = fib_info_laddrhash_bucket(fi->fib_net,  in fib_info_hash_move()
1310 							  fi->fib_prefsrc);  in fib_info_hash_move()
1311 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
1340 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1341 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1351 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1382 	struct fib_info *fi = NULL;  in fib_create_info()  local
1405 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1406 			if (fi) {  in fib_create_info()
1407 				refcount_inc(&fi->fib_treeref);  in fib_create_info()
1408 				return fi;  in fib_create_info()
1452 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1453 	if (!fi)  in fib_create_info()
1455 	fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,  in fib_create_info()
1457 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1458 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1459 		kfree(fi);  in fib_create_info()
1463 	fi->fib_net = net;  in fib_create_info()
1464 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1465 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1466 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1467 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1468 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1469 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1470 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1472 	fi->fib_nhs = nhs;  in fib_create_info()
1479 			fi->nh = nh;  in fib_create_info()
1482 		change_nexthops(fi) {  in fib_create_info()
1483 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1484 		} endfor_nexthops(fi)  in fib_create_info()
1487 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1490 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1522 	if (fi->nh) {  in fib_create_info()
1523 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1527 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1550 		change_nexthops(fi) {  in fib_create_info()
1558 		} endfor_nexthops(fi)  in fib_create_info()
1559 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1560 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1563 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1568 	if (!fi->nh) {  in fib_create_info()
1569 		change_nexthops(fi) {  in fib_create_info()
1571 						  fi->fib_scope);  in fib_create_info()
1573 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1574 		} endfor_nexthops(fi)  in fib_create_info()
1576 		fib_rebalance(fi);  in fib_create_info()
1580 	ofi = fib_find_info(fi);  in fib_create_info()
1582 		fi->fib_dead = 1;  in fib_create_info()
1583 		free_fib_info(fi);  in fib_create_info()
1588 	refcount_set(&fi->fib_treeref, 1);  in fib_create_info()
1589 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1592 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1593 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1594 	if (fi->fib_prefsrc) {  in fib_create_info()
1597 		head = fib_info_laddrhash_bucket(net, fi->fib_prefsrc);  in fib_create_info()
1598 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1600 	if (fi->nh) {  in fib_create_info()
1601 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1603 		change_nexthops(fi) {  in fib_create_info()
1610 		} endfor_nexthops(fi)  in fib_create_info()
1613 	return fi;  in fib_create_info()
1619 	if (fi) {  in fib_create_info()
1620 		fi->fib_dead = 1;  in fib_create_info()
1621 		free_fib_info(fi);  in fib_create_info()
1732 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1740 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1741 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1746 	for_nexthops(fi) {  in fib_add_multipath()
1754 	} endfor_nexthops(fi);  in fib_add_multipath()
1765 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1774 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1775 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1796 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1797 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1798 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1803 	if (fi->fib_priority &&  in fib_dump_info()
1804 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1806 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1809 	if (fi->fib_prefsrc &&  in fib_dump_info()
1810 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1813 	if (fi->nh) {  in fib_dump_info()
1814 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1816 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1818 		if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode))  in fib_dump_info()
1823 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1841 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1872 	struct fib_info *fi;  in fib_sync_down_addr()  local
1879 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1880 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1881 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1883 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1884 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1989 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1992 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1993 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
1995 		prev_fi = fi;  in fib_sync_down_dev()
1997 		change_nexthops(fi) {  in fib_sync_down_dev()
2018 				dead = fi->fib_nhs;  in fib_sync_down_dev()
2022 		} endfor_nexthops(fi)  in fib_sync_down_dev()
2023 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
2027 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
2030 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
2036 		fib_rebalance(fi);  in fib_sync_down_dev()
2045 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2051 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2086 		if (!fi) {  in fib_select_default()
2087 			if (next_fi != res->fi)  in fib_select_default()
2090 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2092 			fib_result_assign(res, fi);  in fib_select_default()
2096 		fi = next_fi;  in fib_select_default()
2100 	if (order <= 0 || !fi) {  in fib_select_default()
2106 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2108 		fib_result_assign(res, fi);  in fib_select_default()
2148 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2151 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2152 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_up()
2155 		prev_fi = fi;  in fib_sync_up()
2157 		change_nexthops(fi) {  in fib_sync_up()
2171 		} endfor_nexthops(fi)  in fib_sync_up()
2174 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2178 		fib_rebalance(fi);  in fib_sync_up()
2213 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2214 	struct net *net = fi->fib_net;  in fib_select_multipath()
2217 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2222 	change_nexthops(fi) {  in fib_select_multipath()
2239 	} endfor_nexthops(fi);  in fib_select_multipath()
2250 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()