63 #define for_nexthops(fi) {						\  argument
65 	for (nhsel = 0, nh = (fi)->fib_nh;				\
66 	     nhsel < (fi)->fib_nhs;					\
69 #define change_nexthops(fi) {						\  argument
71 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
72 	     nhsel < (fi)->fib_nhs;					\
79 #define for_nexthops(fi) {						\  argument
80 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
83 #define change_nexthops(fi) {						\  argument
85 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
90 #define endfor_nexthops(fi) }  argument
210 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
213 	change_nexthops(fi) {  in free_fib_info_rcu()
220 	} endfor_nexthops(fi);  in free_fib_info_rcu()
222 	m = fi->fib_metrics;  in free_fib_info_rcu()
225 	kfree(fi);  in free_fib_info_rcu()
228 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
230 	if (fi->fib_dead == 0) {  in free_fib_info()
231 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
236 	change_nexthops(fi) {  in free_fib_info()
238 			fi->fib_net->ipv4.fib_num_tclassid_users--;  in free_fib_info()
239 	} endfor_nexthops(fi);  in free_fib_info()
241 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
245 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
248 	if (fi && --fi->fib_treeref == 0) {  in fib_release_info()
249 		hlist_del(&fi->fib_hash);  in fib_release_info()
250 		if (fi->fib_prefsrc)  in fib_release_info()
251 			hlist_del(&fi->fib_lhash);  in fib_release_info()
252 		change_nexthops(fi) {  in fib_release_info()
256 		} endfor_nexthops(fi)  in fib_release_info()
257 		fi->fib_dead = 1;  in fib_release_info()
258 		fib_info_put(fi);  in fib_release_info()
263 static inline int nh_comp(const struct fib_info *fi, const struct fib_info *ofi)  in nh_comp()  argument
267 	for_nexthops(fi) {  in nh_comp()
281 	} endfor_nexthops(fi);  in nh_comp()
294 static inline unsigned int fib_info_hashfn(const struct fib_info *fi)  in fib_info_hashfn()  argument
297 	unsigned int val = fi->fib_nhs;  in fib_info_hashfn()
299 	val ^= (fi->fib_protocol << 8) | fi->fib_scope;  in fib_info_hashfn()
300 	val ^= (__force u32)fi->fib_prefsrc;  in fib_info_hashfn()
301 	val ^= fi->fib_priority;  in fib_info_hashfn()
302 	for_nexthops(fi) {  in fib_info_hashfn()
304 	} endfor_nexthops(fi)  in fib_info_hashfn()
312 	struct fib_info *fi;  in fib_find_info()  local
318 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
319 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
321 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
323 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
324 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
325 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
326 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
327 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
328 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
330 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
331 		    (nfi->fib_nhs == 0 || nh_comp(fi, nfi) == 0))  in fib_find_info()
332 			return fi;  in fib_find_info()
365 static inline size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
377 	if (fi->fib_nhs) {  in fib_nlmsg_size()
388 		for_nexthops(fi) {  in fib_nlmsg_size()
396 		} endfor_nexthops(fi);  in fib_nlmsg_size()
399 		payload += nla_total_size((fi->fib_nhs * nhsize) +  in fib_nlmsg_size()
436 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
443 	n = neigh_lookup(&arp_tbl, &fi->fib_nh[0].nh_gw, fi->fib_dev);  in fib_detect_death()
456 		*last_resort = fi;  in fib_detect_death()
484 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
490 	change_nexthops(fi) {  in fib_get_nhs()
520 				fi->fib_net->ipv4.fib_num_tclassid_users++;  in fib_get_nhs()
548 	} endfor_nexthops(fi);  in fib_get_nhs()
559 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
565 	if (fi->fib_nhs < 2)  in fib_rebalance()
569 	for_nexthops(fi) {  in fib_rebalance()
581 	} endfor_nexthops(fi);  in fib_rebalance()
584 	change_nexthops(fi) {  in fib_rebalance()
602 	} endfor_nexthops(fi);  in fib_rebalance()
606 #define fib_rebalance(fi) do { } while (0)  argument
632 int fib_nh_match(struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
640 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
646 					    fi->fib_nh, cfg, extack))  in fib_nh_match()
651 		    cfg->fc_flow != fi->fib_nh->nh_tclassid)  in fib_nh_match()
654 		if ((!cfg->fc_oif || cfg->fc_oif == fi->fib_nh->nh_oif) &&  in fib_nh_match()
655 		    (!cfg->fc_gw  || cfg->fc_gw == fi->fib_nh->nh_gw))  in fib_nh_match()
667 	for_nexthops(fi) {  in fib_nh_match()
691 	} endfor_nexthops(fi);  in fib_nh_match()
696 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
718 			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);  in fib_metrics_match()
725 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
951 		struct fib_info *fi;  in fib_info_hash_move()  local
953 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
957 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
959 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
967 		struct fib_info *fi;  in fib_info_hash_move()  local
969 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
973 			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);  in fib_info_hash_move()
975 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
1022 fib_convert_metrics(struct fib_info *fi, const struct fib_config *cfg)  in fib_convert_metrics()  argument
1024 	return ip_metrics_convert(fi->fib_net, cfg->fc_mx, cfg->fc_mx_len,  in fib_convert_metrics()
1025 				  fi->fib_metrics->metrics);  in fib_convert_metrics()
1032 	struct fib_info *fi = NULL;  in fib_create_info()  local
1082 	fi = kzalloc(sizeof(*fi)+nhs*sizeof(struct fib_nh), GFP_KERNEL);  in fib_create_info()
1083 	if (!fi)  in fib_create_info()
1086 		fi->fib_metrics = kzalloc(sizeof(*fi->fib_metrics), GFP_KERNEL);  in fib_create_info()
1087 		if (unlikely(!fi->fib_metrics)) {  in fib_create_info()
1088 			kfree(fi);  in fib_create_info()
1091 		refcount_set(&fi->fib_metrics->refcnt, 1);  in fib_create_info()
1093 		fi->fib_metrics = (struct dst_metrics *)&dst_default_metrics;  in fib_create_info()
1096 	fi->fib_net = net;  in fib_create_info()
1097 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1098 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1099 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1100 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1101 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1102 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1103 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1105 	fi->fib_nhs = nhs;  in fib_create_info()
1106 	change_nexthops(fi) {  in fib_create_info()
1107 		nexthop_nh->nh_parent = fi;  in fib_create_info()
1111 	} endfor_nexthops(fi)  in fib_create_info()
1113 	err = fib_convert_metrics(fi, cfg);  in fib_create_info()
1119 		err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg, extack);  in fib_create_info()
1122 		if (cfg->fc_oif && fi->fib_nh->nh_oif != cfg->fc_oif) {  in fib_create_info()
1127 		if (cfg->fc_gw && fi->fib_nh->nh_gw != cfg->fc_gw) {  in fib_create_info()
1133 		if (cfg->fc_flow && fi->fib_nh->nh_tclassid != cfg->fc_flow) {  in fib_create_info()
1145 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1169 			fi->fib_net->ipv4.fib_num_tclassid_users++;  in fib_create_info()
1203 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1217 		nh->nh_dev = dev_get_by_index(net, fi->fib_nh->nh_oif);  in fib_create_info()
1224 		change_nexthops(fi) {  in fib_create_info()
1230 		} endfor_nexthops(fi)  in fib_create_info()
1231 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1232 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1235 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1240 	change_nexthops(fi) {  in fib_create_info()
1242 	} endfor_nexthops(fi)  in fib_create_info()
1244 	fib_rebalance(fi);  in fib_create_info()
1247 	ofi = fib_find_info(fi);  in fib_create_info()
1249 		fi->fib_dead = 1;  in fib_create_info()
1250 		free_fib_info(fi);  in fib_create_info()
1255 	fi->fib_treeref++;  in fib_create_info()
1256 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1258 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1259 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1260 	if (fi->fib_prefsrc) {  in fib_create_info()
1263 		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];  in fib_create_info()
1264 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1266 	change_nexthops(fi) {  in fib_create_info()
1275 	} endfor_nexthops(fi)  in fib_create_info()
1277 	return fi;  in fib_create_info()
1283 	if (fi) {  in fib_create_info()
1284 		fi->fib_dead = 1;  in fib_create_info()
1285 		free_fib_info(fi);  in fib_create_info()
1293 		  struct fib_info *fi, unsigned int flags)  in fib_dump_info()  argument
1314 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1315 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1316 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1321 	if (fi->fib_priority &&  in fib_dump_info()
1322 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1324 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1327 	if (fi->fib_prefsrc &&  in fib_dump_info()
1328 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1330 	if (fi->fib_nhs == 1) {  in fib_dump_info()
1331 		if (fi->fib_nh->nh_gw &&  in fib_dump_info()
1332 		    nla_put_in_addr(skb, RTA_GATEWAY, fi->fib_nh->nh_gw))  in fib_dump_info()
1334 		if (fi->fib_nh->nh_oif &&  in fib_dump_info()
1335 		    nla_put_u32(skb, RTA_OIF, fi->fib_nh->nh_oif))  in fib_dump_info()
1337 		if (fi->fib_nh->nh_flags & RTNH_F_LINKDOWN) {  in fib_dump_info()
1341 			in_dev = __in_dev_get_rcu(fi->fib_nh->nh_dev);  in fib_dump_info()
1347 		if (fi->fib_nh->nh_flags & RTNH_F_OFFLOAD)  in fib_dump_info()
1350 		if (fi->fib_nh[0].nh_tclassid &&  in fib_dump_info()
1351 		    nla_put_u32(skb, RTA_FLOW, fi->fib_nh[0].nh_tclassid))  in fib_dump_info()
1354 		if (fi->fib_nh->nh_lwtstate &&  in fib_dump_info()
1355 		    lwtunnel_fill_encap(skb, fi->fib_nh->nh_lwtstate) < 0)  in fib_dump_info()
1359 	if (fi->fib_nhs > 1) {  in fib_dump_info()
1367 		for_nexthops(fi) {  in fib_dump_info()
1400 		} endfor_nexthops(fi);  in fib_dump_info()
1426 	struct fib_info *fi;  in fib_sync_down_addr()  local
1431 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1432 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1433 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1435 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1436 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1542 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1545 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1546 		if (nh->nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
1548 		prev_fi = fi;  in fib_sync_down_dev()
1550 		change_nexthops(fi) {  in fib_sync_down_dev()
1571 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1575 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1576 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1580 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1583 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
1589 		fib_rebalance(fi);  in fib_sync_down_dev()
1598 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
1604 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
1636 		if (!fi) {  in fib_select_default()
1637 			if (next_fi != res->fi)  in fib_select_default()
1640 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
1642 			fib_result_assign(res, fi);  in fib_select_default()
1646 		fi = next_fi;  in fib_select_default()
1650 	if (order <= 0 || !fi) {  in fib_select_default()
1656 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
1658 		fib_result_assign(res, fi);  in fib_select_default()
1698 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
1701 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
1702 		if (nh->nh_dev != dev || fi == prev_fi)  in fib_sync_up()
1705 		prev_fi = fi;  in fib_sync_up()
1707 		change_nexthops(fi) {  in fib_sync_up()
1721 		} endfor_nexthops(fi)  in fib_sync_up()
1724 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
1728 		fib_rebalance(fi);  in fib_sync_up()
1757 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
1758 	struct net *net = fi->fib_net;  in fib_select_multipath()
1761 	for_nexthops(fi) {  in fib_select_multipath()
1776 	} endfor_nexthops(fi);  in fib_select_multipath()
1787 	if (res->fi->fib_nhs > 1) {  in fib_select_path()