121 				       struct nh_group *nhg)  in nh_notifier_mpath_info_init()  argument
123 	u16 num_nh = nhg->num_nh;  in nh_notifier_mpath_info_init()
133 	info->nh_grp->is_fdb = nhg->fdb_nh;  in nh_notifier_mpath_info_init()
136 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_notifier_mpath_info_init()
150 					   struct nh_group *nhg)  in nh_notifier_res_table_info_init()  argument
152 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nh_notifier_res_table_info_init()
183 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_init()  local
185 	if (nhg->hash_threshold)  in nh_notifier_grp_info_init()
186 		return nh_notifier_mpath_info_init(info, nhg);  in nh_notifier_grp_info_init()
187 	else if (nhg->resilient)  in nh_notifier_grp_info_init()
188 		return nh_notifier_res_table_info_init(info, nhg);  in nh_notifier_grp_info_init()
195 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_fini()  local
197 	if (nhg->hash_threshold)  in nh_notifier_grp_info_fini()
199 	else if (nhg->resilient)  in nh_notifier_grp_info_fini()
257 	struct nh_group *nhg;  in nh_notifier_res_bucket_idle_timer_get()  local
283 	nhg = rcu_dereference(nh->nh_grp);  in nh_notifier_res_bucket_idle_timer_get()
284 	res_table = rcu_dereference(nhg->res_table);  in nh_notifier_res_bucket_idle_timer_get()
397 	struct nh_group *nhg;  in call_nexthop_res_table_notifiers()  local
409 	nhg = rtnl_dereference(nh->nh_grp);  in call_nexthop_res_table_notifiers()
410 	err = nh_notifier_mpath_info_init(&info, nhg);  in call_nexthop_res_table_notifiers()
469 	struct nh_group *nhg;  in nexthop_free_group()  local
472 	nhg = rcu_dereference_raw(nh->nh_grp);  in nexthop_free_group()
473 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_free_group()
474 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_free_group()
480 	WARN_ON(nhg->spare == nhg);  in nexthop_free_group()
482 	if (nhg->resilient)  in nexthop_free_group()
483 		vfree(rcu_dereference_raw(nhg->res_table));  in nexthop_free_group()
485 	kfree(nhg->spare);  in nexthop_free_group()
486 	kfree(nhg);  in nexthop_free_group()
534 	struct nh_group *nhg;  in nexthop_grp_alloc()  local
536 	nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);  in nexthop_grp_alloc()
537 	if (nhg)  in nexthop_grp_alloc()
538 		nhg->num_nh = num_nh;  in nexthop_grp_alloc()
540 	return nhg;  in nexthop_grp_alloc()
629 static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)  in nla_put_nh_group_res()  argument
631 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nla_put_nh_group_res()
657 static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)  in nla_put_nh_group()  argument
660 	size_t len = nhg->num_nh * sizeof(*p);  in nla_put_nh_group()
665 	if (nhg->hash_threshold)  in nla_put_nh_group()
667 	else if (nhg->resilient)  in nla_put_nh_group()
678 	for (i = 0; i < nhg->num_nh; ++i) {  in nla_put_nh_group()
679 		p->id = nhg->nh_entries[i].nh->id;  in nla_put_nh_group()
680 		p->weight = nhg->nh_entries[i].weight - 1;  in nla_put_nh_group()
684 	if (nhg->resilient && nla_put_nh_group_res(skb, nhg))  in nla_put_nh_group()
717 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_fill_node()  local
719 		if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))  in nh_fill_node()
721 		if (nla_put_nh_group(skb, nhg))  in nh_fill_node()
774 static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)  in nh_nlmsg_size_grp_res()  argument
785 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_nlmsg_size_grp()  local
786 	size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;  in nh_nlmsg_size_grp()
790 	if (nhg->resilient)  in nh_nlmsg_size_grp()
791 		tot += nh_nlmsg_size_grp_res(nhg);  in nh_nlmsg_size_grp()
993 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in valid_group_nh()  local
996 		if (nhg->hash_threshold) {  in valid_group_nh()
1001 		if (nhg->resilient) {  in valid_group_nh()
1006 		*is_fdb = nhg->fdb_nh;  in valid_group_nh()
1049 	struct nexthop_grp *nhg;  in nh_check_attr_group()  local
1060 	len /= sizeof(*nhg);  in nh_check_attr_group()
1062 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1064 		if (nhg[i].resvd1 || nhg[i].resvd2) {  in nh_check_attr_group()
1068 		if (nhg[i].weight > 254) {  in nh_check_attr_group()
1073 			if (nhg[i].id == nhg[j].id) {  in nh_check_attr_group()
1082 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1087 		nh = nexthop_find_by_id(net, nhg[i].id);  in nh_check_attr_group()
1155 static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)  in nexthop_select_path_hthr()  argument
1160 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_select_path_hthr()
1161 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_select_path_hthr()
1192 static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)  in nexthop_select_path_res()  argument
1194 	struct nh_res_table *res_table = rcu_dereference(nhg->res_table);  in nexthop_select_path_res()
1210 	struct nh_group *nhg;  in nexthop_select_path()  local
1215 	nhg = rcu_dereference(nh->nh_grp);  in nexthop_select_path()
1216 	if (nhg->hash_threshold)  in nexthop_select_path()
1217 		return nexthop_select_path_hthr(nhg, hash);  in nexthop_select_path()
1218 	else if (nhg->resilient)  in nexthop_select_path()
1219 		return nexthop_select_path_res(nhg, hash);  in nexthop_select_path()
1234 		struct nh_group *nhg;  in nexthop_for_each_fib6_nh()  local
1237 		nhg = rcu_dereference_rtnl(nh->nh_grp);  in nexthop_for_each_fib6_nh()
1238 		for (i = 0; i < nhg->num_nh; i++) {  in nexthop_for_each_fib6_nh()
1239 			struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_for_each_fib6_nh()
1283 		struct nh_group *nhg;  in fib6_check_nexthop()  local
1285 		nhg = rtnl_dereference(nh->nh_grp);  in fib6_check_nexthop()
1286 		if (nhg->has_v4)  in fib6_check_nexthop()
1288 		is_fdb_nh = nhg->fdb_nh;  in fib6_check_nexthop()
1355 		struct nh_group *nhg;  in fib_check_nexthop()  local
1357 		nhg = rtnl_dereference(nh->nh_grp);  in fib_check_nexthop()
1358 		if (nhg->fdb_nh) {  in fib_check_nexthop()
1371 		nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);  in fib_check_nexthop()
1631 static void nh_res_group_rebalance(struct nh_group *nhg,  in nh_res_group_rebalance()  argument
1641 	for (i = 0; i < nhg->num_nh; ++i)  in nh_res_group_rebalance()
1642 		total += nhg->nh_entries[i].weight;  in nh_res_group_rebalance()
1644 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_res_group_rebalance()
1645 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_res_group_rebalance()
1668 					 struct nh_group *nhg)  in nh_res_table_migrate_buckets()  argument
1678 		for (j = 0; j < nhg->num_nh; j++) {  in nh_res_table_migrate_buckets()
1679 			struct nh_grp_entry *nhge = &nhg->nh_entries[j];  in nh_res_table_migrate_buckets()
1714 static void nh_hthr_group_rebalance(struct nh_group *nhg)  in nh_hthr_group_rebalance()  argument
1720 	for (i = 0; i < nhg->num_nh; ++i)  in nh_hthr_group_rebalance()
1721 		total += nhg->nh_entries[i].weight;  in nh_hthr_group_rebalance()
1723 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_hthr_group_rebalance()
1724 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_hthr_group_rebalance()
1740 	struct nh_group *nhg, *newg;  in remove_nh_grp_entry()  local
1745 	nhg = rtnl_dereference(nhp->nh_grp);  in remove_nh_grp_entry()
1746 	newg = nhg->spare;  in remove_nh_grp_entry()
1749 	if (nhg->num_nh == 1) {  in remove_nh_grp_entry()
1755 	newg->is_multipath = nhg->is_multipath;  in remove_nh_grp_entry()
1756 	newg->hash_threshold = nhg->hash_threshold;  in remove_nh_grp_entry()
1757 	newg->resilient = nhg->resilient;  in remove_nh_grp_entry()
1758 	newg->fdb_nh = nhg->fdb_nh;  in remove_nh_grp_entry()
1759 	newg->num_nh = nhg->num_nh;  in remove_nh_grp_entry()
1762 	nhges = nhg->nh_entries;  in remove_nh_grp_entry()
1764 	for (i = 0, j = 0; i < nhg->num_nh; ++i) {  in remove_nh_grp_entry()
1768 		if (nhg->nh_entries[i].nh == nh) {  in remove_nh_grp_entry()
1788 		replace_nexthop_grp_res(nhg, newg);  in remove_nh_grp_entry()
1823 	struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);  in remove_nexthop_group()  local
1825 	int i, num_nh = nhg->num_nh;  in remove_nexthop_group()
1828 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in remove_nexthop_group()
1836 	if (nhg->resilient) {  in remove_nexthop_group()
1837 		res_table = rtnl_dereference(nhg->res_table);  in remove_nexthop_group()
1907 	struct nh_group *nhg;  in nh_rt_cache_flush()  local
1922 	nhg = rtnl_dereference(replaced_nh->nh_grp);  in nh_rt_cache_flush()
1923 	for (i = 0; i < nhg->num_nh; i++) {  in nh_rt_cache_flush()
1924 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_rt_cache_flush()
2019 static void nh_group_v4_update(struct nh_group *nhg)  in nh_group_v4_update()  argument
2025 	nhges = nhg->nh_entries;  in nh_group_v4_update()
2026 	for (i = 0; i < nhg->num_nh; i++) {  in nh_group_v4_update()
2033 	nhg->has_v4 = has_v4;  in nh_group_v4_update()
2085 	struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);  in replace_nexthop_single_notify()  local
2088 	if (nhg->hash_threshold) {  in replace_nexthop_single_notify()
2091 	} else if (nhg->resilient) {  in replace_nexthop_single_notify()
2092 		res_table = rtnl_dereference(nhg->res_table);  in replace_nexthop_single_notify()
2155 			struct nh_group *nhg;  in replace_nexthop_single()  local
2157 			nhg = rtnl_dereference(nhp->nh_grp);  in replace_nexthop_single()
2158 			nh_group_v4_update(nhg);  in replace_nexthop_single()
2326 		struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);  in insert_nexthop()  local
2329 		if (nhg->resilient) {  in insert_nexthop()
2330 			res_table = rtnl_dereference(nhg->res_table);  in insert_nexthop()
2341 			nh_res_group_rebalance(nhg, res_table);  in insert_nexthop()
2414 	struct nh_group *nhg;  in nexthop_create_group()  local
2428 	nhg = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2429 	if (!nhg) {  in nexthop_create_group()
2435 	nhg->spare = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2436 	if (!nhg->spare) {  in nexthop_create_group()
2437 		kfree(nhg);  in nexthop_create_group()
2441 	nhg->spare->spare = nhg;  in nexthop_create_group()
2443 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_create_group()
2455 			nhg->has_v4 = true;  in nexthop_create_group()
2457 		nhg->nh_entries[i].nh = nhe;  in nexthop_create_group()
2458 		nhg->nh_entries[i].weight = entry[i].weight + 1;  in nexthop_create_group()
2459 		list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);  in nexthop_create_group()
2460 		nhg->nh_entries[i].nh_parent = nh;  in nexthop_create_group()
2464 		nhg->hash_threshold = 1;  in nexthop_create_group()
2465 		nhg->is_multipath = true;  in nexthop_create_group()
2475 		rcu_assign_pointer(nhg->spare->res_table, res_table);  in nexthop_create_group()
2476 		rcu_assign_pointer(nhg->res_table, res_table);  in nexthop_create_group()
2477 		nhg->resilient = true;  in nexthop_create_group()
2478 		nhg->is_multipath = true;  in nexthop_create_group()
2481 	WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);  in nexthop_create_group()
2483 	if (nhg->hash_threshold)  in nexthop_create_group()
2484 		nh_hthr_group_rebalance(nhg);  in nexthop_create_group()
2487 		nhg->fdb_nh = 1;  in nexthop_create_group()
2489 	rcu_assign_pointer(nh->nh_grp, nhg);  in nexthop_create_group()
2495 		list_del(&nhg->nh_entries[i].nh_list);  in nexthop_create_group()
2496 		nexthop_put(nhg->nh_entries[i].nh);  in nexthop_create_group()
2499 	kfree(nhg->spare);  in nexthop_create_group()
2500 	kfree(nhg);  in nexthop_create_group()
3240 	struct nh_group *nhg;  in nexthop_find_group_resilient()  local
3252 	nhg = rtnl_dereference(nh->nh_grp);  in nexthop_find_group_resilient()
3253 	if (!nhg->resilient) {  in nexthop_find_group_resilient()
3346 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_nh()  local
3353 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_nh()
3354 	res_table = rtnl_dereference(nhg->res_table);  in rtm_dump_nexthop_bucket_nh()
3396 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_cb()  local
3401 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_cb()
3402 	if (!nhg->resilient)  in rtm_dump_nexthop_bucket_cb()
3507 	struct nh_group *nhg;  in rtm_get_nexthop_bucket()  local
3521 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_get_nexthop_bucket()
3522 	res_table = rtnl_dereference(nhg->res_table);  in rtm_get_nexthop_bucket()
3674 	struct nh_group *nhg;  in nexthop_bucket_set_hw_flags()  local
3682 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_bucket_set_hw_flags()
3683 	if (!nhg->resilient)  in nexthop_bucket_set_hw_flags()
3686 	if (bucket_index >= nhg->res_table->num_nh_buckets)  in nexthop_bucket_set_hw_flags()
3689 	res_table = rcu_dereference(nhg->res_table);  in nexthop_bucket_set_hw_flags()
3707 	struct nh_group *nhg;  in nexthop_res_grp_activity_update()  local
3716 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_res_grp_activity_update()
3717 	if (!nhg->resilient)  in nexthop_res_grp_activity_update()
3723 	res_table = rcu_dereference(nhg->res_table);  in nexthop_res_grp_activity_update()