319 	struct rt6_info *rt;  in fib6_rule_lookup()  local
321 	rt = pol_lookup_func(lookup,  in fib6_rule_lookup()
323 	if (rt->dst.error == -EAGAIN) {  in fib6_rule_lookup()
324 		ip6_rt_put_flags(rt, flags);  in fib6_rule_lookup()
325 		rt = net->ipv6.ip6_null_entry;  in fib6_rule_lookup()
327 			dst_hold(&rt->dst);  in fib6_rule_lookup()
330 	return &rt->dst;  in fib6_rule_lookup()
367 				    struct fib6_info *rt,  in call_fib6_entry_notifier()  argument
372 		.rt = rt,  in call_fib6_entry_notifier()
380 					      struct fib6_info *rt,  in call_fib6_multipath_entry_notifier()  argument
386 		.rt = rt,  in call_fib6_multipath_entry_notifier()
395 			      struct fib6_info *rt,  in call_fib6_entry_notifiers()  argument
400 		.rt = rt,  in call_fib6_entry_notifiers()
403 	rt->fib6_table->fib_seq++;  in call_fib6_entry_notifiers()
409 					struct fib6_info *rt,  in call_fib6_multipath_entry_notifiers()  argument
415 		.rt = rt,  in call_fib6_multipath_entry_notifiers()
419 	rt->fib6_table->fib_seq++;  in call_fib6_multipath_entry_notifiers()
423 int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt)  in call_fib6_entry_notifiers_replace()  argument
426 		.rt = rt,  in call_fib6_entry_notifiers_replace()
427 		.nsiblings = rt->fib6_nsiblings,  in call_fib6_entry_notifiers_replace()
430 	rt->fib6_table->fib_seq++;  in call_fib6_entry_notifiers_replace()
440 static int fib6_rt_dump(struct fib6_info *rt, struct fib6_dump_arg *arg)  in fib6_rt_dump()  argument
445 	if (!rt || rt == arg->net->ipv6.fib6_null_entry)  in fib6_rt_dump()
448 	if (rt->fib6_nsiblings)  in fib6_rt_dump()
450 							 rt,  in fib6_rt_dump()
451 							 rt->fib6_nsiblings,  in fib6_rt_dump()
454 		err = call_fib6_entry_notifier(arg->nb, fib_event, rt,  in fib6_rt_dump()
521 	struct fib6_info *rt;  in fib6_dump_node()  local
524 		res = rt6_dump_route(rt, w->args, w->skip_in_node);  in fib6_dump_node()
527 			w->leaf = rt;  in fib6_dump_node()
543 		if (rt->fib6_nsiblings)  in fib6_dump_node()
544 			rt = list_last_entry(&rt->fib6_siblings,  in fib6_dump_node()
1029 static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,  in fib6_purge_rt()  argument
1032 	struct fib6_table *table = rt->fib6_table;  in fib6_purge_rt()
1035 	rt6_flush_exceptions(rt);  in fib6_purge_rt()
1036 	fib6_drop_pcpu_from(rt, table);  in fib6_purge_rt()
1038 	if (rt->nh && !list_empty(&rt->nh_list))  in fib6_purge_rt()
1039 		list_del_init(&rt->nh_list);  in fib6_purge_rt()
1041 	if (refcount_read(&rt->fib6_ref) != 1) {  in fib6_purge_rt()
1052 			if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {  in fib6_purge_rt()
1057 				fib6_info_release(rt);  in fib6_purge_rt()
1064 	fib6_clean_expires_locked(rt);  in fib6_purge_rt()
1071 static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,  in fib6_add_rt2node()  argument
1076 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1085 	bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);  in fib6_add_rt2node()
1097 				lockdep_is_held(&rt->fib6_table->tb6_lock))) {  in fib6_add_rt2node()
1102 		if (iter->fib6_metric == rt->fib6_metric) {  in fib6_add_rt2node()
1120 			if (rt6_duplicate_nexthop(iter, rt)) {  in fib6_add_rt2node()
1121 				if (rt->fib6_nsiblings)  in fib6_add_rt2node()
1122 					rt->fib6_nsiblings = 0;  in fib6_add_rt2node()
1125 				if (!(rt->fib6_flags & RTF_EXPIRES))  in fib6_add_rt2node()
1129 								rt->expires);  in fib6_add_rt2node()
1131 				if (rt->fib6_pmtu)  in fib6_add_rt2node()
1133 							rt->fib6_pmtu);  in fib6_add_rt2node()
1149 				rt->fib6_nsiblings++;  in fib6_add_rt2node()
1152 		if (iter->fib6_metric > rt->fib6_metric)  in fib6_add_rt2node()
1165 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1174 	if (rt->fib6_nsiblings) {  in fib6_add_rt2node()
1182 			if (sibling->fib6_metric == rt->fib6_metric &&  in fib6_add_rt2node()
1184 				list_add_tail(&rt->fib6_siblings,  in fib6_add_rt2node()
1189 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1198 					 &rt->fib6_siblings, fib6_siblings) {  in fib6_add_rt2node()
1200 			BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings);  in fib6_add_rt2node()
1203 		BUG_ON(fib6_nsiblings != rt->fib6_nsiblings);  in fib6_add_rt2node()
1230 							fib_event, rt,  in fib6_add_rt2node()
1238 				if (!rt->fib6_nsiblings)  in fib6_add_rt2node()
1242 							 &rt->fib6_siblings,  in fib6_add_rt2node()
1245 				rt->fib6_nsiblings = 0;  in fib6_add_rt2node()
1246 				list_del_init(&rt->fib6_siblings);  in fib6_add_rt2node()
1252 		rcu_assign_pointer(rt->fib6_next, iter);  in fib6_add_rt2node()
1253 		fib6_info_hold(rt);  in fib6_add_rt2node()
1254 		rcu_assign_pointer(rt->fib6_node, fn);  in fib6_add_rt2node()
1255 		rcu_assign_pointer(*ins, rt);  in fib6_add_rt2node()
1257 			inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);  in fib6_add_rt2node()
1278 							rt, extack);  in fib6_add_rt2node()
1283 		fib6_info_hold(rt);  in fib6_add_rt2node()
1284 		rcu_assign_pointer(rt->fib6_node, fn);  in fib6_add_rt2node()
1285 		rt->fib6_next = iter->fib6_next;  in fib6_add_rt2node()
1286 		rcu_assign_pointer(*ins, rt);  in fib6_add_rt2node()
1288 			inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);  in fib6_add_rt2node()
1302 			ins = &rt->fib6_next;  in fib6_add_rt2node()
1304 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1306 				if (iter->fib6_metric > rt->fib6_metric)  in fib6_add_rt2node()
1321 					lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1330 static void fib6_start_gc(struct net *net, struct fib6_info *rt)  in fib6_start_gc()  argument
1333 	    (rt->fib6_flags & RTF_EXPIRES))  in fib6_start_gc()
1345 static void __fib6_update_sernum_upto_root(struct fib6_info *rt,  in __fib6_update_sernum_upto_root()  argument
1348 	struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,  in __fib6_update_sernum_upto_root()
1349 				lockdep_is_held(&rt->fib6_table->tb6_lock));  in __fib6_update_sernum_upto_root()
1356 				lockdep_is_held(&rt->fib6_table->tb6_lock));  in __fib6_update_sernum_upto_root()
1360 void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt)  in fib6_update_sernum_upto_root()  argument
1362 	__fib6_update_sernum_upto_root(rt, fib6_new_sernum(net));  in fib6_update_sernum_upto_root()
1380 int fib6_add(struct fib6_node *root, struct fib6_info *rt,  in fib6_add()  argument
1383 	struct fib6_table *table = rt->fib6_table;  in fib6_add()
1399 			&rt->fib6_dst.addr, rt->fib6_dst.plen,  in fib6_add()
1411 	if (rt->fib6_src.plen) {  in fib6_add()
1440 					&rt->fib6_src.addr, rt->fib6_src.plen,  in fib6_add()
1459 					&rt->fib6_src.addr, rt->fib6_src.plen,  in fib6_add()
1475 				fib6_info_hold(rt);  in fib6_add()
1476 				rcu_assign_pointer(fn->leaf, rt);  in fib6_add()
1483 	err = fib6_add_rt2node(fn, rt, info, extack);  in fib6_add()
1485 		if (rt->nh)  in fib6_add()
1486 			list_add(&rt->nh_list, &rt->nh->f6i_list);  in fib6_add()
1487 		__fib6_update_sernum_upto_root(rt, fib6_new_sernum(info->nl_net));  in fib6_add()
1489 		if (fib6_has_expires(rt))  in fib6_add()
1490 			hlist_add_head(&rt->gc_link, &table->tb6_gc_hlist);  in fib6_add()
1492 		fib6_start_gc(info->nl_net, rt);  in fib6_add()
1506 			if (pn_leaf == rt) {  in fib6_add()
1509 				fib6_info_release(rt);  in fib6_add()
1527 	} else if (fib6_requires_src(rt)) {  in fib6_add()
1917 	struct fib6_info *rt = rcu_dereference_protected(*rtp,  in fib6_del_route()  local
1930 	if (leaf == rt && !rt->fib6_nsiblings) {  in fib6_del_route()
1931 		if (rcu_access_pointer(rt->fib6_next))  in fib6_del_route()
1932 			replace_rt = rcu_dereference_protected(rt->fib6_next,  in fib6_del_route()
1939 	*rtp = rt->fib6_next;  in fib6_del_route()
1940 	rt->fib6_node = NULL;  in fib6_del_route()
1945 	if (rcu_access_pointer(fn->rr_ptr) == rt)  in fib6_del_route()
1949 	if (rt->fib6_nsiblings) {  in fib6_del_route()
1957 		if (rt->fib6_metric == leaf->fib6_metric &&  in fib6_del_route()
1961 					 &rt->fib6_siblings, fib6_siblings)  in fib6_del_route()
1963 		rt->fib6_nsiblings = 0;  in fib6_del_route()
1964 		list_del_init(&rt->fib6_siblings);  in fib6_del_route()
1971 		if (w->state == FWS_C && w->leaf == rt) {  in fib6_del_route()
1973 			w->leaf = rcu_dereference_protected(rt->fib6_next,  in fib6_del_route()
1993 	fib6_purge_rt(rt, fn, net);  in fib6_del_route()
1998 						  rt, NULL);  in fib6_del_route()
2003 		inet6_rt_notify(RTM_DELROUTE, rt, info, 0);  in fib6_del_route()
2005 	fib6_info_release(rt);  in fib6_del_route()
2009 int fib6_del(struct fib6_info *rt, struct nl_info *info)  in fib6_del()  argument
2017 	if (rt == net->ipv6.fib6_null_entry)  in fib6_del()
2020 	table = rt->fib6_table;  in fib6_del()
2021 	fn = rcu_dereference_protected(rt->fib6_node,  in fib6_del()
2035 		if (rt == cur) {  in fib6_del()
2179 	struct fib6_info *rt;  in fib6_clean_node()  local
2197 		res = c->func(rt, c->arg);  in fib6_clean_node()
2199 			w->leaf = rt;  in fib6_clean_node()
2200 			res = fib6_del(rt, &info);  in fib6_clean_node()
2204 					 __func__, rt,  in fib6_clean_node()
2205 					 rcu_access_pointer(rt->fib6_node),  in fib6_clean_node()
2212 			if (WARN_ON(!rt->fib6_nsiblings))  in fib6_clean_node()
2214 			rt = list_last_entry(&rt->fib6_siblings,  in fib6_clean_node()
2220 	w->leaf = rt;  in fib6_clean_node()
2298 static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args)  in fib6_age()  argument
2307 	if (fib6_has_expires(rt) && rt->expires) {  in fib6_age()
2308 		if (time_after(now, rt->expires)) {  in fib6_age()
2309 			RT6_TRACE("expiring %p\n", rt);  in fib6_age()
2319 	rt6_age_exceptions(rt, gc_args, now);  in fib6_age()
2328 	struct fib6_info *rt;  in fib6_gc_table()  local
2335 	hlist_for_each_entry_safe(rt, n, &tb6->tb6_gc_hlist, gc_link)  in fib6_gc_table()
2336 		if (fib6_age(rt, gc_args) == -1)  in fib6_gc_table()
2337 			fib6_del(rt, &info);  in fib6_gc_table()
2530 	struct fib6_info *rt = v;  in ipv6_route_native_seq_show()  local
2532 	struct fib6_nh *fib6_nh = rt->fib6_nh;  in ipv6_route_native_seq_show()
2533 	unsigned int flags = rt->fib6_flags;  in ipv6_route_native_seq_show()
2536 	if (rt->nh)  in ipv6_route_native_seq_show()
2537 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ipv6_route_native_seq_show()
2539 	seq_printf(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);  in ipv6_route_native_seq_show()
2542 	seq_printf(seq, "%pi6 %02x ", &rt->fib6_src.addr, rt->fib6_src.plen);  in ipv6_route_native_seq_show()
2555 		   rt->fib6_metric, refcount_read(&rt->fib6_ref), 0,  in ipv6_route_native_seq_show()
2710 	ctx.rt = v;  in ipv6_route_prog_seq_show()