315 	struct rt6_info *rt;  in fib6_rule_lookup()  local
317 	rt = lookup(net, net->ipv6.fib6_main_tbl, fl6, skb, flags);  in fib6_rule_lookup()
318 	if (rt->dst.error == -EAGAIN) {  in fib6_rule_lookup()
319 		ip6_rt_put_flags(rt, flags);  in fib6_rule_lookup()
320 		rt = net->ipv6.ip6_null_entry;  in fib6_rule_lookup()
322 			dst_hold(&rt->dst);  in fib6_rule_lookup()
325 	return &rt->dst;  in fib6_rule_lookup()
362 				    struct fib6_info *rt)  in call_fib6_entry_notifier()  argument
365 		.rt = rt,  in call_fib6_entry_notifier()
373 			      struct fib6_info *rt,  in call_fib6_entry_notifiers()  argument
378 		.rt = rt,  in call_fib6_entry_notifiers()
381 	rt->fib6_table->fib_seq++;  in call_fib6_entry_notifiers()
387 					struct fib6_info *rt,  in call_fib6_multipath_entry_notifiers()  argument
393 		.rt = rt,  in call_fib6_multipath_entry_notifiers()
397 	rt->fib6_table->fib_seq++;  in call_fib6_multipath_entry_notifiers()
406 static void fib6_rt_dump(struct fib6_info *rt, struct fib6_dump_arg *arg)  in fib6_rt_dump()  argument
408 	if (rt == arg->net->ipv6.fib6_null_entry)  in fib6_rt_dump()
410 	call_fib6_entry_notifier(arg->nb, arg->net, FIB_EVENT_ENTRY_ADD, rt);  in fib6_rt_dump()
415 	struct fib6_info *rt;  in fib6_node_dump()  local
418 		fib6_rt_dump(rt, w->args);  in fib6_node_dump()
464 	struct fib6_info *rt;  in fib6_dump_node()  local
467 		res = rt6_dump_route(rt, w->args, w->skip_in_node);  in fib6_dump_node()
470 			w->leaf = rt;  in fib6_dump_node()
482 		 * RTA_MULTIPATH attribute. Jump 'rt' to point to the  in fib6_dump_node()
486 		if (rt->fib6_nsiblings)  in fib6_dump_node()
487 			rt = list_last_entry(&rt->fib6_siblings,  in fib6_dump_node()
971 static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,  in fib6_purge_rt()  argument
974 	struct fib6_table *table = rt->fib6_table;  in fib6_purge_rt()
976 	fib6_drop_pcpu_from(rt, table);  in fib6_purge_rt()
978 	if (rt->nh && !list_empty(&rt->nh_list))  in fib6_purge_rt()
979 		list_del_init(&rt->nh_list);  in fib6_purge_rt()
981 	if (refcount_read(&rt->fib6_ref) != 1) {  in fib6_purge_rt()
992 			if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {  in fib6_purge_rt()
997 				fib6_info_release(rt);  in fib6_purge_rt()
1009 static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,  in fib6_add_rt2node()  argument
1014 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1023 	bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);  in fib6_add_rt2node()
1034 				lockdep_is_held(&rt->fib6_table->tb6_lock))) {  in fib6_add_rt2node()
1039 		if (iter->fib6_metric == rt->fib6_metric) {  in fib6_add_rt2node()
1058 			if (rt6_duplicate_nexthop(iter, rt)) {  in fib6_add_rt2node()
1059 				if (rt->fib6_nsiblings)  in fib6_add_rt2node()
1060 					rt->fib6_nsiblings = 0;  in fib6_add_rt2node()
1063 				if (!(rt->fib6_flags & RTF_EXPIRES))  in fib6_add_rt2node()
1066 					fib6_set_expires(iter, rt->expires);  in fib6_add_rt2node()
1068 				if (rt->fib6_pmtu)  in fib6_add_rt2node()
1070 							rt->fib6_pmtu);  in fib6_add_rt2node()
1086 				rt->fib6_nsiblings++;  in fib6_add_rt2node()
1089 		if (iter->fib6_metric > rt->fib6_metric)  in fib6_add_rt2node()
1100 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1109 	if (rt->fib6_nsiblings) {  in fib6_add_rt2node()
1116 			if (sibling->fib6_metric == rt->fib6_metric &&  in fib6_add_rt2node()
1118 				list_add_tail(&rt->fib6_siblings,  in fib6_add_rt2node()
1123 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1131 					 &rt->fib6_siblings, fib6_siblings) {  in fib6_add_rt2node()
1133 			BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings);  in fib6_add_rt2node()
1136 		BUG_ON(fib6_nsiblings != rt->fib6_nsiblings);  in fib6_add_rt2node()
1153 							rt, extack);  in fib6_add_rt2node()
1160 				if (!rt->fib6_nsiblings)  in fib6_add_rt2node()
1164 							 &rt->fib6_siblings,  in fib6_add_rt2node()
1167 				rt->fib6_nsiblings = 0;  in fib6_add_rt2node()
1168 				list_del_init(&rt->fib6_siblings);  in fib6_add_rt2node()
1174 		rcu_assign_pointer(rt->fib6_next, iter);  in fib6_add_rt2node()
1175 		fib6_info_hold(rt);  in fib6_add_rt2node()
1176 		rcu_assign_pointer(rt->fib6_node, fn);  in fib6_add_rt2node()
1177 		rcu_assign_pointer(*ins, rt);  in fib6_add_rt2node()
1179 			inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);  in fib6_add_rt2node()
1200 							rt, extack);  in fib6_add_rt2node()
1205 		fib6_info_hold(rt);  in fib6_add_rt2node()
1206 		rcu_assign_pointer(rt->fib6_node, fn);  in fib6_add_rt2node()
1207 		rt->fib6_next = iter->fib6_next;  in fib6_add_rt2node()
1208 		rcu_assign_pointer(*ins, rt);  in fib6_add_rt2node()
1210 			inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);  in fib6_add_rt2node()
1224 			ins = &rt->fib6_next;  in fib6_add_rt2node()
1226 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1228 				if (iter->fib6_metric > rt->fib6_metric)  in fib6_add_rt2node()
1243 					lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_add_rt2node()
1252 static void fib6_start_gc(struct net *net, struct fib6_info *rt)  in fib6_start_gc()  argument
1255 	    (rt->fib6_flags & RTF_EXPIRES))  in fib6_start_gc()
1267 static void __fib6_update_sernum_upto_root(struct fib6_info *rt,  in __fib6_update_sernum_upto_root()  argument
1270 	struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,  in __fib6_update_sernum_upto_root()
1271 				lockdep_is_held(&rt->fib6_table->tb6_lock));  in __fib6_update_sernum_upto_root()
1278 				lockdep_is_held(&rt->fib6_table->tb6_lock));  in __fib6_update_sernum_upto_root()
1282 void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt)  in fib6_update_sernum_upto_root()  argument
1284 	__fib6_update_sernum_upto_root(rt, fib6_new_sernum(net));  in fib6_update_sernum_upto_root()
1302 int fib6_add(struct fib6_node *root, struct fib6_info *rt,  in fib6_add()  argument
1305 	struct fib6_table *table = rt->fib6_table;  in fib6_add()
1322 			&rt->fib6_dst.addr, rt->fib6_dst.plen,  in fib6_add()
1334 	if (rt->fib6_src.plen) {  in fib6_add()
1363 					&rt->fib6_src.addr, rt->fib6_src.plen,  in fib6_add()
1382 					&rt->fib6_src.addr, rt->fib6_src.plen,  in fib6_add()
1398 				fib6_info_hold(rt);  in fib6_add()
1399 				rcu_assign_pointer(fn->leaf, rt);  in fib6_add()
1406 	err = fib6_add_rt2node(fn, rt, info, extack);  in fib6_add()
1408 		if (rt->nh)  in fib6_add()
1409 			list_add(&rt->nh_list, &rt->nh->f6i_list);  in fib6_add()
1410 		__fib6_update_sernum_upto_root(rt, sernum);  in fib6_add()
1411 		fib6_start_gc(info->nl_net, rt);  in fib6_add()
1425 			if (pn_leaf == rt) {  in fib6_add()
1428 				fib6_info_release(rt);  in fib6_add()
1829 	struct fib6_info *rt = rcu_dereference_protected(*rtp,  in fib6_del_route()  local
1836 	*rtp = rt->fib6_next;  in fib6_del_route()
1837 	rt->fib6_node = NULL;  in fib6_del_route()
1842 	rt6_flush_exceptions(rt);  in fib6_del_route()
1845 	if (rcu_access_pointer(fn->rr_ptr) == rt)  in fib6_del_route()
1849 	if (rt->fib6_nsiblings) {  in fib6_del_route()
1853 					 &rt->fib6_siblings, fib6_siblings)  in fib6_del_route()
1855 		rt->fib6_nsiblings = 0;  in fib6_del_route()
1856 		list_del_init(&rt->fib6_siblings);  in fib6_del_route()
1863 		if (w->state == FWS_C && w->leaf == rt) {  in fib6_del_route()
1865 			w->leaf = rcu_dereference_protected(rt->fib6_next,  in fib6_del_route()
1885 	fib6_purge_rt(rt, fn, net);  in fib6_del_route()
1888 		call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt, NULL);  in fib6_del_route()
1890 		inet6_rt_notify(RTM_DELROUTE, rt, info, 0);  in fib6_del_route()
1892 	fib6_info_release(rt);  in fib6_del_route()
1896 int fib6_del(struct fib6_info *rt, struct nl_info *info)  in fib6_del()  argument
1898 	struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,  in fib6_del()
1899 				    lockdep_is_held(&rt->fib6_table->tb6_lock));  in fib6_del()
1900 	struct fib6_table *table = rt->fib6_table;  in fib6_del()
1905 	if (!fn || rt == net->ipv6.fib6_null_entry)  in fib6_del()
1917 		if (rt == cur) {  in fib6_del()
2059 	struct fib6_info *rt;  in fib6_clean_node()  local
2077 		res = c->func(rt, c->arg);  in fib6_clean_node()
2079 			w->leaf = rt;  in fib6_clean_node()
2080 			res = fib6_del(rt, &info);  in fib6_clean_node()
2083 				pr_debug("%s: del failed: rt=%p@%p err=%d\n",  in fib6_clean_node()
2084 					 __func__, rt,  in fib6_clean_node()
2085 					 rcu_access_pointer(rt->fib6_node),  in fib6_clean_node()
2092 			if (WARN_ON(!rt->fib6_nsiblings))  in fib6_clean_node()
2094 			rt = list_last_entry(&rt->fib6_siblings,  in fib6_clean_node()
2100 	w->leaf = rt;  in fib6_clean_node()
2178 static int fib6_age(struct fib6_info *rt, void *arg)  in fib6_age()  argument
2188 	if (rt->fib6_flags & RTF_EXPIRES && rt->expires) {  in fib6_age()
2189 		if (time_after(now, rt->expires)) {  in fib6_age()
2190 			RT6_TRACE("expiring %p\n", rt);  in fib6_age()
2200 	rt6_age_exceptions(rt, gc_args, now);  in fib6_age()
2373 	struct fib6_info *rt = v;  in ipv6_route_seq_show()  local
2375 	struct fib6_nh *fib6_nh = rt->fib6_nh;  in ipv6_route_seq_show()
2376 	unsigned int flags = rt->fib6_flags;  in ipv6_route_seq_show()
2379 	if (rt->nh)  in ipv6_route_seq_show()
2380 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ipv6_route_seq_show()
2382 	seq_printf(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);  in ipv6_route_seq_show()
2385 	seq_printf(seq, "%pi6 %02x ", &rt->fib6_src.addr, rt->fib6_src.plen);  in ipv6_route_seq_show()
2398 		   rt->fib6_metric, refcount_read(&rt->fib6_ref), 0,  in ipv6_route_seq_show()