55 int fib_default_rule_add(struct fib_rules_ops *ops,  in fib_default_rule_add()  argument
60 	r = kzalloc(ops->rule_size, GFP_KERNEL_ACCOUNT);  in fib_default_rule_add()
70 	r->fr_net = ops->fro_net;  in fib_default_rule_add()
78 	list_add_tail(&r->list, &ops->rules_list);  in fib_default_rule_add()
83 static u32 fib_default_rule_pref(struct fib_rules_ops *ops)  in fib_default_rule_pref()  argument
88 	if (!list_empty(&ops->rules_list)) {  in fib_default_rule_pref()
89 		pos = ops->rules_list.next;  in fib_default_rule_pref()
90 		if (pos->next != &ops->rules_list) {  in fib_default_rule_pref()
101 			       struct fib_rules_ops *ops, struct nlmsghdr *nlh,
106 	struct fib_rules_ops *ops;  in lookup_rules_ops()  local
109 	list_for_each_entry_rcu(ops, &net->rules_ops, list) {  in lookup_rules_ops()
110 		if (ops->family == family) {  in lookup_rules_ops()
111 			if (!try_module_get(ops->owner))  in lookup_rules_ops()
112 				ops = NULL;  in lookup_rules_ops()
114 			return ops;  in lookup_rules_ops()
122 static void rules_ops_put(struct fib_rules_ops *ops)  in rules_ops_put()  argument
124 	if (ops)  in rules_ops_put()
125 		module_put(ops->owner);  in rules_ops_put()
128 static void flush_route_cache(struct fib_rules_ops *ops)  in flush_route_cache()  argument
130 	if (ops->flush_cache)  in flush_route_cache()
131 		ops->flush_cache(ops);  in flush_route_cache()
134 static int __fib_rules_register(struct fib_rules_ops *ops)  in __fib_rules_register()  argument
140 	net = ops->fro_net;  in __fib_rules_register()
142 	if (ops->rule_size < sizeof(struct fib_rule))  in __fib_rules_register()
145 	if (ops->match == NULL || ops->configure == NULL ||  in __fib_rules_register()
146 	    ops->compare == NULL || ops->fill == NULL ||  in __fib_rules_register()
147 	    ops->action == NULL)  in __fib_rules_register()
152 		if (ops->family == o->family)  in __fib_rules_register()
155 	list_add_tail_rcu(&ops->list, &net->rules_ops);  in __fib_rules_register()
166 	struct fib_rules_ops *ops;  in fib_rules_register()  local
169 	ops = kmemdup(tmpl, sizeof(*ops), GFP_KERNEL);  in fib_rules_register()
170 	if (ops == NULL)  in fib_rules_register()
173 	INIT_LIST_HEAD(&ops->rules_list);  in fib_rules_register()
174 	ops->fro_net = net;  in fib_rules_register()
176 	err = __fib_rules_register(ops);  in fib_rules_register()
178 		kfree(ops);  in fib_rules_register()
179 		ops = ERR_PTR(err);  in fib_rules_register()
182 	return ops;  in fib_rules_register()
186 static void fib_rules_cleanup_ops(struct fib_rules_ops *ops)  in fib_rules_cleanup_ops()  argument
190 	list_for_each_entry_safe(rule, tmp, &ops->rules_list, list) {  in fib_rules_cleanup_ops()
192 		if (ops->delete)  in fib_rules_cleanup_ops()
193 			ops->delete(rule);  in fib_rules_cleanup_ops()
198 void fib_rules_unregister(struct fib_rules_ops *ops)  in fib_rules_unregister()  argument
200 	struct net *net = ops->fro_net;  in fib_rules_unregister()
203 	list_del_rcu(&ops->list);  in fib_rules_unregister()
206 	fib_rules_cleanup_ops(ops);  in fib_rules_unregister()
207 	kfree_rcu(ops, rcu);  in fib_rules_unregister()
259 static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,  in fib_rule_match()  argument
284 	ret = INDIRECT_CALL_MT(ops->match,  in fib_rule_match()
292 int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,  in fib_rules_lookup()  argument
300 	list_for_each_entry_rcu(rule, &ops->rules_list, list) {  in fib_rules_lookup()
302 		if (!fib_rule_match(rule, ops, fl, flags, arg))  in fib_rules_lookup()
318 			err = INDIRECT_CALL_MT(ops->action,  in fib_rules_lookup()
323 		if (!err && ops->suppress && INDIRECT_CALL_MT(ops->suppress,  in fib_rules_lookup()
364 				   struct fib_rules_ops *ops,  in call_fib_rule_notifiers()  argument
368 		.info.family = ops->family,  in call_fib_rule_notifiers()
373 	ops->fib_rules_seq++;  in call_fib_rule_notifiers()
381 	struct fib_rules_ops *ops;  in fib_rules_dump()  local
385 	ops = lookup_rules_ops(net, family);  in fib_rules_dump()
386 	if (!ops)  in fib_rules_dump()
388 	list_for_each_entry_rcu(rule, &ops->rules_list, list) {  in fib_rules_dump()
394 	rules_ops_put(ops);  in fib_rules_dump()
403 	struct fib_rules_ops *ops;  in fib_rules_seq_read()  local
407 	ops = lookup_rules_ops(net, family);  in fib_rules_seq_read()
408 	if (!ops)  in fib_rules_seq_read()
410 	fib_rules_seq = ops->fib_rules_seq;  in fib_rules_seq_read()
411 	rules_ops_put(ops);  in fib_rules_seq_read()
417 static struct fib_rule *rule_find(struct fib_rules_ops *ops,  in rule_find()  argument
425 	list_for_each_entry(r, &ops->rules_list, list) {  in rule_find()
487 		if (!ops->compare(r, frh, tb))  in rule_find()
518 		       struct fib_rules_ops *ops,  in fib_nl2rule()  argument
530 		    frh->src_len > (ops->addr_size * 8) ||  in fib_nl2rule()
531 		    nla_len(tb[FRA_SRC]) != ops->addr_size) {  in fib_nl2rule()
538 		    frh->dst_len > (ops->addr_size * 8) ||  in fib_nl2rule()
539 		    nla_len(tb[FRA_DST]) != ops->addr_size) {  in fib_nl2rule()
544 	nlrule = kzalloc(ops->rule_size, GFP_KERNEL_ACCOUNT);  in fib_nl2rule()
556 		nlrule->pref = fib_default_rule_pref(ops);  in fib_nl2rule()
686 static int rule_exists(struct fib_rules_ops *ops, struct fib_rule_hdr *frh,  in rule_exists()  argument
691 	list_for_each_entry(r, &ops->rules_list, list) {  in rule_exists()
746 		if (!ops->compare(r, frh, tb))  in rule_exists()
779 	struct fib_rules_ops *ops = NULL;  in fib_nl_newrule()  local
790 	ops = lookup_rules_ops(net, frh->family);  in fib_nl_newrule()
791 	if (!ops) {  in fib_nl_newrule()
804 	err = fib_nl2rule(skb, nlh, extack, ops, tb, &rule, &user_priority);  in fib_nl_newrule()
809 	    rule_exists(ops, frh, tb, rule)) {  in fib_nl_newrule()
814 	err = ops->configure(rule, skb, frh, tb, extack);  in fib_nl_newrule()
818 	err = call_fib_rule_notifiers(net, FIB_EVENT_RULE_ADD, rule, ops,  in fib_nl_newrule()
823 	list_for_each_entry(r, &ops->rules_list, list) {  in fib_nl_newrule()
833 	list_for_each_entry(r, &ops->rules_list, list) {  in fib_nl_newrule()
842 		list_add_rcu(&rule->list, &ops->rules_list);  in fib_nl_newrule()
844 	if (ops->unresolved_rules) {  in fib_nl_newrule()
849 		list_for_each_entry(r, &ops->rules_list, list) {  in fib_nl_newrule()
854 				if (--ops->unresolved_rules == 0)  in fib_nl_newrule()
861 		ops->nr_goto_rules++;  in fib_nl_newrule()
864 		ops->unresolved_rules++;  in fib_nl_newrule()
869 	notify_rule_change(RTM_NEWRULE, rule, ops, nlh, NETLINK_CB(skb).portid);  in fib_nl_newrule()
870 	flush_route_cache(ops);  in fib_nl_newrule()
871 	rules_ops_put(ops);  in fib_nl_newrule()
877 	rules_ops_put(ops);  in fib_nl_newrule()
887 	struct fib_rules_ops *ops = NULL;  in fib_nl_delrule()  local
898 	ops = lookup_rules_ops(net, frh->family);  in fib_nl_delrule()
899 	if (ops == NULL) {  in fib_nl_delrule()
912 	err = fib_nl2rule(skb, nlh, extack, ops, tb, &nlrule, &user_priority);  in fib_nl_delrule()
916 	rule = rule_find(ops, frh, tb, nlrule, user_priority);  in fib_nl_delrule()
927 	if (ops->delete) {  in fib_nl_delrule()
928 		err = ops->delete(rule);  in fib_nl_delrule()
939 		ops->nr_goto_rules--;  in fib_nl_delrule()
941 			ops->unresolved_rules--;  in fib_nl_delrule()
951 	if (ops->nr_goto_rules > 0) {  in fib_nl_delrule()
955 		if (&n->list == &ops->rules_list || n->pref != rule->pref)  in fib_nl_delrule()
957 		list_for_each_entry(r, &ops->rules_list, list) {  in fib_nl_delrule()
962 				ops->unresolved_rules++;  in fib_nl_delrule()
966 	call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops,  in fib_nl_delrule()
968 	notify_rule_change(RTM_DELRULE, rule, ops, nlh,  in fib_nl_delrule()
971 	flush_route_cache(ops);  in fib_nl_delrule()
972 	rules_ops_put(ops);  in fib_nl_delrule()
978 	rules_ops_put(ops);  in fib_nl_delrule()
983 static inline size_t fib_rule_nlmsg_size(struct fib_rules_ops *ops,  in fib_rule_nlmsg_size()  argument
1002 	if (ops->nlmsg_payload)  in fib_rule_nlmsg_size()
1003 		payload += ops->nlmsg_payload(rule);  in fib_rule_nlmsg_size()
1010 			    struct fib_rules_ops *ops)  in fib_nl_fill_rule()  argument
1020 	frh->family = ops->family;  in fib_nl_fill_rule()
1078 	if (ops->fill(rule, skb, frh) < 0)  in fib_nl_fill_rule()
1090 		      struct fib_rules_ops *ops)  in dump_rules()  argument
1097 	list_for_each_entry_rcu(rule, &ops->rules_list, list) {  in dump_rules()
1103 				       NLM_F_MULTI, ops);  in dump_rules()
1111 	rules_ops_put(ops);  in dump_rules()
1146 	struct fib_rules_ops *ops;  in fib_nl_dumprule()  local
1159 		ops = lookup_rules_ops(net, family);  in fib_nl_dumprule()
1160 		if (ops == NULL)  in fib_nl_dumprule()
1163 		dump_rules(skb, cb, ops);  in fib_nl_dumprule()
1169 	list_for_each_entry_rcu(ops, &net->rules_ops, list) {  in fib_nl_dumprule()
1170 		if (idx < cb->args[0] || !try_module_get(ops->owner))  in fib_nl_dumprule()
1173 		if (dump_rules(skb, cb, ops) < 0)  in fib_nl_dumprule()
1187 			       struct fib_rules_ops *ops, struct nlmsghdr *nlh,  in notify_rule_change()  argument
1194 	net = ops->fro_net;  in notify_rule_change()
1195 	skb = nlmsg_new(fib_rule_nlmsg_size(ops, rule), GFP_KERNEL);  in notify_rule_change()
1199 	err = fib_nl_fill_rule(skb, rule, pid, nlh->nlmsg_seq, event, 0, ops);  in notify_rule_change()
1207 	rtnl_notify(skb, net, pid, ops->nlgroup, nlh, GFP_KERNEL);  in notify_rule_change()
1211 		rtnl_set_sk_err(net, ops->nlgroup, err);  in notify_rule_change()
1246 	struct fib_rules_ops *ops;  in fib_rules_event()  local
1252 		list_for_each_entry(ops, &net->rules_ops, list)  in fib_rules_event()
1253 			attach_rules(&ops->rules_list, dev);  in fib_rules_event()
1257 		list_for_each_entry(ops, &net->rules_ops, list) {  in fib_rules_event()
1258 			detach_rules(&ops->rules_list, dev);  in fib_rules_event()
1259 			attach_rules(&ops->rules_list, dev);  in fib_rules_event()
1264 		list_for_each_entry(ops, &net->rules_ops, list)  in fib_rules_event()
1265 			detach_rules(&ops->rules_list, dev);  in fib_rules_event()