168 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
169 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
525 	struct xfrm_policy *pol;  in xfrm_dst_hash_transfer()  local
531 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {  in xfrm_dst_hash_transfer()
534 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
535 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,  in xfrm_dst_hash_transfer()
536 				pol->family, nhashmask, dbits, sbits);  in xfrm_dst_hash_transfer()
538 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
539 			hlist_add_head_rcu(&pol->bydst, ndsttable + h);  in xfrm_dst_hash_transfer()
544 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
545 			hlist_add_behind_rcu(&pol->bydst, entry0);  in xfrm_dst_hash_transfer()
547 		entry0 = &pol->bydst;  in xfrm_dst_hash_transfer()
560 	struct xfrm_policy *pol;  in xfrm_idx_hash_transfer()  local
562 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {  in xfrm_idx_hash_transfer()
565 		h = __idx_hash(pol->index, nhashmask);  in xfrm_idx_hash_transfer()
566 		hlist_add_head(&pol->byidx, nidxtable+h);  in xfrm_idx_hash_transfer()
695 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)  in xfrm_policy_inexact_alloc_bin()  argument
699 		.family = pol->family,  in xfrm_policy_inexact_alloc_bin()
700 		.type = pol->type,  in xfrm_policy_inexact_alloc_bin()
702 		.if_id = pol->if_id,  in xfrm_policy_inexact_alloc_bin()
704 	struct net *net = xp_net(pol);  in xfrm_policy_inexact_alloc_bin()
1215 	struct xfrm_policy *pol;  in xfrm_hash_rebuild()  local
1333 		hlist_for_each_entry(pol, chain, bydst) {  in xfrm_hash_rebuild()
1334 			if (policy->priority >= pol->priority)  in xfrm_hash_rebuild()
1335 				newpos = &pol->bydst;  in xfrm_hash_rebuild()
1437 					  struct xfrm_policy *pol)  in xfrm_policy_mark_match()  argument
1439 	return mark->v == pol->mark.v && mark->m == pol->mark.m;  in xfrm_policy_mark_match()
1494 	struct xfrm_policy *pol, *delpol = NULL;  in xfrm_policy_insert_inexact_list()  local
1498 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1499 		if (pol->type == policy->type &&  in xfrm_policy_insert_inexact_list()
1500 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_inexact_list()
1501 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_inexact_list()
1502 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_inexact_list()
1503 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_inexact_list()
1505 			delpol = pol;  in xfrm_policy_insert_inexact_list()
1506 			if (policy->priority > pol->priority)  in xfrm_policy_insert_inexact_list()
1508 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_inexact_list()
1509 			newpos = &pol->bydst_inexact_list;  in xfrm_policy_insert_inexact_list()
1521 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1522 		pol->pos = i;  in xfrm_policy_insert_inexact_list()
1531 	struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;  in xfrm_policy_insert_list()  local
1533 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_insert_list()
1534 		if (pol->type == policy->type &&  in xfrm_policy_insert_list()
1535 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_list()
1536 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_list()
1537 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_list()
1538 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_list()
1542 			delpol = pol;  in xfrm_policy_insert_list()
1543 			if (policy->priority > pol->priority)  in xfrm_policy_insert_list()
1545 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_list()
1546 			newpos = pol;  in xfrm_policy_insert_list()
1613 	struct xfrm_policy *pol;  in __xfrm_policy_bysel_ctx()  local
1618 	hlist_for_each_entry(pol, chain, bydst) {  in __xfrm_policy_bysel_ctx()
1619 		if (pol->type == type &&  in __xfrm_policy_bysel_ctx()
1620 		    pol->if_id == if_id &&  in __xfrm_policy_bysel_ctx()
1621 		    xfrm_policy_mark_match(mark, pol) &&  in __xfrm_policy_bysel_ctx()
1622 		    !selector_cmp(sel, &pol->selector) &&  in __xfrm_policy_bysel_ctx()
1623 		    xfrm_sec_ctx_match(ctx, pol->security))  in __xfrm_policy_bysel_ctx()
1624 			return pol;  in __xfrm_policy_bysel_ctx()
1636 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_policy_bysel_ctx()  local
1660 		pol = NULL;  in xfrm_policy_bysel_ctx()
1670 			if (!pol || tmp->pos < pol->pos)  in xfrm_policy_bysel_ctx()
1671 				pol = tmp;  in xfrm_policy_bysel_ctx()
1674 		pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,  in xfrm_policy_bysel_ctx()
1678 	if (pol) {  in xfrm_policy_bysel_ctx()
1679 		xfrm_pol_hold(pol);  in xfrm_policy_bysel_ctx()
1681 			*err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_bysel_ctx()
1684 				return pol;  in xfrm_policy_bysel_ctx()
1686 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
1688 		ret = pol;  in xfrm_policy_bysel_ctx()
1704 	struct xfrm_policy *pol, *ret;  in xfrm_policy_byid()  local
1715 	hlist_for_each_entry(pol, chain, byidx) {  in xfrm_policy_byid()
1716 		if (pol->type == type && pol->index == id &&  in xfrm_policy_byid()
1717 		    pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {  in xfrm_policy_byid()
1718 			xfrm_pol_hold(pol);  in xfrm_policy_byid()
1721 								pol->security);  in xfrm_policy_byid()
1724 					return pol;  in xfrm_policy_byid()
1726 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
1728 			ret = pol;  in xfrm_policy_byid()
1744 	struct xfrm_policy *pol;  in xfrm_policy_flush_secctx_check()  local
1747 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush_secctx_check()
1748 		if (pol->walk.dead ||  in xfrm_policy_flush_secctx_check()
1749 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||  in xfrm_policy_flush_secctx_check()
1750 		    pol->type != type)  in xfrm_policy_flush_secctx_check()
1753 		err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_flush_secctx_check()
1755 			xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_policy_flush_secctx_check()
1772 	struct xfrm_policy *pol;  in xfrm_policy_flush()  local
1781 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush()
1782 		dir = xfrm_policy_id2dir(pol->index);  in xfrm_policy_flush()
1783 		if (pol->walk.dead ||  in xfrm_policy_flush()
1785 		    pol->type != type)  in xfrm_policy_flush()
1788 		__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
1791 		xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
1792 		xfrm_policy_kill(pol);  in xfrm_policy_flush()
1810 	struct xfrm_policy *pol;  in xfrm_policy_walk()  local
1831 		pol = container_of(x, struct xfrm_policy, walk);  in xfrm_policy_walk()
1833 		    walk->type != pol->type)  in xfrm_policy_walk()
1835 		error = func(pol, xfrm_policy_id2dir(pol->index),  in xfrm_policy_walk()
1879 static int xfrm_policy_match(const struct xfrm_policy *pol,  in xfrm_policy_match()  argument
1883 	const struct xfrm_selector *sel = &pol->selector;  in xfrm_policy_match()
1887 	if (pol->family != family ||  in xfrm_policy_match()
1888 	    pol->if_id != if_id ||  in xfrm_policy_match()
1889 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||  in xfrm_policy_match()
1890 	    pol->type != type)  in xfrm_policy_match()
1895 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,  in xfrm_policy_match()
2010 	struct xfrm_policy *pol;  in __xfrm_policy_eval_candidates()  local
2015 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in __xfrm_policy_eval_candidates()
2018 		if (pol->priority > priority)  in __xfrm_policy_eval_candidates()
2021 		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);  in __xfrm_policy_eval_candidates()
2031 			if (pol->priority == priority &&  in __xfrm_policy_eval_candidates()
2032 			    prefer->pos < pol->pos)  in __xfrm_policy_eval_candidates()
2036 		return pol;  in __xfrm_policy_eval_candidates()
2075 	struct xfrm_policy *pol, *ret;  in xfrm_policy_lookup_bytype()  local
2093 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
2094 		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);  in xfrm_policy_lookup_bytype()
2103 			ret = pol;  in xfrm_policy_lookup_bytype()
2112 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,  in xfrm_policy_lookup_bytype()
2114 	if (pol) {  in xfrm_policy_lookup_bytype()
2115 		ret = pol;  in xfrm_policy_lookup_bytype()
2116 		if (IS_ERR(pol))  in xfrm_policy_lookup_bytype()
2137 	struct xfrm_policy *pol;  in xfrm_policy_lookup()  local
2139 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,  in xfrm_policy_lookup()
2141 	if (pol != NULL)  in xfrm_policy_lookup()
2142 		return pol;  in xfrm_policy_lookup()
2152 	struct xfrm_policy *pol;  in xfrm_sk_policy_lookup()  local
2156 	pol = rcu_dereference(sk->sk_policy[dir]);  in xfrm_sk_policy_lookup()
2157 	if (pol != NULL) {  in xfrm_sk_policy_lookup()
2161 		if (pol->family != family) {  in xfrm_sk_policy_lookup()
2162 			pol = NULL;  in xfrm_sk_policy_lookup()
2166 		match = xfrm_selector_match(&pol->selector, fl, family);  in xfrm_sk_policy_lookup()
2168 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v ||  in xfrm_sk_policy_lookup()
2169 			    pol->if_id != if_id) {  in xfrm_sk_policy_lookup()
2170 				pol = NULL;  in xfrm_sk_policy_lookup()
2173 			err = security_xfrm_policy_lookup(pol->security,  in xfrm_sk_policy_lookup()
2177 				if (!xfrm_pol_hold_rcu(pol))  in xfrm_sk_policy_lookup()
2180 				pol = NULL;  in xfrm_sk_policy_lookup()
2182 				pol = ERR_PTR(err);  in xfrm_sk_policy_lookup()
2185 			pol = NULL;  in xfrm_sk_policy_lookup()
2189 	return pol;  in xfrm_sk_policy_lookup()
2192 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
2194 	struct net *net = xp_net(pol);  in __xfrm_policy_link()
2196 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
2198 	xfrm_pol_hold(pol);  in __xfrm_policy_link()
2201 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,  in __xfrm_policy_unlink()  argument
2204 	struct net *net = xp_net(pol);  in __xfrm_policy_unlink()
2206 	if (list_empty(&pol->walk.all))  in __xfrm_policy_unlink()
2210 	if (!hlist_unhashed(&pol->bydst)) {  in __xfrm_policy_unlink()
2211 		hlist_del_rcu(&pol->bydst);  in __xfrm_policy_unlink()
2212 		hlist_del_init(&pol->bydst_inexact_list);  in __xfrm_policy_unlink()
2213 		hlist_del(&pol->byidx);  in __xfrm_policy_unlink()
2216 	list_del_init(&pol->walk.all);  in __xfrm_policy_unlink()
2219 	return pol;  in __xfrm_policy_unlink()
2222 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
2224 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
2227 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
2229 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
2232 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
2234 	struct net *net = xp_net(pol);  in xfrm_policy_delete()
2237 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
2239 	if (pol) {  in xfrm_policy_delete()
2240 		xfrm_policy_kill(pol);  in xfrm_policy_delete()
2247 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
2253 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)  in xfrm_sk_policy_insert()
2260 	if (pol) {  in xfrm_sk_policy_insert()
2261 		pol->curlft.add_time = ktime_get_real_seconds();  in xfrm_sk_policy_insert()
2262 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
2263 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
2265 	rcu_assign_pointer(sk->sk_policy[dir], pol);  in xfrm_sk_policy_insert()
2267 		if (pol)  in xfrm_sk_policy_insert()
2268 			xfrm_policy_requeue(old_pol, pol);  in xfrm_sk_policy_insert()
2749 	struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);  in xfrm_policy_queue_process()  local
2750 	struct net *net = xp_net(pol);  in xfrm_policy_queue_process()
2751 	struct xfrm_policy_queue *pq = &pol->polq;  in xfrm_policy_queue_process()
2767 	skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2785 			xfrm_pol_hold(pol);  in xfrm_policy_queue_process()
2803 		skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2822 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2828 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2836 	struct xfrm_policy *pol = xdst->pols[0];  in xdst_queue_output()  local
2837 	struct xfrm_policy_queue *pq = &pol->polq;  in xdst_queue_output()
2861 		xfrm_pol_put(pol);  in xdst_queue_output()
2866 		xfrm_pol_hold(pol);  in xdst_queue_output()
3512 	struct xfrm_policy *pol;  in __xfrm_policy_check()  local
3561 	pol = NULL;  in __xfrm_policy_check()
3564 		pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);  in __xfrm_policy_check()
3565 		if (IS_ERR(pol)) {  in __xfrm_policy_check()
3571 	if (!pol)  in __xfrm_policy_check()
3572 		pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);  in __xfrm_policy_check()
3574 	if (IS_ERR(pol)) {  in __xfrm_policy_check()
3579 	if (!pol) {  in __xfrm_policy_check()
3588 	pol->curlft.use_time = ktime_get_real_seconds();  in __xfrm_policy_check()
3590 	pols[0] = pol;  in __xfrm_policy_check()
3608 	if (pol->action == XFRM_POLICY_ALLOW) {  in __xfrm_policy_check()
3621 			if (pols[pi] != pol &&  in __xfrm_policy_check()
4261 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_migrate_policy_find()  local
4267 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
4268 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4269 		    pol->type == type) {  in xfrm_migrate_policy_find()
4270 			ret = pol;  in xfrm_migrate_policy_find()
4276 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_migrate_policy_find()
4277 		if ((pol->priority >= priority) && ret)  in xfrm_migrate_policy_find()
4280 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4281 		    pol->type == type) {  in xfrm_migrate_policy_find()
4282 			ret = pol;  in xfrm_migrate_policy_find()
4324 static int xfrm_policy_migrate(struct xfrm_policy *pol,  in xfrm_policy_migrate()  argument
4330 	write_lock_bh(&pol->lock);  in xfrm_policy_migrate()
4331 	if (unlikely(pol->walk.dead)) {  in xfrm_policy_migrate()
4333 		write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4337 	for (i = 0; i < pol->xfrm_nr; i++) {  in xfrm_policy_migrate()
4339 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))  in xfrm_policy_migrate()
4342 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&  in xfrm_policy_migrate()
4343 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)  in xfrm_policy_migrate()
4346 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,  in xfrm_policy_migrate()
4347 			       sizeof(pol->xfrm_vec[i].id.daddr));  in xfrm_policy_migrate()
4348 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,  in xfrm_policy_migrate()
4349 			       sizeof(pol->xfrm_vec[i].saddr));  in xfrm_policy_migrate()
4350 			pol->xfrm_vec[i].encap_family = mp->new_family;  in xfrm_policy_migrate()
4352 			atomic_inc(&pol->genid);  in xfrm_policy_migrate()
4356 	write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4399 	struct xfrm_policy *pol = NULL;  in xfrm_migrate()  local
4415 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {  in xfrm_migrate()
4437 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)  in xfrm_migrate()
4449 	xfrm_pol_put(pol);  in xfrm_migrate()
4456 	if (pol)  in xfrm_migrate()
4457 		xfrm_pol_put(pol);  in xfrm_migrate()