Lines Matching +full:sp +full:- +full:disabled +full:- +full:ports
1 // SPDX-License-Identifier: GPL-2.0-only
12 * Split up af-specific portion
80 * +---- root_d: sorted by daddr:prefix
84 * | +- root: sorted by saddr/prefix
92 * | +- coarse policies and all any:daddr policies
94 * +---- root_s: sorted by saddr:prefix
102 * +---- coarse policies and all any:any policies
105 * 1. any:any list from top-level xfrm_pol_inexact_bin
194 return refcount_inc_not_zero(&policy->refcnt); in xfrm_pol_hold_rcu()
200 const struct flowi4 *fl4 = &fl->u.ip4; in __xfrm4_selector_match()
202 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && in __xfrm4_selector_match()
203 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && in __xfrm4_selector_match()
204 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm4_selector_match()
205 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm4_selector_match()
206 (fl4->flowi4_proto == sel->proto || !sel->proto) && in __xfrm4_selector_match()
207 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); in __xfrm4_selector_match()
213 const struct flowi6 *fl6 = &fl->u.ip6; in __xfrm6_selector_match()
215 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && in __xfrm6_selector_match()
216 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && in __xfrm6_selector_match()
217 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm6_selector_match()
218 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm6_selector_match()
219 (fl6->flowi6_proto == sel->proto || !sel->proto) && in __xfrm6_selector_match()
220 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); in __xfrm6_selector_match()
264 return ERR_PTR(-EAFNOSUPPORT); in __xfrm_dst_lookup()
266 dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark); in __xfrm_dst_lookup()
281 xfrm_address_t *saddr = &x->props.saddr; in xfrm_dst_lookup()
282 xfrm_address_t *daddr = &x->id.daddr; in xfrm_dst_lookup()
285 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { in xfrm_dst_lookup()
286 saddr = x->coaddr; in xfrm_dst_lookup()
289 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { in xfrm_dst_lookup()
291 daddr = x->coaddr; in xfrm_dst_lookup()
308 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) in make_jiffies()
309 return MAX_SCHEDULE_TIMEOUT-1; in make_jiffies()
322 read_lock(&xp->lock); in xfrm_policy_timer()
324 if (unlikely(xp->walk.dead)) in xfrm_policy_timer()
327 dir = xfrm_policy_id2dir(xp->index); in xfrm_policy_timer()
329 if (xp->lft.hard_add_expires_seconds) { in xfrm_policy_timer()
330 time64_t tmo = xp->lft.hard_add_expires_seconds + in xfrm_policy_timer()
331 xp->curlft.add_time - now; in xfrm_policy_timer()
337 if (xp->lft.hard_use_expires_seconds) { in xfrm_policy_timer()
338 time64_t tmo = xp->lft.hard_use_expires_seconds + in xfrm_policy_timer()
339 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
345 if (xp->lft.soft_add_expires_seconds) { in xfrm_policy_timer()
346 time64_t tmo = xp->lft.soft_add_expires_seconds + in xfrm_policy_timer()
347 xp->curlft.add_time - now; in xfrm_policy_timer()
355 if (xp->lft.soft_use_expires_seconds) { in xfrm_policy_timer()
356 time64_t tmo = xp->lft.soft_use_expires_seconds + in xfrm_policy_timer()
357 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
369 !mod_timer(&xp->timer, jiffies + make_jiffies(next))) in xfrm_policy_timer()
373 read_unlock(&xp->lock); in xfrm_policy_timer()
378 read_unlock(&xp->lock); in xfrm_policy_timer()
395 write_pnet(&policy->xp_net, net); in xfrm_policy_alloc()
396 INIT_LIST_HEAD(&policy->walk.all); in xfrm_policy_alloc()
397 INIT_HLIST_NODE(&policy->bydst_inexact_list); in xfrm_policy_alloc()
398 INIT_HLIST_NODE(&policy->bydst); in xfrm_policy_alloc()
399 INIT_HLIST_NODE(&policy->byidx); in xfrm_policy_alloc()
400 rwlock_init(&policy->lock); in xfrm_policy_alloc()
401 refcount_set(&policy->refcnt, 1); in xfrm_policy_alloc()
402 skb_queue_head_init(&policy->polq.hold_queue); in xfrm_policy_alloc()
403 timer_setup(&policy->timer, xfrm_policy_timer, 0); in xfrm_policy_alloc()
404 timer_setup(&policy->polq.hold_timer, in xfrm_policy_alloc()
415 security_xfrm_policy_free(policy->security); in xfrm_policy_destroy_rcu()
423 BUG_ON(!policy->walk.dead); in xfrm_policy_destroy()
425 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) in xfrm_policy_destroy()
429 call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); in xfrm_policy_destroy()
439 write_lock_bh(&policy->lock); in xfrm_policy_kill()
440 policy->walk.dead = 1; in xfrm_policy_kill()
441 write_unlock_bh(&policy->lock); in xfrm_policy_kill()
443 atomic_inc(&policy->genid); in xfrm_policy_kill()
445 if (del_timer(&policy->polq.hold_timer)) in xfrm_policy_kill()
447 skb_queue_purge(&policy->polq.hold_queue); in xfrm_policy_kill()
449 if (del_timer(&policy->timer)) in xfrm_policy_kill()
459 return __idx_hash(index, net->xfrm.policy_idx_hmask); in idx_hash()
469 *dbits = net->xfrm.policy_bydst[dir].dbits4; in __get_hash_thresh()
470 *sbits = net->xfrm.policy_bydst[dir].sbits4; in __get_hash_thresh()
474 *dbits = net->xfrm.policy_bydst[dir].dbits6; in __get_hash_thresh()
475 *sbits = net->xfrm.policy_bydst[dir].sbits6; in __get_hash_thresh()
488 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_bysel()
499 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_bysel()
500 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_bysel()
508 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_direct()
516 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_direct()
517 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_direct()
536 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); in xfrm_dst_hash_transfer()
537 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, in xfrm_dst_hash_transfer()
538 pol->family, nhashmask, dbits, sbits); in xfrm_dst_hash_transfer()
539 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { in xfrm_dst_hash_transfer()
540 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
541 hlist_add_head_rcu(&pol->bydst, ndsttable + h); in xfrm_dst_hash_transfer()
546 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
547 hlist_add_behind_rcu(&pol->bydst, entry0); in xfrm_dst_hash_transfer()
549 entry0 = &pol->bydst; in xfrm_dst_hash_transfer()
567 h = __idx_hash(pol->index, nhashmask); in xfrm_idx_hash_transfer()
568 hlist_add_head(&pol->byidx, nidxtable+h); in xfrm_idx_hash_transfer()
574 return ((old_hmask + 1) << 1) - 1; in xfrm_new_hash_mask()
579 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_resize()
589 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
590 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
592 odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, in xfrm_bydst_resize()
593 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_bydst_resize()
595 for (i = hmask; i >= 0; i--) in xfrm_bydst_resize()
598 rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); in xfrm_bydst_resize()
599 net->xfrm.policy_bydst[dir].hmask = nhashmask; in xfrm_bydst_resize()
601 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
602 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
611 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_resize()
614 struct hlist_head *oidx = net->xfrm.policy_byidx; in xfrm_byidx_resize()
621 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
623 for (i = hmask; i >= 0; i--) in xfrm_byidx_resize()
626 net->xfrm.policy_byidx = nidx; in xfrm_byidx_resize()
627 net->xfrm.policy_idx_hmask = nhashmask; in xfrm_byidx_resize()
629 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
636 unsigned int cnt = net->xfrm.policy_count[dir]; in xfrm_bydst_should_resize()
637 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_should_resize()
651 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_should_resize()
662 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; in xfrm_spd_getinfo()
663 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; in xfrm_spd_getinfo()
664 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; in xfrm_spd_getinfo()
665 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
666 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
667 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
668 si->spdhcnt = net->xfrm.policy_idx_hmask; in xfrm_spd_getinfo()
669 si->spdhmcnt = xfrm_policy_hashmax; in xfrm_spd_getinfo()
701 .family = pol->family, in xfrm_policy_inexact_alloc_bin()
702 .type = pol->type, in xfrm_policy_inexact_alloc_bin()
704 .if_id = pol->if_id, in xfrm_policy_inexact_alloc_bin()
708 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
720 bin->k = k; in xfrm_policy_inexact_alloc_bin()
721 INIT_HLIST_HEAD(&bin->hhead); in xfrm_policy_inexact_alloc_bin()
722 bin->root_d = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
723 bin->root_s = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
724 seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
727 &bin->k, &bin->head, in xfrm_policy_inexact_alloc_bin()
730 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins); in xfrm_policy_inexact_alloc_bin()
761 addr = &policy->selector.saddr; in xfrm_policy_inexact_insert_use_any_list()
762 prefixlen = policy->selector.prefixlen_s; in xfrm_policy_inexact_insert_use_any_list()
765 policy->family, in xfrm_policy_inexact_insert_use_any_list()
767 addr = &policy->selector.daddr; in xfrm_policy_inexact_insert_use_any_list()
768 prefixlen = policy->selector.prefixlen_d; in xfrm_policy_inexact_insert_use_any_list()
770 policy->family, in xfrm_policy_inexact_insert_use_any_list()
778 node->addr = *addr; in xfrm_pol_inexact_node_init()
779 node->prefixlen = prefixlen; in xfrm_pol_inexact_node_init()
806 mask = ~0U << (32 - prefixlen); in xfrm_policy_addr_delta()
807 ma = ntohl(a->a4) & mask; in xfrm_policy_addr_delta()
808 mb = ntohl(b->a4) & mask; in xfrm_policy_addr_delta()
810 delta = -1; in xfrm_policy_addr_delta()
819 delta = memcmp(a->a6, b->a6, pdw << 2); in xfrm_policy_addr_delta()
824 mask = ~0U << (32 - pbi); in xfrm_policy_addr_delta()
825 ma = ntohl(a->a6[pdw]) & mask; in xfrm_policy_addr_delta()
826 mb = ntohl(b->a6[pdw]) & mask; in xfrm_policy_addr_delta()
828 delta = -1; in xfrm_policy_addr_delta()
850 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_policy_inexact_list_reinsert()
854 if (policy->walk.dead || !policy->bydst_reinsert) in xfrm_policy_inexact_list_reinsert()
857 WARN_ON_ONCE(policy->family != family); in xfrm_policy_inexact_list_reinsert()
859 policy->bydst_reinsert = false; in xfrm_policy_inexact_list_reinsert()
860 hlist_for_each_entry(p, &n->hhead, bydst) { in xfrm_policy_inexact_list_reinsert()
861 if (policy->priority > p->priority) in xfrm_policy_inexact_list_reinsert()
862 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
863 else if (policy->priority == p->priority && in xfrm_policy_inexact_list_reinsert()
864 policy->pos > p->pos) in xfrm_policy_inexact_list_reinsert()
865 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
870 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_inexact_list_reinsert()
871 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_policy_inexact_list_reinsert()
873 hlist_add_head_rcu(&policy->bydst, &n->hhead); in xfrm_policy_inexact_list_reinsert()
883 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr, in xfrm_policy_inexact_list_reinsert()
884 &n->addr, in xfrm_policy_inexact_list_reinsert()
885 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
887 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr, in xfrm_policy_inexact_list_reinsert()
888 &n->addr, in xfrm_policy_inexact_list_reinsert()
889 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
912 WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); in xfrm_policy_inexact_node_reinsert()
915 p = &new->rb_node; in xfrm_policy_inexact_node_reinsert()
923 prefixlen = min(node->prefixlen, n->prefixlen); in xfrm_policy_inexact_node_reinsert()
925 delta = xfrm_policy_addr_delta(&n->addr, &node->addr, in xfrm_policy_inexact_node_reinsert()
928 p = &parent->rb_left; in xfrm_policy_inexact_node_reinsert()
930 p = &parent->rb_right; in xfrm_policy_inexact_node_reinsert()
932 bool same_prefixlen = node->prefixlen == n->prefixlen; in xfrm_policy_inexact_node_reinsert()
935 hlist_for_each_entry(tmp, &n->hhead, bydst) { in xfrm_policy_inexact_node_reinsert()
936 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_reinsert()
937 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_reinsert()
940 node->prefixlen = prefixlen; in xfrm_policy_inexact_node_reinsert()
956 rb_link_node_rcu(&n->node, parent, p); in xfrm_policy_inexact_node_reinsert()
957 rb_insert_color(&n->node, new); in xfrm_policy_inexact_node_reinsert()
970 /* To-be-merged node v has a subtree. in xfrm_policy_inexact_node_merge()
972 * Dismantle it and insert its nodes to n->root. in xfrm_policy_inexact_node_merge()
974 while ((rnode = rb_first(&v->root)) != NULL) { in xfrm_policy_inexact_node_merge()
976 rb_erase(&node->node, &v->root); in xfrm_policy_inexact_node_merge()
977 xfrm_policy_inexact_node_reinsert(net, node, &n->root, in xfrm_policy_inexact_node_merge()
981 hlist_for_each_entry(tmp, &v->hhead, bydst) { in xfrm_policy_inexact_node_merge()
982 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_merge()
983 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_merge()
999 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1006 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1007 node->prefixlen, in xfrm_policy_inexact_insert_node()
1009 if (delta == 0 && prefixlen >= node->prefixlen) { in xfrm_policy_inexact_insert_node()
1015 p = &parent->rb_left; in xfrm_policy_inexact_insert_node()
1017 p = &parent->rb_right; in xfrm_policy_inexact_insert_node()
1019 if (prefixlen < node->prefixlen) { in xfrm_policy_inexact_insert_node()
1020 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1027 * to be removed and re-inserted with the smaller in xfrm_policy_inexact_insert_node()
1031 rb_erase(&node->node, root); in xfrm_policy_inexact_insert_node()
1039 * prefixlen. Merge the to-be-reinserted in xfrm_policy_inexact_insert_node()
1048 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1060 rb_link_node_rcu(&node->node, parent, p); in xfrm_policy_inexact_insert_node()
1061 rb_insert_color(&node->node, root); in xfrm_policy_inexact_insert_node()
1074 xfrm_policy_inexact_gc_tree(&node->root, rm); in xfrm_policy_inexact_gc_tree()
1077 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) { in xfrm_policy_inexact_gc_tree()
1082 rb_erase(&node->node, r); in xfrm_policy_inexact_gc_tree()
1089 write_seqcount_begin(&b->count); in __xfrm_policy_inexact_prune_bin()
1090 xfrm_policy_inexact_gc_tree(&b->root_d, net_exit); in __xfrm_policy_inexact_prune_bin()
1091 xfrm_policy_inexact_gc_tree(&b->root_s, net_exit); in __xfrm_policy_inexact_prune_bin()
1092 write_seqcount_end(&b->count); in __xfrm_policy_inexact_prune_bin()
1094 if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || in __xfrm_policy_inexact_prune_bin()
1095 !hlist_empty(&b->hhead)) { in __xfrm_policy_inexact_prune_bin()
1100 if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head, in __xfrm_policy_inexact_prune_bin()
1102 list_del(&b->inexact_bins); in __xfrm_policy_inexact_prune_bin()
1109 struct net *net = read_pnet(&b->k.net); in xfrm_policy_inexact_prune_bin()
1111 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1113 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1120 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in __xfrm_policy_inexact_flush()
1122 list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) in __xfrm_policy_inexact_flush()
1134 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_chain()
1137 return &bin->hhead; in xfrm_policy_inexact_alloc_chain()
1139 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1140 policy->family, in xfrm_policy_inexact_alloc_chain()
1141 policy->selector.prefixlen_d)) { in xfrm_policy_inexact_alloc_chain()
1142 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1144 &bin->root_s, in xfrm_policy_inexact_alloc_chain()
1145 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1146 policy->family, in xfrm_policy_inexact_alloc_chain()
1147 policy->selector.prefixlen_s, in xfrm_policy_inexact_alloc_chain()
1149 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1153 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1157 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1159 &bin->root_d, in xfrm_policy_inexact_alloc_chain()
1160 &policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1161 policy->family, in xfrm_policy_inexact_alloc_chain()
1162 policy->selector.prefixlen_d, dir); in xfrm_policy_inexact_alloc_chain()
1163 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1168 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1169 policy->family, in xfrm_policy_inexact_alloc_chain()
1170 policy->selector.prefixlen_s)) in xfrm_policy_inexact_alloc_chain()
1171 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1173 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1175 &n->root, in xfrm_policy_inexact_alloc_chain()
1176 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1177 policy->family, in xfrm_policy_inexact_alloc_chain()
1178 policy->selector.prefixlen_s, dir); in xfrm_policy_inexact_alloc_chain()
1179 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1183 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1196 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1199 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_insert()
1204 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1210 return ERR_PTR(-EEXIST); in xfrm_policy_inexact_insert()
1213 chain = &net->xfrm.policy_inexact[dir]; in xfrm_policy_inexact_insert()
1241 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock); in xfrm_hash_rebuild()
1243 lbits4 = net->xfrm.policy_hthresh.lbits4; in xfrm_hash_rebuild()
1244 rbits4 = net->xfrm.policy_hthresh.rbits4; in xfrm_hash_rebuild()
1245 lbits6 = net->xfrm.policy_hthresh.lbits6; in xfrm_hash_rebuild()
1246 rbits6 = net->xfrm.policy_hthresh.rbits6; in xfrm_hash_rebuild()
1247 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq)); in xfrm_hash_rebuild()
1249 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1250 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1255 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1259 if (policy->walk.dead) in xfrm_hash_rebuild()
1262 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1267 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1275 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1284 if (policy->selector.prefixlen_d < dbits || in xfrm_hash_rebuild()
1285 policy->selector.prefixlen_s < sbits) in xfrm_hash_rebuild()
1301 &net->xfrm.policy_inexact[dir], in xfrm_hash_rebuild()
1303 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1304 hlist_del_init(&policy->bydst_inexact_list); in xfrm_hash_rebuild()
1307 hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_hash_rebuild()
1308 odst = net->xfrm.policy_bydst[dir].table; in xfrm_hash_rebuild()
1309 for (i = hmask; i >= 0; i--) { in xfrm_hash_rebuild()
1311 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1315 net->xfrm.policy_bydst[dir].dbits4 = rbits4; in xfrm_hash_rebuild()
1316 net->xfrm.policy_bydst[dir].sbits4 = lbits4; in xfrm_hash_rebuild()
1317 net->xfrm.policy_bydst[dir].dbits6 = rbits6; in xfrm_hash_rebuild()
1318 net->xfrm.policy_bydst[dir].sbits6 = lbits6; in xfrm_hash_rebuild()
1321 net->xfrm.policy_bydst[dir].dbits4 = lbits4; in xfrm_hash_rebuild()
1322 net->xfrm.policy_bydst[dir].sbits4 = rbits4; in xfrm_hash_rebuild()
1323 net->xfrm.policy_bydst[dir].dbits6 = lbits6; in xfrm_hash_rebuild()
1324 net->xfrm.policy_bydst[dir].sbits6 = rbits6; in xfrm_hash_rebuild()
1328 /* re-insert all policies by order of creation */ in xfrm_hash_rebuild()
1329 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1330 if (policy->walk.dead) in xfrm_hash_rebuild()
1332 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1338 chain = policy_hash_bysel(net, &policy->selector, in xfrm_hash_rebuild()
1339 policy->family, dir); in xfrm_hash_rebuild()
1349 if (policy->priority >= pol->priority) in xfrm_hash_rebuild()
1350 newpos = &pol->bydst; in xfrm_hash_rebuild()
1354 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_hash_rebuild()
1355 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_hash_rebuild()
1357 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_hash_rebuild()
1362 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1363 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1370 schedule_work(&net->xfrm.policy_hthresh.work); in xfrm_policy_hash_rebuild()
1385 idx = (net->xfrm.idx_generator | dir); in xfrm_gen_index()
1386 net->xfrm.idx_generator += 8; in xfrm_gen_index()
1394 list = net->xfrm.policy_byidx + idx_hash(net, idx); in xfrm_gen_index()
1397 if (p->index == idx) { in xfrm_gen_index()
1425 struct xfrm_policy_queue *pq = &old->polq; in xfrm_policy_requeue()
1428 if (skb_queue_empty(&pq->hold_queue)) in xfrm_policy_requeue()
1433 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1434 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_requeue()
1435 if (del_timer(&pq->hold_timer)) in xfrm_policy_requeue()
1437 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1439 pq = &new->polq; in xfrm_policy_requeue()
1441 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1442 skb_queue_splice(&list, &pq->hold_queue); in xfrm_policy_requeue()
1443 pq->timeout = XFRM_QUEUE_TMO_MIN; in xfrm_policy_requeue()
1444 if (!mod_timer(&pq->hold_timer, jiffies)) in xfrm_policy_requeue()
1446 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1452 return mark->v == pol->mark.v && mark->m == pol->mark.m; in xfrm_policy_mark_match()
1458 u32 a = k->type << 24 | k->dir << 16 | k->family; in xfrm_pol_bin_key()
1460 return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)), in xfrm_pol_bin_key()
1468 return xfrm_pol_bin_key(&b->k, 0, seed); in xfrm_pol_bin_obj()
1474 const struct xfrm_pol_inexact_key *key = arg->key; in xfrm_pol_bin_cmp()
1478 if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net))) in xfrm_pol_bin_cmp()
1479 return -1; in xfrm_pol_bin_cmp()
1481 ret = b->k.dir ^ key->dir; in xfrm_pol_bin_cmp()
1485 ret = b->k.type ^ key->type; in xfrm_pol_bin_cmp()
1489 ret = b->k.family ^ key->family; in xfrm_pol_bin_cmp()
1493 return b->k.if_id ^ key->if_id; in xfrm_pol_bin_cmp()
1512 if (pol->type == policy->type && in xfrm_policy_insert_inexact_list()
1513 pol->if_id == policy->if_id && in xfrm_policy_insert_inexact_list()
1514 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_inexact_list()
1515 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_inexact_list()
1516 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_inexact_list()
1519 if (policy->priority > pol->priority) in xfrm_policy_insert_inexact_list()
1521 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_inexact_list()
1522 newpos = &pol->bydst_inexact_list; in xfrm_policy_insert_inexact_list()
1529 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_inexact_list()
1530 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); in xfrm_policy_insert_inexact_list()
1532 hlist_add_head_rcu(&policy->bydst_inexact_list, chain); in xfrm_policy_insert_inexact_list()
1535 pol->pos = i; in xfrm_policy_insert_inexact_list()
1547 if (pol->type == policy->type && in xfrm_policy_insert_list()
1548 pol->if_id == policy->if_id && in xfrm_policy_insert_list()
1549 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_list()
1550 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_list()
1551 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_list()
1554 return ERR_PTR(-EEXIST); in xfrm_policy_insert_list()
1556 if (policy->priority > pol->priority) in xfrm_policy_insert_list()
1558 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_list()
1566 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_list()
1567 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); in xfrm_policy_insert_list()
1570 * to speed-up lookups. in xfrm_policy_insert_list()
1572 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_policy_insert_list()
1583 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1584 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); in xfrm_policy_insert()
1591 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1598 if (policy->family == AF_INET) in xfrm_policy_insert()
1607 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index); in xfrm_policy_insert()
1608 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); in xfrm_policy_insert()
1609 policy->curlft.add_time = ktime_get_real_seconds(); in xfrm_policy_insert()
1610 policy->curlft.use_time = 0; in xfrm_policy_insert()
1611 if (!mod_timer(&policy->timer, jiffies + HZ)) in xfrm_policy_insert()
1613 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1618 schedule_work(&net->xfrm.policy_hash_work); in xfrm_policy_insert()
1635 if (pol->type == type && in __xfrm_policy_bysel_ctx()
1636 pol->if_id == if_id && in __xfrm_policy_bysel_ctx()
1638 !selector_cmp(sel, &pol->selector) && in __xfrm_policy_bysel_ctx()
1639 xfrm_sec_ctx_match(ctx, pol->security)) in __xfrm_policy_bysel_ctx()
1656 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1657 chain = policy_hash_bysel(net, sel, sel->family, dir); in xfrm_policy_bysel_ctx()
1663 sel->family, dir, if_id); in xfrm_policy_bysel_ctx()
1665 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1670 &sel->saddr, in xfrm_policy_bysel_ctx()
1671 &sel->daddr)) { in xfrm_policy_bysel_ctx()
1672 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1686 if (!pol || tmp->pos < pol->pos) in xfrm_policy_bysel_ctx()
1697 *err = security_xfrm_policy_delete(pol->security); in xfrm_policy_bysel_ctx()
1699 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1706 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1723 *err = -ENOENT; in xfrm_policy_byid()
1728 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1729 chain = net->xfrm.policy_byidx + idx_hash(net, id); in xfrm_policy_byid()
1732 if (pol->type == type && pol->index == id && in xfrm_policy_byid()
1733 pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { in xfrm_policy_byid()
1737 pol->security); in xfrm_policy_byid()
1739 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1748 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1763 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush_secctx_check()
1764 if (pol->walk.dead || in xfrm_policy_flush_secctx_check()
1765 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_policy_flush_secctx_check()
1766 pol->type != type) in xfrm_policy_flush_secctx_check()
1769 err = security_xfrm_policy_delete(pol->security); in xfrm_policy_flush_secctx_check()
1785 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush_secctx_check()
1786 if (pol->walk.dead || in xfrm_dev_policy_flush_secctx_check()
1787 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_dev_policy_flush_secctx_check()
1788 pol->xdo.dev != dev) in xfrm_dev_policy_flush_secctx_check()
1791 err = security_xfrm_policy_delete(pol->security); in xfrm_dev_policy_flush_secctx_check()
1819 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1826 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush()
1827 if (pol->walk.dead) in xfrm_policy_flush()
1830 dir = xfrm_policy_id2dir(pol->index); in xfrm_policy_flush()
1832 pol->type != type) in xfrm_policy_flush()
1836 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1841 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1847 err = -ESRCH; in xfrm_policy_flush()
1849 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1860 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1867 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush()
1868 if (pol->walk.dead) in xfrm_dev_policy_flush()
1871 dir = xfrm_policy_id2dir(pol->index); in xfrm_dev_policy_flush()
1873 pol->xdo.dev != dev) in xfrm_dev_policy_flush()
1877 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1882 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1888 err = -ESRCH; in xfrm_dev_policy_flush()
1890 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1903 if (walk->type >= XFRM_POLICY_TYPE_MAX && in xfrm_policy_walk()
1904 walk->type != XFRM_POLICY_TYPE_ANY) in xfrm_policy_walk()
1905 return -EINVAL; in xfrm_policy_walk()
1907 if (list_empty(&walk->walk.all) && walk->seq != 0) in xfrm_policy_walk()
1910 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1911 if (list_empty(&walk->walk.all)) in xfrm_policy_walk()
1912 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); in xfrm_policy_walk()
1914 x = list_first_entry(&walk->walk.all, in xfrm_policy_walk()
1917 list_for_each_entry_from(x, &net->xfrm.policy_all, all) { in xfrm_policy_walk()
1918 if (x->dead) in xfrm_policy_walk()
1921 if (walk->type != XFRM_POLICY_TYPE_ANY && in xfrm_policy_walk()
1922 walk->type != pol->type) in xfrm_policy_walk()
1924 error = func(pol, xfrm_policy_id2dir(pol->index), in xfrm_policy_walk()
1925 walk->seq, data); in xfrm_policy_walk()
1927 list_move_tail(&walk->walk.all, &x->all); in xfrm_policy_walk()
1930 walk->seq++; in xfrm_policy_walk()
1932 if (walk->seq == 0) { in xfrm_policy_walk()
1933 error = -ENOENT; in xfrm_policy_walk()
1936 list_del_init(&walk->walk.all); in xfrm_policy_walk()
1938 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1945 INIT_LIST_HEAD(&walk->walk.all); in xfrm_policy_walk_init()
1946 walk->walk.dead = 1; in xfrm_policy_walk_init()
1947 walk->type = type; in xfrm_policy_walk_init()
1948 walk->seq = 0; in xfrm_policy_walk_init()
1954 if (list_empty(&walk->walk.all)) in xfrm_policy_walk_done()
1957 spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ in xfrm_policy_walk_done()
1958 list_del(&walk->walk.all); in xfrm_policy_walk_done()
1959 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk_done()
1966 * Returns 0 if policy found, else an -errno.
1972 const struct xfrm_selector *sel = &pol->selector; in xfrm_policy_match()
1973 int ret = -ESRCH; in xfrm_policy_match()
1976 if (pol->family != family || in xfrm_policy_match()
1977 pol->if_id != if_id || in xfrm_policy_match()
1978 (fl->flowi_mark & pol->mark.m) != pol->mark.v || in xfrm_policy_match()
1979 pol->type != type) in xfrm_policy_match()
1984 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid); in xfrm_policy_match()
1999 parent = rcu_dereference_raw(r->rb_node); in xfrm_policy_lookup_inexact_addr()
2006 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_lookup_inexact_addr()
2007 node->prefixlen, family); in xfrm_policy_lookup_inexact_addr()
2009 parent = rcu_dereference_raw(parent->rb_left); in xfrm_policy_lookup_inexact_addr()
2012 parent = rcu_dereference_raw(parent->rb_right); in xfrm_policy_lookup_inexact_addr()
2037 family = b->k.family; in xfrm_policy_find_inexact_candidates()
2039 cand->res[XFRM_POL_CAND_ANY] = &b->hhead; in xfrm_policy_find_inexact_candidates()
2041 n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr, in xfrm_policy_find_inexact_candidates()
2044 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2045 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2048 cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2051 n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2054 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2082 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_lookup()
2097 u32 priority = prefer ? prefer->priority : ~0u; in __xfrm_policy_eval_candidates()
2106 if (pol->priority > priority) in __xfrm_policy_eval_candidates()
2111 if (err != -ESRCH) in __xfrm_policy_eval_candidates()
2119 if (pol->priority == priority && in __xfrm_policy_eval_candidates()
2120 prefer->pos < pol->pos) in __xfrm_policy_eval_candidates()
2139 for (i = 0; i < ARRAY_SIZE(cand->res); i++) { in xfrm_policy_eval_candidates()
2140 tmp = __xfrm_policy_eval_candidates(cand->res[i], in xfrm_policy_eval_candidates()
2175 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_policy_lookup_bytype()
2177 } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); in xfrm_policy_lookup_bytype()
2183 if (err == -ESRCH) in xfrm_policy_lookup_bytype()
2194 if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_lookup_bytype()
2211 if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) in xfrm_policy_lookup_bytype()
2246 pol = rcu_dereference(sk->sk_policy[dir]); in xfrm_sk_policy_lookup()
2251 if (pol->family != family) { in xfrm_sk_policy_lookup()
2256 match = xfrm_selector_match(&pol->selector, fl, family); in xfrm_sk_policy_lookup()
2258 if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || in xfrm_sk_policy_lookup()
2259 pol->if_id != if_id) { in xfrm_sk_policy_lookup()
2263 err = security_xfrm_policy_lookup(pol->security, in xfrm_sk_policy_lookup()
2264 fl->flowi_secid); in xfrm_sk_policy_lookup()
2268 } else if (err == -ESRCH) { in xfrm_sk_policy_lookup()
2285 list_add(&pol->walk.all, &net->xfrm.policy_all); in __xfrm_policy_link()
2286 net->xfrm.policy_count[dir]++; in __xfrm_policy_link()
2295 if (list_empty(&pol->walk.all)) in __xfrm_policy_unlink()
2299 if (!hlist_unhashed(&pol->bydst)) { in __xfrm_policy_unlink()
2300 hlist_del_rcu(&pol->bydst); in __xfrm_policy_unlink()
2301 hlist_del_init(&pol->bydst_inexact_list); in __xfrm_policy_unlink()
2302 hlist_del(&pol->byidx); in __xfrm_policy_unlink()
2305 list_del_init(&pol->walk.all); in __xfrm_policy_unlink()
2306 net->xfrm.policy_count[dir]--; in __xfrm_policy_unlink()
2325 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2327 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2333 return -ENOENT; in xfrm_policy_delete()
2343 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) in xfrm_sk_policy_insert()
2344 return -EINVAL; in xfrm_sk_policy_insert()
2347 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2348 old_pol = rcu_dereference_protected(sk->sk_policy[dir], in xfrm_sk_policy_insert()
2349 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_sk_policy_insert()
2351 pol->curlft.add_time = ktime_get_real_seconds(); in xfrm_sk_policy_insert()
2352 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0); in xfrm_sk_policy_insert()
2355 rcu_assign_pointer(sk->sk_policy[dir], pol); in xfrm_sk_policy_insert()
2365 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2379 newp->selector = old->selector; in clone_policy()
2380 if (security_xfrm_policy_clone(old->security, in clone_policy()
2381 &newp->security)) { in clone_policy()
2385 newp->lft = old->lft; in clone_policy()
2386 newp->curlft = old->curlft; in clone_policy()
2387 newp->mark = old->mark; in clone_policy()
2388 newp->if_id = old->if_id; in clone_policy()
2389 newp->action = old->action; in clone_policy()
2390 newp->flags = old->flags; in clone_policy()
2391 newp->xfrm_nr = old->xfrm_nr; in clone_policy()
2392 newp->index = old->index; in clone_policy()
2393 newp->type = old->type; in clone_policy()
2394 newp->family = old->family; in clone_policy()
2395 memcpy(newp->xfrm_vec, old->xfrm_vec, in clone_policy()
2396 newp->xfrm_nr*sizeof(struct xfrm_tmpl)); in clone_policy()
2397 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2399 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2413 p = rcu_dereference(osk->sk_policy[i]); in __xfrm_sk_clone_policy()
2417 ret = -ENOMEM; in __xfrm_sk_clone_policy()
2420 rcu_assign_pointer(sk->sk_policy[i], np); in __xfrm_sk_clone_policy()
2435 return -EINVAL; in xfrm_get_saddr()
2436 err = afinfo->get_saddr(net, oif, local, remote, mark); in xfrm_get_saddr()
2454 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { in xfrm_tmpl_resolve_one()
2458 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; in xfrm_tmpl_resolve_one()
2460 if (tmpl->mode == XFRM_MODE_TUNNEL || in xfrm_tmpl_resolve_one()
2461 tmpl->mode == XFRM_MODE_BEET) { in xfrm_tmpl_resolve_one()
2462 remote = &tmpl->id.daddr; in xfrm_tmpl_resolve_one()
2463 local = &tmpl->saddr; in xfrm_tmpl_resolve_one()
2464 if (xfrm_addr_any(local, tmpl->encap_family)) { in xfrm_tmpl_resolve_one()
2465 error = xfrm_get_saddr(net, fl->flowi_oif, in xfrm_tmpl_resolve_one()
2467 tmpl->encap_family, 0); in xfrm_tmpl_resolve_one()
2475 family, policy->if_id); in xfrm_tmpl_resolve_one()
2477 if (x && x->km.state == XFRM_STATE_VALID) { in xfrm_tmpl_resolve_one()
2484 error = (x->km.state == XFRM_STATE_ERROR ? in xfrm_tmpl_resolve_one()
2485 -EINVAL : -EAGAIN); in xfrm_tmpl_resolve_one()
2487 } else if (error == -ESRCH) { in xfrm_tmpl_resolve_one()
2488 error = -EAGAIN; in xfrm_tmpl_resolve_one()
2491 if (!tmpl->optional) in xfrm_tmpl_resolve_one()
2497 for (nx--; nx >= 0; nx--) in xfrm_tmpl_resolve_one()
2514 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { in xfrm_tmpl_resolve()
2515 error = -ENOBUFS; in xfrm_tmpl_resolve()
2534 for (cnx--; cnx >= 0; cnx--) in xfrm_tmpl_resolve()
2543 return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos; in xfrm_get_tos()
2555 return ERR_PTR(-EINVAL); in xfrm_alloc_dst()
2559 dst_ops = &net->xfrm.xfrm4_dst_ops; in xfrm_alloc_dst()
2563 dst_ops = &net->xfrm.xfrm6_dst_ops; in xfrm_alloc_dst()
2574 xdst = ERR_PTR(-ENOBUFS); in xfrm_alloc_dst()
2584 if (dst->ops->family == AF_INET6) { in xfrm_init_path()
2586 path->path_cookie = rt6_get_cookie(rt); in xfrm_init_path()
2587 path->u.rt6.rt6i_nfheader_len = nfheader_len; in xfrm_init_path()
2595 xfrm_policy_get_afinfo(xdst->u.dst.ops->family); in xfrm_fill_dst()
2599 return -EINVAL; in xfrm_fill_dst()
2601 err = afinfo->fill_dst(xdst, dev, fl); in xfrm_fill_dst()
2633 int family = policy->selector.family; in xfrm_bundle_create()
2644 struct dst_entry *dst1 = &xdst->u.dst; in xfrm_bundle_create()
2659 xfrm_dst_set_child(xdst_prev, &xdst->u.dst); in xfrm_bundle_create()
2661 if (xfrm[i]->sel.family == AF_UNSPEC) { in xfrm_bundle_create()
2665 err = -EAFNOSUPPORT; in xfrm_bundle_create()
2670 inner_mode = &xfrm[i]->inner_mode; in xfrm_bundle_create()
2672 xdst->route = dst; in xfrm_bundle_create()
2675 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_bundle_create()
2679 if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) in xfrm_bundle_create()
2680 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]); in xfrm_bundle_create()
2682 family = xfrm[i]->props.family; in xfrm_bundle_create()
2683 oif = fl->flowi_oif ? : fl->flowi_l3mdev; in xfrm_bundle_create()
2692 dst1->xfrm = xfrm[i]; in xfrm_bundle_create()
2693 xdst->xfrm_genid = xfrm[i]->genid; in xfrm_bundle_create()
2695 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_bundle_create()
2696 dst1->lastuse = now; in xfrm_bundle_create()
2698 dst1->input = dst_discard; in xfrm_bundle_create()
2701 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); in xfrm_bundle_create()
2703 dst1->output = afinfo->output; in xfrm_bundle_create()
2705 dst1->output = dst_discard_out; in xfrm_bundle_create()
2710 header_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2711 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) in xfrm_bundle_create()
2712 nfheader_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2713 trailer_len += xfrm[i]->props.trailer_len; in xfrm_bundle_create()
2717 xdst0->path = dst; in xfrm_bundle_create()
2719 err = -ENODEV; in xfrm_bundle_create()
2720 dev = dst->dev; in xfrm_bundle_create()
2728 xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { in xfrm_bundle_create()
2733 xdst_prev->u.dst.header_len = header_len; in xfrm_bundle_create()
2734 xdst_prev->u.dst.trailer_len = trailer_len; in xfrm_bundle_create()
2735 header_len -= xdst_prev->u.dst.xfrm->props.header_len; in xfrm_bundle_create()
2736 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; in xfrm_bundle_create()
2739 return &xdst0->u.dst; in xfrm_bundle_create()
2746 dst_release_immediate(&xdst0->u.dst); in xfrm_bundle_create()
2767 *num_xfrms = pols[0]->xfrm_nr; in xfrm_expand_policies()
2770 if (pols[0]->action == XFRM_POLICY_ALLOW && in xfrm_expand_policies()
2771 pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in xfrm_expand_policies()
2776 pols[0]->if_id); in xfrm_expand_policies()
2784 (*num_xfrms) += pols[1]->xfrm_nr; in xfrm_expand_policies()
2789 if (pols[i]->action != XFRM_POLICY_ALLOW) { in xfrm_expand_policies()
2790 *num_xfrms = -1; in xfrm_expand_policies()
2817 if (err != -EAGAIN) in xfrm_resolve_and_create_bundle()
2829 xdst->num_xfrms = err; in xfrm_resolve_and_create_bundle()
2830 xdst->num_pols = num_pols; in xfrm_resolve_and_create_bundle()
2831 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_resolve_and_create_bundle()
2832 xdst->policy_genid = atomic_read(&pols[0]->genid); in xfrm_resolve_and_create_bundle()
2844 struct xfrm_policy_queue *pq = &pol->polq; in xfrm_policy_queue_process()
2849 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2850 skb = skb_peek(&pq->hold_queue); in xfrm_policy_queue_process()
2852 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2856 sk = skb->sk; in xfrm_policy_queue_process()
2859 skb_mark = skb->mark; in xfrm_policy_queue_process()
2860 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2861 xfrm_decode_session(skb, &fl, dst->ops->family); in xfrm_policy_queue_process()
2862 skb->mark = skb_mark; in xfrm_policy_queue_process()
2863 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2870 if (dst->flags & DST_XFRM_QUEUE) { in xfrm_policy_queue_process()
2873 if (pq->timeout >= XFRM_QUEUE_TMO_MAX) in xfrm_policy_queue_process()
2876 pq->timeout = pq->timeout << 1; in xfrm_policy_queue_process()
2877 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout)) in xfrm_policy_queue_process()
2886 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2887 pq->timeout = 0; in xfrm_policy_queue_process()
2888 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_queue_process()
2889 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2895 skb_mark = skb->mark; in xfrm_policy_queue_process()
2896 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2897 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family); in xfrm_policy_queue_process()
2898 skb->mark = skb_mark; in xfrm_policy_queue_process()
2901 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0); in xfrm_policy_queue_process()
2911 dst_output(net, skb->sk, skb); in xfrm_policy_queue_process()
2919 pq->timeout = 0; in xfrm_policy_queue_process()
2920 skb_queue_purge(&pq->hold_queue); in xfrm_policy_queue_process()
2929 struct xfrm_policy *pol = xdst->pols[0]; in xdst_queue_output()
2930 struct xfrm_policy_queue *pq = &pol->polq; in xdst_queue_output()
2937 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { in xdst_queue_output()
2939 return -EAGAIN; in xdst_queue_output()
2944 spin_lock_bh(&pq->hold_queue.lock); in xdst_queue_output()
2946 if (!pq->timeout) in xdst_queue_output()
2947 pq->timeout = XFRM_QUEUE_TMO_MIN; in xdst_queue_output()
2949 sched_next = jiffies + pq->timeout; in xdst_queue_output()
2951 if (del_timer(&pq->hold_timer)) { in xdst_queue_output()
2952 if (time_before(pq->hold_timer.expires, sched_next)) in xdst_queue_output()
2953 sched_next = pq->hold_timer.expires; in xdst_queue_output()
2957 __skb_queue_tail(&pq->hold_queue, skb); in xdst_queue_output()
2958 if (!mod_timer(&pq->hold_timer, sched_next)) in xdst_queue_output()
2961 spin_unlock_bh(&pq->hold_queue.lock); in xdst_queue_output()
2982 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || in xfrm_create_dummy_bundle()
2983 net->xfrm.sysctl_larval_drop || in xfrm_create_dummy_bundle()
2987 dst = xflo->dst_orig; in xfrm_create_dummy_bundle()
2988 dst1 = &xdst->u.dst; in xfrm_create_dummy_bundle()
2990 xdst->route = dst; in xfrm_create_dummy_bundle()
2994 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_create_dummy_bundle()
2995 dst1->flags |= DST_XFRM_QUEUE; in xfrm_create_dummy_bundle()
2996 dst1->lastuse = jiffies; in xfrm_create_dummy_bundle()
2998 dst1->input = dst_discard; in xfrm_create_dummy_bundle()
2999 dst1->output = xdst_queue_output; in xfrm_create_dummy_bundle()
3003 xdst->path = dst; in xfrm_create_dummy_bundle()
3007 err = -ENODEV; in xfrm_create_dummy_bundle()
3008 dev = dst->dev; in xfrm_create_dummy_bundle()
3048 xflo->dst_orig); in xfrm_bundle_lookup()
3051 if (err == -EREMOTE) { in xfrm_bundle_lookup()
3056 if (err != -EAGAIN) in xfrm_bundle_lookup()
3075 xdst->num_pols = num_pols; in xfrm_bundle_lookup()
3076 xdst->num_xfrms = num_xfrms; in xfrm_bundle_lookup()
3077 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_bundle_lookup()
3096 return ERR_PTR(-EINVAL); in make_blackhole()
3098 ret = afinfo->blackhole_route(net, dst_orig); in make_blackhole()
3108 * on interfaces with disabled IPsec.
3122 u16 family = dst_orig->ops->family; in xfrm_lookup_with_ifid()
3131 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { in xfrm_lookup_with_ifid()
3153 if (err == -EREMOTE) in xfrm_lookup_with_ifid()
3163 route = xdst->route; in xfrm_lookup_with_ifid()
3174 if (!if_id && ((dst_orig->flags & DST_NOXFRM) || in xfrm_lookup_with_ifid()
3175 !net->xfrm.policy_count[XFRM_POLICY_OUT])) in xfrm_lookup_with_ifid()
3186 num_pols = xdst->num_pols; in xfrm_lookup_with_ifid()
3187 num_xfrms = xdst->num_xfrms; in xfrm_lookup_with_ifid()
3188 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_lookup_with_ifid()
3189 route = xdst->route; in xfrm_lookup_with_ifid()
3192 dst = &xdst->u.dst; in xfrm_lookup_with_ifid()
3200 if (net->xfrm.sysctl_larval_drop) { in xfrm_lookup_with_ifid()
3202 err = -EREMOTE; in xfrm_lookup_with_ifid()
3206 err = -EAGAIN; in xfrm_lookup_with_ifid()
3217 !(pols[0]->flags & XFRM_POLICY_ICMP)) { in xfrm_lookup_with_ifid()
3218 err = -ENOENT; in xfrm_lookup_with_ifid()
3223 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); in xfrm_lookup_with_ifid()
3228 err = -EPERM; in xfrm_lookup_with_ifid()
3240 if (dst && dst->xfrm && in xfrm_lookup_with_ifid()
3241 dst->xfrm->props.mode == XFRM_MODE_TUNNEL) in xfrm_lookup_with_ifid()
3242 dst->flags |= DST_XFRM_TUNNEL; in xfrm_lookup_with_ifid()
3246 if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && in xfrm_lookup_with_ifid()
3247 net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in xfrm_lookup_with_ifid()
3248 err = -EPERM; in xfrm_lookup_with_ifid()
3255 err = -ENOENT; in xfrm_lookup_with_ifid()
3269 * on interfaces with disabled IPsec.
3290 if (PTR_ERR(dst) == -EREMOTE) in xfrm_lookup_route()
3291 return make_blackhole(net, dst_orig->ops->family, dst_orig); in xfrm_lookup_route()
3303 struct sec_path *sp = skb_sec_path(skb); in xfrm_secpath_reject() local
3306 if (!sp || idx < 0 || idx >= sp->len) in xfrm_secpath_reject()
3308 x = sp->xvec[idx]; in xfrm_secpath_reject()
3309 if (!x->type->reject) in xfrm_secpath_reject()
3311 return x->type->reject(x, skb, fl); in xfrm_secpath_reject()
3316 * stupid way. Shame on me. :-) Of course, connected sockets must
3325 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); in xfrm_state_ok()
3326 return x->id.proto == tmpl->id.proto && in xfrm_state_ok()
3327 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && in xfrm_state_ok()
3328 (x->props.reqid == tmpl->reqid || !tmpl->reqid) && in xfrm_state_ok()
3329 x->props.mode == tmpl->mode && in xfrm_state_ok()
3330 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || in xfrm_state_ok()
3331 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && in xfrm_state_ok()
3332 !(x->props.mode != XFRM_MODE_TRANSPORT && in xfrm_state_ok()
3334 (if_id == 0 || if_id == x->if_id); in xfrm_state_ok()
3341 * -1 is returned when no matching template is found.
3342 * Otherwise "-2 - errored_index" is returned.
3345 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start, in xfrm_policy_ok() argument
3350 if (tmpl->optional) { in xfrm_policy_ok()
3351 if (tmpl->mode == XFRM_MODE_TRANSPORT) in xfrm_policy_ok()
3354 start = -1; in xfrm_policy_ok()
3355 for (; idx < sp->len; idx++) { in xfrm_policy_ok()
3356 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id)) in xfrm_policy_ok()
3358 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_policy_ok()
3359 if (idx < sp->verified_cnt) { in xfrm_policy_ok()
3366 if (start == -1) in xfrm_policy_ok()
3367 start = -2-idx; in xfrm_policy_ok()
3378 int ihl = iph->ihl; in decode_session4()
3380 struct flowi4 *fl4 = &fl->u.ip4; in decode_session4()
3383 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session4()
3384 oif = skb_dst(skb)->dev->ifindex; in decode_session4()
3387 fl4->flowi4_mark = skb->mark; in decode_session4()
3388 fl4->flowi4_oif = reverse ? skb->skb_iif : oif; in decode_session4()
3390 fl4->flowi4_proto = iph->protocol; in decode_session4()
3391 fl4->daddr = reverse ? iph->saddr : iph->daddr; in decode_session4()
3392 fl4->saddr = reverse ? iph->daddr : iph->saddr; in decode_session4()
3393 fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK; in decode_session4()
3396 switch (iph->protocol) { in decode_session4()
3402 if (xprth + 4 < skb->data || in decode_session4()
3403 pskb_may_pull(skb, xprth + 4 - skb->data)) { in decode_session4()
3404 __be16 *ports; in decode_session4() local
3407 ports = (__be16 *)xprth; in decode_session4()
3409 fl4->fl4_sport = ports[!!reverse]; in decode_session4()
3410 fl4->fl4_dport = ports[!reverse]; in decode_session4()
3414 if (xprth + 2 < skb->data || in decode_session4()
3415 pskb_may_pull(skb, xprth + 2 - skb->data)) { in decode_session4()
3421 fl4->fl4_icmp_type = icmp[0]; in decode_session4()
3422 fl4->fl4_icmp_code = icmp[1]; in decode_session4()
3426 if (xprth + 12 < skb->data || in decode_session4()
3427 pskb_may_pull(skb, xprth + 12 - skb->data)) { in decode_session4()
3438 fl4->fl4_gre_key = gre_hdr[1]; in decode_session4()
3452 struct flowi6 *fl6 = &fl->u.ip6; in decode_session6()
3458 u16 nhoff = IP6CB(skb)->nhoff; in decode_session6()
3467 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session6()
3468 oif = skb_dst(skb)->dev->ifindex; in decode_session6()
3471 fl6->flowi6_mark = skb->mark; in decode_session6()
3472 fl6->flowi6_oif = reverse ? skb->skb_iif : oif; in decode_session6()
3474 fl6->daddr = reverse ? hdr->saddr : hdr->daddr; in decode_session6()
3475 fl6->saddr = reverse ? hdr->daddr : hdr->saddr; in decode_session6()
3477 while (nh + offset + sizeof(*exthdr) < skb->data || in decode_session6()
3478 pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) { in decode_session6()
3490 nexthdr = exthdr->nexthdr; in decode_session6()
3497 if (!onlyproto && (nh + offset + 4 < skb->data || in decode_session6()
3498 pskb_may_pull(skb, nh + offset + 4 - skb->data))) { in decode_session6()
3499 __be16 *ports; in decode_session6() local
3502 ports = (__be16 *)(nh + offset); in decode_session6()
3503 fl6->fl6_sport = ports[!!reverse]; in decode_session6()
3504 fl6->fl6_dport = ports[!reverse]; in decode_session6()
3506 fl6->flowi6_proto = nexthdr; in decode_session6()
3509 if (!onlyproto && (nh + offset + 2 < skb->data || in decode_session6()
3510 pskb_may_pull(skb, nh + offset + 2 - skb->data))) { in decode_session6()
3515 fl6->fl6_icmp_type = icmp[0]; in decode_session6()
3516 fl6->fl6_icmp_code = icmp[1]; in decode_session6()
3518 fl6->flowi6_proto = nexthdr; in decode_session6()
3522 (nh + offset + 12 < skb->data || in decode_session6()
3523 pskb_may_pull(skb, nh + offset + 12 - skb->data))) { in decode_session6()
3531 if (gre_hdr->flags & GRE_KEY) { in decode_session6()
3532 if (gre_hdr->flags & GRE_CSUM) in decode_session6()
3534 fl6->fl6_gre_key = *gre_key; in decode_session6()
3537 fl6->flowi6_proto = nexthdr; in decode_session6()
3543 if (!onlyproto && (nh + offset + 3 < skb->data || in decode_session6()
3544 pskb_may_pull(skb, nh + offset + 3 - skb->data))) { in decode_session6()
3549 fl6->fl6_mh_type = mh->ip6mh_type; in decode_session6()
3551 fl6->flowi6_proto = nexthdr; in decode_session6()
3555 fl6->flowi6_proto = nexthdr; in decode_session6()
3575 return -EAFNOSUPPORT; in __xfrm_decode_session()
3578 return security_xfrm_decode_session(skb, &fl->flowi_secid); in __xfrm_decode_session()
3582 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp) in secpath_has_nontransport() argument
3584 for (; k < sp->len; k++) { in secpath_has_nontransport()
3585 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { in secpath_has_nontransport()
3597 struct net *net = dev_net(skb->dev); in __xfrm_policy_check()
3605 int xerr_idx = -1; in __xfrm_policy_check()
3607 struct sec_path *sp; in __xfrm_policy_check() local
3616 if (ifcb->decode_session(skb, family, &r)) { in __xfrm_policy_check()
3634 sp = skb_sec_path(skb); in __xfrm_policy_check()
3635 if (sp) { in __xfrm_policy_check()
3638 for (i = sp->len - 1; i >= 0; i--) { in __xfrm_policy_check()
3639 struct xfrm_state *x = sp->xvec[i]; in __xfrm_policy_check()
3640 if (!xfrm_selector_match(&x->sel, &fl, family)) { in __xfrm_policy_check()
3649 if (sk && sk->sk_policy[dir]) { in __xfrm_policy_check()
3666 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in __xfrm_policy_check()
3671 if (sp && secpath_has_nontransport(sp, 0, &xerr_idx)) { in __xfrm_policy_check()
3680 WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); in __xfrm_policy_check()
3685 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in __xfrm_policy_check()
3696 WRITE_ONCE(pols[1]->curlft.use_time, in __xfrm_policy_check()
3703 if (pol->action == XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3711 sp = skb_sec_path(skb); in __xfrm_policy_check()
3712 if (!sp) in __xfrm_policy_check()
3713 sp = &dummy; in __xfrm_policy_check()
3717 pols[pi]->action != XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3721 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { in __xfrm_policy_check()
3725 for (i = 0; i < pols[pi]->xfrm_nr; i++) in __xfrm_policy_check()
3726 tpp[ti++] = &pols[pi]->xfrm_vec[i]; in __xfrm_policy_check()
3744 for (i = xfrm_nr-1, k = 0; i >= 0; i--) { in __xfrm_policy_check()
3745 k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); in __xfrm_policy_check()
3747 if (k < -1) in __xfrm_policy_check()
3748 /* "-2 - errored_index" returned */ in __xfrm_policy_check()
3749 xerr_idx = -(2+k); in __xfrm_policy_check()
3755 if (secpath_has_nontransport(sp, k, &xerr_idx)) { in __xfrm_policy_check()
3761 sp->verified_cnt = k; in __xfrm_policy_check()
3777 struct net *net = dev_net(skb->dev); in __xfrm_route_forward()
3807 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete in xfrm_dst_check()
3809 * get validated by dst_ops->check on every use. We do this in xfrm_dst_check()
3816 * XFRM dst A --> IPv4 dst X in xfrm_dst_check()
3818 * X is the "xdst->route" of A (X is also the "dst->path" of A in xfrm_dst_check()
3828 if (dst->obsolete < 0 && !stale_bundle(dst)) in xfrm_dst_check()
3841 while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { in xfrm_dst_ifdown()
3842 dst->dev = blackhole_netdev; in xfrm_dst_ifdown()
3843 dev_hold(dst->dev); in xfrm_dst_ifdown()
3857 if (dst->obsolete) { in xfrm_negative_advice()
3867 while (nr--) { in xfrm_init_pmtu()
3872 dst = &xdst->u.dst; in xfrm_init_pmtu()
3874 xdst->child_mtu_cached = pmtu; in xfrm_init_pmtu()
3876 pmtu = xfrm_state_mtu(dst->xfrm, pmtu); in xfrm_init_pmtu()
3878 route_mtu_cached = dst_mtu(xdst->route); in xfrm_init_pmtu()
3879 xdst->route_mtu_cached = route_mtu_cached; in xfrm_init_pmtu()
3895 struct dst_entry *dst = &first->u.dst; in xfrm_bundle_ok()
3900 if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) || in xfrm_bundle_ok()
3901 (dst->dev && !netif_running(dst->dev))) in xfrm_bundle_ok()
3904 if (dst->flags & DST_XFRM_QUEUE) in xfrm_bundle_ok()
3911 if (dst->xfrm->km.state != XFRM_STATE_VALID) in xfrm_bundle_ok()
3913 if (xdst->xfrm_genid != dst->xfrm->genid) in xfrm_bundle_ok()
3915 if (xdst->num_pols > 0 && in xfrm_bundle_ok()
3916 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) in xfrm_bundle_ok()
3922 if (xdst->child_mtu_cached != mtu) { in xfrm_bundle_ok()
3924 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3927 if (!dst_check(xdst->route, xdst->route_cookie)) in xfrm_bundle_ok()
3929 mtu = dst_mtu(xdst->route); in xfrm_bundle_ok()
3930 if (xdst->route_mtu_cached != mtu) { in xfrm_bundle_ok()
3932 xdst->route_mtu_cached = mtu; in xfrm_bundle_ok()
3936 } while (dst->xfrm); in xfrm_bundle_ok()
3941 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3942 mtu = xdst->child_mtu_cached; in xfrm_bundle_ok()
3943 while (start_from--) { in xfrm_bundle_ok()
3944 dst = &xdst->u.dst; in xfrm_bundle_ok()
3946 mtu = xfrm_state_mtu(dst->xfrm, mtu); in xfrm_bundle_ok()
3947 if (mtu > xdst->route_mtu_cached) in xfrm_bundle_ok()
3948 mtu = xdst->route_mtu_cached; in xfrm_bundle_ok()
3953 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3954 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3975 while (dst->xfrm) { in xfrm_get_dst_nexthop()
3976 const struct xfrm_state *xfrm = dst->xfrm; in xfrm_get_dst_nexthop()
3980 if (xfrm->props.mode == XFRM_MODE_TRANSPORT) in xfrm_get_dst_nexthop()
3982 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) in xfrm_get_dst_nexthop()
3983 daddr = xfrm->coaddr; in xfrm_get_dst_nexthop()
3984 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) in xfrm_get_dst_nexthop()
3985 daddr = &xfrm->id.daddr; in xfrm_get_dst_nexthop()
3998 return path->ops->neigh_lookup(path, skb, daddr); in xfrm_neigh_lookup()
4006 path->ops->confirm_neigh(path, daddr); in xfrm_confirm_neigh()
4014 return -EAFNOSUPPORT; in xfrm_policy_register_afinfo()
4018 err = -EEXIST; in xfrm_policy_register_afinfo()
4020 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_register_afinfo()
4021 if (likely(dst_ops->kmem_cachep == NULL)) in xfrm_policy_register_afinfo()
4022 dst_ops->kmem_cachep = xfrm_dst_cache; in xfrm_policy_register_afinfo()
4023 if (likely(dst_ops->check == NULL)) in xfrm_policy_register_afinfo()
4024 dst_ops->check = xfrm_dst_check; in xfrm_policy_register_afinfo()
4025 if (likely(dst_ops->default_advmss == NULL)) in xfrm_policy_register_afinfo()
4026 dst_ops->default_advmss = xfrm_default_advmss; in xfrm_policy_register_afinfo()
4027 if (likely(dst_ops->mtu == NULL)) in xfrm_policy_register_afinfo()
4028 dst_ops->mtu = xfrm_mtu; in xfrm_policy_register_afinfo()
4029 if (likely(dst_ops->negative_advice == NULL)) in xfrm_policy_register_afinfo()
4030 dst_ops->negative_advice = xfrm_negative_advice; in xfrm_policy_register_afinfo()
4031 if (likely(dst_ops->link_failure == NULL)) in xfrm_policy_register_afinfo()
4032 dst_ops->link_failure = xfrm_link_failure; in xfrm_policy_register_afinfo()
4033 if (likely(dst_ops->neigh_lookup == NULL)) in xfrm_policy_register_afinfo()
4034 dst_ops->neigh_lookup = xfrm_neigh_lookup; in xfrm_policy_register_afinfo()
4035 if (likely(!dst_ops->confirm_neigh)) in xfrm_policy_register_afinfo()
4036 dst_ops->confirm_neigh = xfrm_confirm_neigh; in xfrm_policy_register_afinfo()
4047 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_unregister_afinfo()
4059 dst_ops->kmem_cachep = NULL; in xfrm_policy_unregister_afinfo()
4060 dst_ops->check = NULL; in xfrm_policy_unregister_afinfo()
4061 dst_ops->negative_advice = NULL; in xfrm_policy_unregister_afinfo()
4062 dst_ops->link_failure = NULL; in xfrm_policy_unregister_afinfo()
4085 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); in xfrm_statistics_init()
4086 if (!net->mib.xfrm_statistics) in xfrm_statistics_init()
4087 return -ENOMEM; in xfrm_statistics_init()
4090 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_init()
4097 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_fini()
4125 hmask = 8 - 1; in xfrm_policy_init()
4128 net->xfrm.policy_byidx = xfrm_hash_alloc(sz); in xfrm_policy_init()
4129 if (!net->xfrm.policy_byidx) in xfrm_policy_init()
4131 net->xfrm.policy_idx_hmask = hmask; in xfrm_policy_init()
4136 net->xfrm.policy_count[dir] = 0; in xfrm_policy_init()
4137 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; in xfrm_policy_init()
4138 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); in xfrm_policy_init()
4140 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4141 htab->table = xfrm_hash_alloc(sz); in xfrm_policy_init()
4142 if (!htab->table) in xfrm_policy_init()
4144 htab->hmask = hmask; in xfrm_policy_init()
4145 htab->dbits4 = 32; in xfrm_policy_init()
4146 htab->sbits4 = 32; in xfrm_policy_init()
4147 htab->dbits6 = 128; in xfrm_policy_init()
4148 htab->sbits6 = 128; in xfrm_policy_init()
4150 net->xfrm.policy_hthresh.lbits4 = 32; in xfrm_policy_init()
4151 net->xfrm.policy_hthresh.rbits4 = 32; in xfrm_policy_init()
4152 net->xfrm.policy_hthresh.lbits6 = 128; in xfrm_policy_init()
4153 net->xfrm.policy_hthresh.rbits6 = 128; in xfrm_policy_init()
4155 seqlock_init(&net->xfrm.policy_hthresh.lock); in xfrm_policy_init()
4157 INIT_LIST_HEAD(&net->xfrm.policy_all); in xfrm_policy_init()
4158 INIT_LIST_HEAD(&net->xfrm.inexact_bins); in xfrm_policy_init()
4159 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); in xfrm_policy_init()
4160 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); in xfrm_policy_init()
4164 for (dir--; dir >= 0; dir--) { in xfrm_policy_init()
4167 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4168 xfrm_hash_free(htab->table, sz); in xfrm_policy_init()
4170 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_init()
4172 return -ENOMEM; in xfrm_policy_init()
4181 flush_work(&net->xfrm.policy_hash_work); in xfrm_policy_fini()
4187 WARN_ON(!list_empty(&net->xfrm.policy_all)); in xfrm_policy_fini()
4192 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); in xfrm_policy_fini()
4194 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_fini()
4195 sz = (htab->hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4196 WARN_ON(!hlist_empty(htab->table)); in xfrm_policy_fini()
4197 xfrm_hash_free(htab->table, sz); in xfrm_policy_fini()
4200 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4201 WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); in xfrm_policy_fini()
4202 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_fini()
4204 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4205 list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) in xfrm_policy_fini()
4207 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4214 /* Initialize the per-net locks here */ in xfrm_net_init()
4215 spin_lock_init(&net->xfrm.xfrm_state_lock); in xfrm_net_init()
4216 spin_lock_init(&net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4217 seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4218 mutex_init(&net->xfrm.xfrm_cfg_mutex); in xfrm_net_init()
4219 net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4220 net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4221 net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4276 struct xfrm_sec_ctx *ctx = xp->security; in xfrm_audit_common_policyinfo()
4277 struct xfrm_selector *sel = &xp->selector; in xfrm_audit_common_policyinfo()
4281 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); in xfrm_audit_common_policyinfo()
4283 switch (sel->family) { in xfrm_audit_common_policyinfo()
4285 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); in xfrm_audit_common_policyinfo()
4286 if (sel->prefixlen_s != 32) in xfrm_audit_common_policyinfo()
4288 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4289 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); in xfrm_audit_common_policyinfo()
4290 if (sel->prefixlen_d != 32) in xfrm_audit_common_policyinfo()
4292 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4295 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); in xfrm_audit_common_policyinfo()
4296 if (sel->prefixlen_s != 128) in xfrm_audit_common_policyinfo()
4298 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4299 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); in xfrm_audit_common_policyinfo()
4300 if (sel->prefixlen_d != 128) in xfrm_audit_common_policyinfo()
4302 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4311 audit_buf = xfrm_audit_start("SPD-add"); in xfrm_audit_policy_add()
4326 audit_buf = xfrm_audit_start("SPD-delete"); in xfrm_audit_policy_delete()
4341 if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { in xfrm_migrate_selector_match()
4342 if (sel_tgt->family == sel_cmp->family && in xfrm_migrate_selector_match()
4343 xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr, in xfrm_migrate_selector_match()
4344 sel_cmp->family) && in xfrm_migrate_selector_match()
4345 xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr, in xfrm_migrate_selector_match()
4346 sel_cmp->family) && in xfrm_migrate_selector_match()
4347 sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && in xfrm_migrate_selector_match()
4348 sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { in xfrm_migrate_selector_match()
4366 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4367 chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir); in xfrm_migrate_policy_find()
4369 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4370 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4371 pol->type == type) { in xfrm_migrate_policy_find()
4373 priority = ret->priority; in xfrm_migrate_policy_find()
4377 chain = &net->xfrm.policy_inexact[dir]; in xfrm_migrate_policy_find()
4379 if ((pol->priority >= priority) && ret) in xfrm_migrate_policy_find()
4382 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4383 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4384 pol->type == type) { in xfrm_migrate_policy_find()
4392 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4401 if (t->mode == m->mode && t->id.proto == m->proto && in migrate_tmpl_match()
4402 (m->reqid == 0 || t->reqid == m->reqid)) { in migrate_tmpl_match()
4403 switch (t->mode) { in migrate_tmpl_match()
4406 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, in migrate_tmpl_match()
4407 m->old_family) && in migrate_tmpl_match()
4408 xfrm_addr_equal(&t->saddr, &m->old_saddr, in migrate_tmpl_match()
4409 m->old_family)) { in migrate_tmpl_match()
4434 write_lock_bh(&pol->lock); in xfrm_policy_migrate()
4435 if (unlikely(pol->walk.dead)) { in xfrm_policy_migrate()
4438 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4439 return -ENOENT; in xfrm_policy_migrate()
4442 for (i = 0; i < pol->xfrm_nr; i++) { in xfrm_policy_migrate()
4444 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) in xfrm_policy_migrate()
4447 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && in xfrm_policy_migrate()
4448 pol->xfrm_vec[i].mode != XFRM_MODE_BEET) in xfrm_policy_migrate()
4451 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, in xfrm_policy_migrate()
4452 sizeof(pol->xfrm_vec[i].id.daddr)); in xfrm_policy_migrate()
4453 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, in xfrm_policy_migrate()
4454 sizeof(pol->xfrm_vec[i].saddr)); in xfrm_policy_migrate()
4455 pol->xfrm_vec[i].encap_family = mp->new_family; in xfrm_policy_migrate()
4457 atomic_inc(&pol->genid); in xfrm_policy_migrate()
4461 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4464 return -ENODATA; in xfrm_policy_migrate()
4476 return -EINVAL; in xfrm_migrate_check()
4483 return -EINVAL; in xfrm_migrate_check()
4497 return -EINVAL; in xfrm_migrate_check()
4518 /* Stage 0 - sanity checks */ in xfrm_migrate()
4525 err = -EINVAL; in xfrm_migrate()
4529 /* Stage 1 - find policy */ in xfrm_migrate()
4533 err = -ENOENT; in xfrm_migrate()
4537 /* Stage 2 - find and update state(s) */ in xfrm_migrate()
4547 err = -ENODATA; in xfrm_migrate()
4553 /* Stage 3 - update policy */ in xfrm_migrate()
4558 /* Stage 4 - delete old state(s) */ in xfrm_migrate()
4564 /* Stage 5 - announce */ in xfrm_migrate()