Lines Matching +full:key +full:- +full:release
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright 2020-2022 Xilinx Inc.
48 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
50 if (dev == efx->net_dev) in efx_tc_flower_lookup_efv()
53 if (dev->netdev_ops != &efx_ef100_rep_netdev_ops) in efx_tc_flower_lookup_efv()
54 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
59 if (efv->parent != efx) in efx_tc_flower_lookup_efv()
60 return ERR_PTR(-EOPNOTSUPP); in efx_tc_flower_lookup_efv()
64 /* Convert a driver-internal vport ID into an internal device (PF or VF) */
74 efx_mae_mport_mport(efx, efv->mport, &mport); in efx_tc_flower_internal_mport()
78 /* Convert a driver-internal vport ID into an external device (wire or VF) */
88 efx_mae_mport_mport(efx, efv->mport, &mport); in efx_tc_flower_external_mport()
131 return ERR_PTR(-ENOMEM); in efx_tc_flower_get_mac()
132 memcpy(ped->h_addr, h_addr, ETH_ALEN); in efx_tc_flower_get_mac()
133 old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht, in efx_tc_flower_get_mac()
134 &ped->linkage, in efx_tc_flower_get_mac()
141 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_flower_get_mac()
142 return ERR_PTR(-EAGAIN); in efx_tc_flower_get_mac()
154 refcount_set(&ped->ref, 1); in efx_tc_flower_get_mac()
157 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, in efx_tc_flower_get_mac()
166 if (!refcount_dec_and_test(&ped->ref)) in efx_tc_flower_put_mac()
168 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage, in efx_tc_flower_put_mac()
182 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_free_action_set()
186 list_del(&act->list); in efx_tc_free_action_set()
188 if (act->count) { in efx_tc_free_action_set()
189 spin_lock_bh(&act->count->cnt->lock); in efx_tc_free_action_set()
190 if (!list_empty(&act->count_user)) in efx_tc_free_action_set()
191 list_del(&act->count_user); in efx_tc_free_action_set()
192 spin_unlock_bh(&act->count->cnt->lock); in efx_tc_free_action_set()
193 efx_tc_flower_put_counter_index(efx, act->count); in efx_tc_free_action_set()
195 if (act->encap_md) { in efx_tc_free_action_set()
196 list_del(&act->encap_user); in efx_tc_free_action_set()
197 efx_tc_flower_release_encap_md(efx, act->encap_md); in efx_tc_free_action_set()
199 if (act->src_mac) in efx_tc_free_action_set()
200 efx_tc_flower_put_mac(efx, act->src_mac); in efx_tc_free_action_set()
201 if (act->dst_mac) in efx_tc_free_action_set()
202 efx_tc_flower_put_mac(efx, act->dst_mac); in efx_tc_free_action_set()
219 list_for_each_entry_safe(act, next, &acts->list, list) in efx_tc_free_action_set_list()
230 match->value._field = fm.key->_tcfield; \
231 match->mask._field = fm.mask->_tcfield; \
243 struct flow_dissector *dissector = rule->match.dissector; in efx_tc_flower_parse_match()
246 /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set in efx_tc_flower_parse_match()
247 * even on IPv4 filters; so rather than relying on dissector->used_keys in efx_tc_flower_parse_match()
248 * we check the addr_type in the CONTROL key. If we don't find it (or in efx_tc_flower_parse_match()
256 if (IS_ALL_ONES(fm.mask->addr_type)) in efx_tc_flower_parse_match()
257 switch (fm.key->addr_type) { in efx_tc_flower_parse_match()
268 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) { in efx_tc_flower_parse_match()
269 match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT; in efx_tc_flower_parse_match()
270 match->mask.ip_frag = true; in efx_tc_flower_parse_match()
272 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) { in efx_tc_flower_parse_match()
273 match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG; in efx_tc_flower_parse_match()
274 match->mask.ip_firstfrag = true; in efx_tc_flower_parse_match()
276 if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) { in efx_tc_flower_parse_match()
278 fm.mask->flags); in efx_tc_flower_parse_match()
279 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
282 if (dissector->used_keys & in efx_tc_flower_parse_match()
301 dissector->used_keys); in efx_tc_flower_parse_match()
302 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
307 if (!IS_ALL_ONES(match->mask.eth_proto) || in efx_tc_flower_parse_match()
308 !(match->value.eth_proto == htons(ETH_P_IP) || in efx_tc_flower_parse_match()
309 match->value.eth_proto == htons(ETH_P_IPV6))) in efx_tc_flower_parse_match()
310 if (dissector->used_keys & in efx_tc_flower_parse_match()
318 dissector->used_keys); in efx_tc_flower_parse_match()
319 return -EINVAL; in efx_tc_flower_parse_match()
326 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { in efx_tc_flower_parse_match()
327 match->value.vlan_proto[0] = fm.key->vlan_tpid; in efx_tc_flower_parse_match()
328 match->mask.vlan_proto[0] = fm.mask->vlan_tpid; in efx_tc_flower_parse_match()
329 match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 | in efx_tc_flower_parse_match()
330 fm.key->vlan_id); in efx_tc_flower_parse_match()
331 match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 | in efx_tc_flower_parse_match()
332 fm.mask->vlan_id); in efx_tc_flower_parse_match()
340 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) { in efx_tc_flower_parse_match()
341 match->value.vlan_proto[1] = fm.key->vlan_tpid; in efx_tc_flower_parse_match()
342 match->mask.vlan_proto[1] = fm.mask->vlan_tpid; in efx_tc_flower_parse_match()
343 match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 | in efx_tc_flower_parse_match()
344 fm.key->vlan_id); in efx_tc_flower_parse_match()
345 match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 | in efx_tc_flower_parse_match()
346 fm.mask->vlan_id); in efx_tc_flower_parse_match()
354 ether_addr_copy(match->value.eth_saddr, fm.key->src); in efx_tc_flower_parse_match()
355 ether_addr_copy(match->value.eth_daddr, fm.key->dst); in efx_tc_flower_parse_match()
356 ether_addr_copy(match->mask.eth_saddr, fm.mask->src); in efx_tc_flower_parse_match()
357 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst); in efx_tc_flower_parse_match()
362 if ((match->value.ip_proto != IPPROTO_UDP && in efx_tc_flower_parse_match()
363 match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto)) in efx_tc_flower_parse_match()
364 if (dissector->used_keys & in efx_tc_flower_parse_match()
369 dissector->used_keys); in efx_tc_flower_parse_match()
370 return -EINVAL; in efx_tc_flower_parse_match()
391 if (fm.mask->flags) { in efx_tc_flower_parse_match()
393 fm.mask->flags); in efx_tc_flower_parse_match()
394 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
396 if (!IS_ALL_ONES(fm.mask->addr_type)) { in efx_tc_flower_parse_match()
397 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)", in efx_tc_flower_parse_match()
398 fm.mask->addr_type, in efx_tc_flower_parse_match()
399 fm.key->addr_type); in efx_tc_flower_parse_match()
400 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
402 switch (fm.key->addr_type) { in efx_tc_flower_parse_match()
420 fm.key->addr_type); in efx_tc_flower_parse_match()
421 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
428 } else if (dissector->used_keys & in efx_tc_flower_parse_match()
436 dissector->used_keys); in efx_tc_flower_parse_match()
437 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
443 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); in efx_tc_flower_parse_match()
444 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED); in efx_tc_flower_parse_match()
445 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); in efx_tc_flower_parse_match()
446 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED); in efx_tc_flower_parse_match()
447 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED | in efx_tc_flower_parse_match()
451 fm.mask->ct_state); in efx_tc_flower_parse_match()
452 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
454 match->value.ct_mark = fm.key->ct_mark; in efx_tc_flower_parse_match()
455 match->mask.ct_mark = fm.mask->ct_mark; in efx_tc_flower_parse_match()
456 match->value.ct_zone = fm.key->ct_zone; in efx_tc_flower_parse_match()
457 match->mask.ct_zone = fm.mask->ct_zone; in efx_tc_flower_parse_match()
459 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) { in efx_tc_flower_parse_match()
461 return -EOPNOTSUPP; in efx_tc_flower_parse_match()
473 if (!refcount_dec_and_test(&encap->ref)) in efx_tc_flower_release_encap_match()
476 if (encap->type == EFX_TC_EM_DIRECT) { in efx_tc_flower_release_encap_match()
482 netif_err(efx, drv, efx->net_dev, in efx_tc_flower_release_encap_match()
483 "Failed to release encap match %#x, rc %d\n", in efx_tc_flower_release_encap_match()
484 encap->fw_id, rc); in efx_tc_flower_release_encap_match()
486 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, in efx_tc_flower_release_encap_match()
488 if (encap->pseudo) in efx_tc_flower_release_encap_match()
489 efx_tc_flower_release_encap_match(efx, encap->pseudo); in efx_tc_flower_release_encap_match()
505 /* We require that the socket-defining fields (IP addrs and UDP dest in efx_tc_flower_record_encap_match()
506 * port) are present and exact-match. Other fields may only be used in efx_tc_flower_record_encap_match()
507 * if the field-set (and any masks) are the same for all encap in efx_tc_flower_record_encap_match()
511 if (match->mask.enc_dst_ip | match->mask.enc_src_ip) { in efx_tc_flower_record_encap_match()
512 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) { in efx_tc_flower_record_encap_match()
515 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
517 if (!IS_ALL_ONES(match->mask.enc_src_ip)) { in efx_tc_flower_record_encap_match()
520 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
523 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) || in efx_tc_flower_record_encap_match()
524 !ipv6_addr_any(&match->mask.enc_src_ip6)) { in efx_tc_flower_record_encap_match()
527 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
531 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) { in efx_tc_flower_record_encap_match()
534 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
536 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) { in efx_tc_flower_record_encap_match()
539 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
543 if (!IS_ALL_ONES(match->mask.enc_dport)) { in efx_tc_flower_record_encap_match()
545 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
547 if (match->mask.enc_sport || match->mask.enc_ip_tos) { in efx_tc_flower_record_encap_match()
552 return -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
560 match->mask.enc_ip_tos, in efx_tc_flower_record_encap_match()
561 match->mask.enc_sport, in efx_tc_flower_record_encap_match()
567 if (match->mask.enc_ip_ttl) { in efx_tc_flower_record_encap_match()
569 rc = -EOPNOTSUPP; in efx_tc_flower_record_encap_match()
573 rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos, in efx_tc_flower_record_encap_match()
574 match->mask.enc_sport, extack); in efx_tc_flower_record_encap_match()
580 rc = -ENOMEM; in efx_tc_flower_record_encap_match()
583 encap->src_ip = match->value.enc_src_ip; in efx_tc_flower_record_encap_match()
584 encap->dst_ip = match->value.enc_dst_ip; in efx_tc_flower_record_encap_match()
586 encap->src_ip6 = match->value.enc_src_ip6; in efx_tc_flower_record_encap_match()
587 encap->dst_ip6 = match->value.enc_dst_ip6; in efx_tc_flower_record_encap_match()
589 encap->udp_dport = match->value.enc_dport; in efx_tc_flower_record_encap_match()
590 encap->tun_type = type; in efx_tc_flower_record_encap_match()
591 encap->ip_tos = match->value.enc_ip_tos; in efx_tc_flower_record_encap_match()
592 encap->ip_tos_mask = match->mask.enc_ip_tos; in efx_tc_flower_record_encap_match()
593 encap->child_ip_tos_mask = child_ip_tos_mask; in efx_tc_flower_record_encap_match()
594 encap->udp_sport = match->value.enc_sport; in efx_tc_flower_record_encap_match()
595 encap->udp_sport_mask = match->mask.enc_sport; in efx_tc_flower_record_encap_match()
596 encap->child_udp_sport_mask = child_udp_sport_mask; in efx_tc_flower_record_encap_match()
597 encap->type = em_type; in efx_tc_flower_record_encap_match()
598 encap->pseudo = pseudo; in efx_tc_flower_record_encap_match()
599 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht, in efx_tc_flower_record_encap_match()
600 &encap->linkage, in efx_tc_flower_record_encap_match()
610 switch (old->type) { in efx_tc_flower_record_encap_match()
618 return -EEXIST; in efx_tc_flower_record_encap_match()
620 /* old EM is protecting a ToS- or src port-qualified in efx_tc_flower_record_encap_match()
628 return -EEXIST; in efx_tc_flower_record_encap_match()
630 if (child_ip_tos_mask != old->child_ip_tos_mask) { in efx_tc_flower_record_encap_match()
634 old->child_ip_tos_mask); in efx_tc_flower_record_encap_match()
635 return -EEXIST; in efx_tc_flower_record_encap_match()
637 if (child_udp_sport_mask != old->child_udp_sport_mask) { in efx_tc_flower_record_encap_match()
641 old->child_udp_sport_mask); in efx_tc_flower_record_encap_match()
642 return -EEXIST; in efx_tc_flower_record_encap_match()
645 default: /* Unrecognised pseudo-type. Just say no */ in efx_tc_flower_record_encap_match()
649 old->type); in efx_tc_flower_record_encap_match()
650 return -EEXIST; in efx_tc_flower_record_encap_match()
653 if (old->tun_type != type) { in efx_tc_flower_record_encap_match()
656 old->tun_type, type); in efx_tc_flower_record_encap_match()
657 return -EEXIST; in efx_tc_flower_record_encap_match()
659 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_flower_record_encap_match()
660 return -EAGAIN; in efx_tc_flower_record_encap_match()
671 refcount_set(&encap->ref, 1); in efx_tc_flower_record_encap_match()
673 match->encap = encap; in efx_tc_flower_record_encap_match()
676 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage, in efx_tc_flower_record_encap_match()
694 return ERR_PTR(-ENOMEM); in efx_tc_get_recirc_id()
695 rid->chain_index = chain_index; in efx_tc_get_recirc_id()
696 /* We don't take a reference here, because it's implied - if there's in efx_tc_get_recirc_id()
700 rid->net_dev = net_dev; in efx_tc_get_recirc_id()
701 old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht, in efx_tc_get_recirc_id()
702 &rid->linkage, in efx_tc_get_recirc_id()
709 if (!refcount_inc_not_zero(&old->ref)) in efx_tc_get_recirc_id()
710 return ERR_PTR(-EAGAIN); in efx_tc_get_recirc_id()
714 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER); in efx_tc_get_recirc_id()
716 rhashtable_remove_fast(&efx->tc->recirc_ht, in efx_tc_get_recirc_id()
717 &rid->linkage, in efx_tc_get_recirc_id()
722 rid->fw_id = rc; in efx_tc_get_recirc_id()
723 refcount_set(&rid->ref, 1); in efx_tc_get_recirc_id()
730 if (!refcount_dec_and_test(&rid->ref)) in efx_tc_put_recirc_id()
732 rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage, in efx_tc_put_recirc_id()
734 ida_free(&efx->tc->recirc_ida, rid->fw_id); in efx_tc_put_recirc_id()
740 efx_mae_delete_rule(efx, rule->fw_id); in efx_tc_delete_rule()
742 /* Release entries in subsidiary tables */ in efx_tc_delete_rule()
743 efx_tc_free_action_set_list(efx, &rule->acts, true); in efx_tc_delete_rule()
744 if (rule->match.rid) in efx_tc_delete_rule()
745 efx_tc_put_recirc_id(efx, rule->match.rid); in efx_tc_delete_rule()
746 if (rule->match.encap) in efx_tc_delete_rule()
747 efx_tc_flower_release_encap_match(efx, rule->match.encap); in efx_tc_delete_rule()
748 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_tc_delete_rule()
766 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
783 if (act->decap) in efx_tc_flower_action_order_ok()
788 if (act->dst_mac || act->src_mac) in efx_tc_flower_action_order_ok()
792 if (act->do_ttl_dec) in efx_tc_flower_action_order_ok()
796 if (act->vlan_pop >= 2) in efx_tc_flower_action_order_ok()
802 if (act->vlan_push) in efx_tc_flower_action_order_ok()
806 if (act->vlan_push >= 2) in efx_tc_flower_action_order_ok()
810 if (act->count) in efx_tc_flower_action_order_ok()
815 if (act->encap_md) in efx_tc_flower_action_order_ok()
819 return !act->deliver; in efx_tc_flower_action_order_ok()
821 if (act->encap_md) in efx_tc_flower_action_order_ok()
823 return !act->do_ttl_dec; in efx_tc_flower_action_order_ok()
835 * consequently we support conntrack through the notion of a "left-hand side
837 * "goto chain N", and corresponds to one or more "right-hand side rules" in
854 flow_action_for_each(i, fa, &fr->action) { in efx_tc_rule_is_lhs_rule()
855 switch (fa->id) { in efx_tc_rule_is_lhs_rule()
859 /* If rule is -trk, or doesn't mention trk at all, then in efx_tc_rule_is_lhs_rule()
865 if (!match->mask.ct_state_trk || !match->value.ct_state_trk) in efx_tc_rule_is_lhs_rule()
882 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_handle_lhs_actions()
883 struct efx_tc_lhs_action *act = &rule->lhs_act; in efx_tc_flower_handle_lhs_actions()
888 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_handle_lhs_actions()
893 /* more actions after a non-pipe action */ in efx_tc_flower_handle_lhs_actions()
894 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); in efx_tc_flower_handle_lhs_actions()
895 return -EINVAL; in efx_tc_flower_handle_lhs_actions()
897 switch (fa->id) { in efx_tc_flower_handle_lhs_actions()
899 if (!fa->chain_index) { in efx_tc_flower_handle_lhs_actions()
901 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
903 rid = efx_tc_get_recirc_id(efx, fa->chain_index, in efx_tc_flower_handle_lhs_actions()
909 act->rid = rid; in efx_tc_flower_handle_lhs_actions()
910 if (fa->hw_stats) { in efx_tc_flower_handle_lhs_actions()
913 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_handle_lhs_actions()
916 fa->hw_stats); in efx_tc_flower_handle_lhs_actions()
917 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
919 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie, in efx_tc_flower_handle_lhs_actions()
925 WARN_ON(act->count); /* can't happen */ in efx_tc_flower_handle_lhs_actions()
926 act->count = cnt; in efx_tc_flower_handle_lhs_actions()
931 if (act->zone) { in efx_tc_flower_handle_lhs_actions()
933 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
935 if (fa->ct.action & (TCA_CT_ACT_COMMIT | in efx_tc_flower_handle_lhs_actions()
938 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
940 if (fa->ct.action & TCA_CT_ACT_CLEAR) { in efx_tc_flower_handle_lhs_actions()
942 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
944 if (fa->ct.action & (TCA_CT_ACT_NAT | in efx_tc_flower_handle_lhs_actions()
947 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet"); in efx_tc_flower_handle_lhs_actions()
948 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
950 if (fa->ct.action) { in efx_tc_flower_handle_lhs_actions()
952 fa->ct.action); in efx_tc_flower_handle_lhs_actions()
953 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
955 ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone, in efx_tc_flower_handle_lhs_actions()
956 fa->ct.flow_table); in efx_tc_flower_handle_lhs_actions()
961 act->zone = ct_zone; in efx_tc_flower_handle_lhs_actions()
965 fa->id); in efx_tc_flower_handle_lhs_actions()
966 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
972 return -EOPNOTSUPP; in efx_tc_flower_handle_lhs_actions()
980 if (act->rid) in efx_tc_flower_release_lhs_actions()
981 efx_tc_put_recirc_id(efx, act->rid); in efx_tc_flower_release_lhs_actions()
982 if (act->zone) in efx_tc_flower_release_lhs_actions()
983 efx_tc_ct_unregister_zone(efx, act->zone); in efx_tc_flower_release_lhs_actions()
984 if (act->count) in efx_tc_flower_release_lhs_actions()
985 efx_tc_flower_put_counter_index(efx, act->count); in efx_tc_flower_release_lhs_actions()
989 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
998 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1004 u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1005 u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1006 u8 src_mac_16:1; /* eth->h_source[0:1] */
1007 u8 src_mac_32:1; /* eth->h_source[2:5] */
1012 /** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1030 if (mung->dst_mac_32 && mung->dst_mac_16) { in efx_tc_complete_mac_mangle()
1031 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack); in efx_tc_complete_mac_mangle()
1036 if (act->dst_mac) in efx_tc_complete_mac_mangle()
1037 efx_tc_flower_put_mac(efx, act->dst_mac); in efx_tc_complete_mac_mangle()
1039 act->dst_mac = ped; in efx_tc_complete_mac_mangle()
1042 mung->dst_mac_32 = 0; in efx_tc_complete_mac_mangle()
1043 mung->dst_mac_16 = 0; in efx_tc_complete_mac_mangle()
1045 if (mung->src_mac_16 && mung->src_mac_32) { in efx_tc_complete_mac_mangle()
1046 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack); in efx_tc_complete_mac_mangle()
1051 if (act->src_mac) in efx_tc_complete_mac_mangle()
1052 efx_tc_flower_put_mac(efx, act->src_mac); in efx_tc_complete_mac_mangle()
1054 act->src_mac = ped; in efx_tc_complete_mac_mangle()
1057 mung->src_mac_32 = 0; in efx_tc_complete_mac_mangle()
1058 mung->src_mac_16 = 0; in efx_tc_complete_mac_mangle()
1067 switch (fa->mangle.htype) { in efx_tc_pedit_add()
1069 switch (fa->mangle.offset) { in efx_tc_pedit_add()
1072 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) in efx_tc_pedit_add()
1078 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX) in efx_tc_pedit_add()
1085 return -EOPNOTSUPP; in efx_tc_pedit_add()
1087 act->do_ttl_dec = 1; in efx_tc_pedit_add()
1094 switch (fa->mangle.offset) { in efx_tc_pedit_add()
1097 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) in efx_tc_pedit_add()
1103 if ((fa->mangle.val >> 24) != U8_MAX) in efx_tc_pedit_add()
1110 return -EOPNOTSUPP; in efx_tc_pedit_add()
1112 act->do_ttl_dec = 1; in efx_tc_pedit_add()
1124 fa->mangle.htype, fa->mangle.offset, in efx_tc_pedit_add()
1125 fa->mangle.val, fa->mangle.mask); in efx_tc_pedit_add()
1126 return -EOPNOTSUPP; in efx_tc_pedit_add()
1130 * efx_tc_mangle() - handle a single 32-bit (or less) pedit
1154 switch (fa->mangle.htype) { in efx_tc_mangle()
1161 return -EOPNOTSUPP; in efx_tc_mangle()
1163 switch (fa->mangle.offset) { in efx_tc_mangle()
1165 if (fa->mangle.mask) { in efx_tc_mangle()
1168 fa->mangle.mask); in efx_tc_mangle()
1169 return -EOPNOTSUPP; in efx_tc_mangle()
1171 /* Ethernet address is little-endian */ in efx_tc_mangle()
1172 mac32 = cpu_to_le32(fa->mangle.val); in efx_tc_mangle()
1173 memcpy(mung->dst_mac, &mac32, sizeof(mac32)); in efx_tc_mangle()
1174 mung->dst_mac_32 = 1; in efx_tc_mangle()
1177 if (fa->mangle.mask == 0xffff) { in efx_tc_mangle()
1178 mac16 = cpu_to_le16(fa->mangle.val >> 16); in efx_tc_mangle()
1179 memcpy(mung->src_mac, &mac16, sizeof(mac16)); in efx_tc_mangle()
1180 mung->src_mac_16 = 1; in efx_tc_mangle()
1181 } else if (fa->mangle.mask == 0xffff0000) { in efx_tc_mangle()
1182 mac16 = cpu_to_le16((u16)fa->mangle.val); in efx_tc_mangle()
1183 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16)); in efx_tc_mangle()
1184 mung->dst_mac_16 = 1; in efx_tc_mangle()
1188 fa->mangle.mask); in efx_tc_mangle()
1189 return -EOPNOTSUPP; in efx_tc_mangle()
1193 if (fa->mangle.mask) { in efx_tc_mangle()
1196 fa->mangle.mask); in efx_tc_mangle()
1197 return -EOPNOTSUPP; in efx_tc_mangle()
1199 mac32 = cpu_to_le32(fa->mangle.val); in efx_tc_mangle()
1200 memcpy(mung->src_mac + 2, &mac32, sizeof(mac32)); in efx_tc_mangle()
1201 mung->src_mac_32 = 1; in efx_tc_mangle()
1205 fa->mangle.offset, fa->mangle.val, fa->mangle.mask); in efx_tc_mangle()
1206 return -EOPNOTSUPP; in efx_tc_mangle()
1210 switch (fa->mangle.offset) { in efx_tc_mangle()
1218 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) { in efx_tc_mangle()
1221 fa->mangle.mask); in efx_tc_mangle()
1222 return -EOPNOTSUPP; in efx_tc_mangle()
1228 if (match->mask.ip_ttl != U8_MAX) { in efx_tc_mangle()
1231 match->mask.ip_ttl); in efx_tc_mangle()
1232 return -EOPNOTSUPP; in efx_tc_mangle()
1238 if (match->value.ip_ttl == 0) { in efx_tc_mangle()
1241 return -EOPNOTSUPP; in efx_tc_mangle()
1249 return -EOPNOTSUPP; in efx_tc_mangle()
1253 tr_ttl = match->value.ip_ttl - 1; in efx_tc_mangle()
1254 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) { in efx_tc_mangle()
1255 act->do_ttl_dec = 1; in efx_tc_mangle()
1263 fa->mangle.offset); in efx_tc_mangle()
1264 return -EOPNOTSUPP; in efx_tc_mangle()
1268 switch (fa->mangle.offset) { in efx_tc_mangle()
1276 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) { in efx_tc_mangle()
1279 fa->mangle.mask); in efx_tc_mangle()
1281 return -EOPNOTSUPP; in efx_tc_mangle()
1287 if (match->mask.ip_ttl != U8_MAX) { in efx_tc_mangle()
1290 match->mask.ip_ttl); in efx_tc_mangle()
1291 return -EOPNOTSUPP; in efx_tc_mangle()
1297 if (match->value.ip_ttl == 0) { in efx_tc_mangle()
1300 return -EOPNOTSUPP; in efx_tc_mangle()
1308 return -EOPNOTSUPP; in efx_tc_mangle()
1312 tr_ttl = match->value.ip_ttl - 1; in efx_tc_mangle()
1313 if ((fa->mangle.val >> 24) == tr_ttl) { in efx_tc_mangle()
1314 act->do_ttl_dec = 1; in efx_tc_mangle()
1322 return -EOPNOTSUPP; in efx_tc_mangle()
1326 fa->mangle.htype); in efx_tc_mangle()
1327 return -EOPNOTSUPP; in efx_tc_mangle()
1333 * efx_tc_incomplete_mangle() - check for leftover partial pedits
1340 * and reject them with -%EOPNOTSUPP.
1346 if (mung->dst_mac_32 || mung->dst_mac_16) { in efx_tc_incomplete_mangle()
1348 return -EOPNOTSUPP; in efx_tc_incomplete_mangle()
1350 if (mung->src_mac_16 || mung->src_mac_32) { in efx_tc_incomplete_mangle()
1352 return -EOPNOTSUPP; in efx_tc_incomplete_mangle()
1362 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_foreign()
1380 * the filter with source m-port == wire. in efx_tc_flower_replace_foreign()
1384 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter"); in efx_tc_flower_replace_foreign()
1390 if (tc->common.chain_index) { in efx_tc_flower_replace_foreign()
1393 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev); in efx_tc_flower_replace_foreign()
1397 tc->common.chain_index); in efx_tc_flower_replace_foreign()
1401 match.value.recirc_id = rid->fw_id; in efx_tc_flower_replace_foreign()
1412 * match +trk-est (CT_HIT=0) despite being on an established connection. in efx_tc_flower_replace_foreign()
1413 * So make -est imply -tcp_syn_fin_rst match to ensure these packets in efx_tc_flower_replace_foreign()
1419 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1420 goto release; in efx_tc_flower_replace_foreign()
1425 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace_foreign()
1426 switch (fa->id) { in efx_tc_flower_replace_foreign()
1429 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace_foreign()
1439 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1441 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1442 goto release; in efx_tc_flower_replace_foreign()
1447 goto release; in efx_tc_flower_replace_foreign()
1456 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1457 goto release; in efx_tc_flower_replace_foreign()
1465 goto release; in efx_tc_flower_replace_foreign()
1472 goto release; in efx_tc_flower_replace_foreign()
1475 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1477 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1478 goto release; in efx_tc_flower_replace_foreign()
1483 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1484 goto release; in efx_tc_flower_replace_foreign()
1486 INIT_LIST_HEAD(&rule->acts.list); in efx_tc_flower_replace_foreign()
1487 rule->cookie = tc->cookie; in efx_tc_flower_replace_foreign()
1488 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace_foreign()
1489 &rule->linkage, in efx_tc_flower_replace_foreign()
1493 goto release; in efx_tc_flower_replace_foreign()
1495 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1496 "Ignoring already-offloaded rule (cookie %lx)\n", in efx_tc_flower_replace_foreign()
1497 tc->cookie); in efx_tc_flower_replace_foreign()
1498 rc = -EEXIST; in efx_tc_flower_replace_foreign()
1499 goto release; in efx_tc_flower_replace_foreign()
1504 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1505 goto release; in efx_tc_flower_replace_foreign()
1512 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace_foreign()
1515 switch (fa->id) { in efx_tc_flower_replace_foreign()
1522 if (fa->hw_stats) { in efx_tc_flower_replace_foreign()
1525 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_replace_foreign()
1528 fa->hw_stats); in efx_tc_flower_replace_foreign()
1529 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1530 goto release; in efx_tc_flower_replace_foreign()
1533 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1534 goto release; in efx_tc_flower_replace_foreign()
1538 tc->cookie, in efx_tc_flower_replace_foreign()
1543 goto release; in efx_tc_flower_replace_foreign()
1545 act->count = ctr; in efx_tc_flower_replace_foreign()
1546 INIT_LIST_HEAD(&act->count_user); in efx_tc_flower_replace_foreign()
1551 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1554 goto release; in efx_tc_flower_replace_foreign()
1556 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace_foreign()
1571 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); in efx_tc_flower_replace_foreign()
1572 goto release; in efx_tc_flower_replace_foreign()
1574 act->dest_mport = rc; in efx_tc_flower_replace_foreign()
1575 act->deliver = 1; in efx_tc_flower_replace_foreign()
1580 goto release; in efx_tc_flower_replace_foreign()
1582 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace_foreign()
1584 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace_foreign()
1589 rc = -ENOMEM; in efx_tc_flower_replace_foreign()
1590 goto release; in efx_tc_flower_replace_foreign()
1596 rc = -EINVAL; in efx_tc_flower_replace_foreign()
1598 goto release; in efx_tc_flower_replace_foreign()
1600 act->decap = 1; in efx_tc_flower_replace_foreign()
1609 fa->id); in efx_tc_flower_replace_foreign()
1610 rc = -EOPNOTSUPP; in efx_tc_flower_replace_foreign()
1611 goto release; in efx_tc_flower_replace_foreign()
1620 efx_mae_mport_uplink(efx, &act->dest_mport); in efx_tc_flower_replace_foreign()
1621 act->deliver = 1; in efx_tc_flower_replace_foreign()
1626 goto release; in efx_tc_flower_replace_foreign()
1628 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace_foreign()
1629 act = NULL; /* Prevent double-free in error path */ in efx_tc_flower_replace_foreign()
1632 rule->match = match; in efx_tc_flower_replace_foreign()
1634 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_foreign()
1636 tc->cookie); in efx_tc_flower_replace_foreign()
1638 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); in efx_tc_flower_replace_foreign()
1641 goto release; in efx_tc_flower_replace_foreign()
1643 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, in efx_tc_flower_replace_foreign()
1644 rule->acts.fw_id, &rule->fw_id); in efx_tc_flower_replace_foreign()
1652 efx_mae_free_action_set_list(efx, &rule->acts); in efx_tc_flower_replace_foreign()
1653 release: in efx_tc_flower_replace_foreign()
1663 rhashtable_remove_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace_foreign()
1664 &rule->linkage, in efx_tc_flower_replace_foreign()
1666 efx_tc_free_action_set_list(efx, &rule->acts, false); in efx_tc_flower_replace_foreign()
1681 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace_lhs()
1685 if (tc->common.chain_index) { in efx_tc_flower_replace_lhs()
1687 return -EOPNOTSUPP; in efx_tc_flower_replace_lhs()
1690 if (match->mask.ct_state_trk && match->value.ct_state_trk) { in efx_tc_flower_replace_lhs()
1692 return -EOPNOTSUPP; in efx_tc_flower_replace_lhs()
1694 /* LHS rules are always -trk, so we don't need to match on that */ in efx_tc_flower_replace_lhs()
1695 match->mask.ct_state_trk = 0; in efx_tc_flower_replace_lhs()
1696 match->value.ct_state_trk = 0; in efx_tc_flower_replace_lhs()
1698 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack); in efx_tc_flower_replace_lhs()
1704 return -ENOMEM; in efx_tc_flower_replace_lhs()
1705 rule->cookie = tc->cookie; in efx_tc_flower_replace_lhs()
1706 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht, in efx_tc_flower_replace_lhs()
1707 &rule->linkage, in efx_tc_flower_replace_lhs()
1711 goto release; in efx_tc_flower_replace_lhs()
1713 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_lhs()
1714 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace_lhs()
1715 rc = -EEXIST; in efx_tc_flower_replace_lhs()
1717 goto release; in efx_tc_flower_replace_lhs()
1724 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule); in efx_tc_flower_replace_lhs()
1726 goto release; in efx_tc_flower_replace_lhs()
1728 rule->match = *match; in efx_tc_flower_replace_lhs()
1733 goto release; in efx_tc_flower_replace_lhs()
1735 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace_lhs()
1737 tc->cookie); in efx_tc_flower_replace_lhs()
1740 release: in efx_tc_flower_replace_lhs()
1741 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act); in efx_tc_flower_replace_lhs()
1743 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage, in efx_tc_flower_replace_lhs()
1755 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_replace()
1767 if (!tc_can_offload_extack(efx->net_dev, extack)) in efx_tc_flower_replace()
1768 return -EOPNOTSUPP; in efx_tc_flower_replace()
1769 if (WARN_ON(!efx->tc)) in efx_tc_flower_replace()
1770 return -ENETDOWN; in efx_tc_flower_replace()
1771 if (WARN_ON(!efx->tc->up)) in efx_tc_flower_replace()
1772 return -ENETDOWN; in efx_tc_flower_replace()
1783 netdev_name(net_dev), efv ? "non-" : "", in efx_tc_flower_replace()
1784 from_efv ? "non-" : ""); in efx_tc_flower_replace()
1785 return -EINVAL; in efx_tc_flower_replace()
1792 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port"); in efx_tc_flower_replace()
1802 return -EOPNOTSUPP; in efx_tc_flower_replace()
1813 if (tc->common.chain_index) { in efx_tc_flower_replace()
1822 * PF, but that would require a bunch of additional IDAs - in efx_tc_flower_replace()
1823 * one for each representor - and that's not likely to be in efx_tc_flower_replace()
1826 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, in efx_tc_flower_replace()
1827 efx->net_dev); in efx_tc_flower_replace()
1831 tc->common.chain_index); in efx_tc_flower_replace()
1835 match.value.recirc_id = rid->fw_id; in efx_tc_flower_replace()
1846 * match +trk-est (CT_HIT=0) despite being on an established connection. in efx_tc_flower_replace()
1847 * So make -est imply -tcp_syn_fin_rst match to ensure these packets in efx_tc_flower_replace()
1853 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
1854 goto release; in efx_tc_flower_replace()
1861 goto release; in efx_tc_flower_replace()
1865 rc = -ENOMEM; in efx_tc_flower_replace()
1866 goto release; in efx_tc_flower_replace()
1868 INIT_LIST_HEAD(&rule->acts.list); in efx_tc_flower_replace()
1869 rule->cookie = tc->cookie; in efx_tc_flower_replace()
1870 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace()
1871 &rule->linkage, in efx_tc_flower_replace()
1875 goto release; in efx_tc_flower_replace()
1877 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace()
1878 "Already offloaded rule (cookie %lx)\n", tc->cookie); in efx_tc_flower_replace()
1880 rc = -EEXIST; in efx_tc_flower_replace()
1881 goto release; in efx_tc_flower_replace()
1887 rc = -ENOMEM; in efx_tc_flower_replace()
1888 goto release; in efx_tc_flower_replace()
1900 * which describes the cumulative effect of all the packet-mutating in efx_tc_flower_replace()
1902 * action, once the action-set has been inserted into hardware, we in efx_tc_flower_replace()
1903 * append @act to the action-set list (@rule->acts); if this is a pipe in efx_tc_flower_replace()
1907 * This ensures that every allocated action-set is either attached to in efx_tc_flower_replace()
1908 * @rule->acts or pointed to by @act (and never both), and that only in efx_tc_flower_replace()
1909 * those action-sets in @rule->acts exist in hardware. Consequently, in efx_tc_flower_replace()
1911 * for @rule->acts we remove each action-set from hardware before in efx_tc_flower_replace()
1912 * freeing it (efx_tc_free_action_set_list()), even if the action-set in efx_tc_flower_replace()
1915 flow_action_for_each(i, fa, &fr->action) { in efx_tc_flower_replace()
1920 /* more actions after a non-pipe action */ in efx_tc_flower_replace()
1921 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action"); in efx_tc_flower_replace()
1922 rc = -EINVAL; in efx_tc_flower_replace()
1923 goto release; in efx_tc_flower_replace()
1926 if ((fa->id == FLOW_ACTION_REDIRECT || in efx_tc_flower_replace()
1927 fa->id == FLOW_ACTION_MIRRED || in efx_tc_flower_replace()
1928 fa->id == FLOW_ACTION_DROP) && fa->hw_stats) { in efx_tc_flower_replace()
1932 * mirred and gact (ok, shot, trap, goto-chain), which in efx_tc_flower_replace()
1945 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)"); in efx_tc_flower_replace()
1946 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
1947 goto release; in efx_tc_flower_replace()
1950 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) { in efx_tc_flower_replace()
1952 fa->hw_stats); in efx_tc_flower_replace()
1953 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
1954 goto release; in efx_tc_flower_replace()
1957 ctr = efx_tc_flower_get_counter_index(efx, tc->cookie, in efx_tc_flower_replace()
1962 goto release; in efx_tc_flower_replace()
1964 act->count = ctr; in efx_tc_flower_replace()
1965 INIT_LIST_HEAD(&act->count_user); in efx_tc_flower_replace()
1968 switch (fa->id) { in efx_tc_flower_replace()
1973 goto release; in efx_tc_flower_replace()
1975 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
1987 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
1989 goto release; in efx_tc_flower_replace()
1992 efx, encap_info, fa->dev, extack); in efx_tc_flower_replace()
1996 rc = -EIO; /* arbitrary */ in efx_tc_flower_replace()
1997 goto release; in efx_tc_flower_replace()
1999 act->encap_md = encap; in efx_tc_flower_replace()
2000 list_add_tail(&act->encap_user, &encap->users); in efx_tc_flower_replace()
2001 act->dest_mport = encap->dest_mport; in efx_tc_flower_replace()
2002 act->deliver = 1; in efx_tc_flower_replace()
2003 if (act->count && !WARN_ON(!act->count->cnt)) { in efx_tc_flower_replace()
2009 spin_lock_bh(&act->count->cnt->lock); in efx_tc_flower_replace()
2010 list_add_tail(&act->count_user, in efx_tc_flower_replace()
2011 &act->count->cnt->users); in efx_tc_flower_replace()
2012 spin_unlock_bh(&act->count->cnt->lock); in efx_tc_flower_replace()
2017 goto release; in efx_tc_flower_replace()
2019 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2020 act->user = &rule->acts; in efx_tc_flower_replace()
2022 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace()
2028 rc = -ENOMEM; in efx_tc_flower_replace()
2029 goto release; in efx_tc_flower_replace()
2037 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2039 goto release; in efx_tc_flower_replace()
2042 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); in efx_tc_flower_replace()
2046 goto release; in efx_tc_flower_replace()
2050 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port"); in efx_tc_flower_replace()
2051 goto release; in efx_tc_flower_replace()
2053 act->dest_mport = rc; in efx_tc_flower_replace()
2054 act->deliver = 1; in efx_tc_flower_replace()
2058 goto release; in efx_tc_flower_replace()
2060 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2062 if (fa->id == FLOW_ACTION_REDIRECT) in efx_tc_flower_replace()
2068 rc = -ENOMEM; in efx_tc_flower_replace()
2069 goto release; in efx_tc_flower_replace()
2074 if (act->vlan_push) { in efx_tc_flower_replace()
2075 act->vlan_push--; in efx_tc_flower_replace()
2077 act->vlan_pop++; in efx_tc_flower_replace()
2081 rc = -EINVAL; in efx_tc_flower_replace()
2082 goto release; in efx_tc_flower_replace()
2087 rc = -EINVAL; in efx_tc_flower_replace()
2090 goto release; in efx_tc_flower_replace()
2092 tci = fa->vlan.vid & VLAN_VID_MASK; in efx_tc_flower_replace()
2093 tci |= fa->vlan.prio << VLAN_PRIO_SHIFT; in efx_tc_flower_replace()
2094 act->vlan_tci[act->vlan_push] = cpu_to_be16(tci); in efx_tc_flower_replace()
2095 act->vlan_proto[act->vlan_push] = fa->vlan.proto; in efx_tc_flower_replace()
2096 act->vlan_push++; in efx_tc_flower_replace()
2101 goto release; in efx_tc_flower_replace()
2106 goto release; in efx_tc_flower_replace()
2115 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set"); in efx_tc_flower_replace()
2116 rc = -EINVAL; in efx_tc_flower_replace()
2117 goto release; in efx_tc_flower_replace()
2119 if (!fa->tunnel) { in efx_tc_flower_replace()
2120 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key"); in efx_tc_flower_replace()
2121 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2122 goto release; in efx_tc_flower_replace()
2124 encap_info = fa->tunnel; in efx_tc_flower_replace()
2132 * (and if we did there'd be no tunnel-device to give in efx_tc_flower_replace()
2137 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2138 goto release; in efx_tc_flower_replace()
2141 fa->id); in efx_tc_flower_replace()
2142 rc = -EOPNOTSUPP; in efx_tc_flower_replace()
2143 goto release; in efx_tc_flower_replace()
2149 goto release; in efx_tc_flower_replace()
2156 efx_mae_mport_uplink(efx, &act->dest_mport); in efx_tc_flower_replace()
2162 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, in efx_tc_flower_replace()
2163 &act->dest_mport); in efx_tc_flower_replace()
2164 act->deliver = 1; in efx_tc_flower_replace()
2168 goto release; in efx_tc_flower_replace()
2170 list_add_tail(&act->list, &rule->acts.list); in efx_tc_flower_replace()
2171 act = NULL; /* Prevent double-free in error path */ in efx_tc_flower_replace()
2174 netif_dbg(efx, drv, efx->net_dev, in efx_tc_flower_replace()
2176 tc->cookie); in efx_tc_flower_replace()
2178 rule->match = match; in efx_tc_flower_replace()
2180 rc = efx_mae_alloc_action_set_list(efx, &rule->acts); in efx_tc_flower_replace()
2183 goto release; in efx_tc_flower_replace()
2187 rule->fallback = &efx->tc->facts.pf; in efx_tc_flower_replace()
2190 rule->fallback = &efx->tc->facts.reps; in efx_tc_flower_replace()
2192 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n"); in efx_tc_flower_replace()
2193 acts_id = rule->fallback->fw_id; in efx_tc_flower_replace()
2195 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n"); in efx_tc_flower_replace()
2196 acts_id = rule->acts.fw_id; in efx_tc_flower_replace()
2198 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, in efx_tc_flower_replace()
2199 acts_id, &rule->fw_id); in efx_tc_flower_replace()
2207 efx_mae_free_action_set_list(efx, &rule->acts); in efx_tc_flower_replace()
2208 release: in efx_tc_flower_replace()
2218 rhashtable_remove_fast(&efx->tc->match_action_ht, in efx_tc_flower_replace()
2219 &rule->linkage, in efx_tc_flower_replace()
2221 efx_tc_free_action_set_list(efx, &rule->acts, false); in efx_tc_flower_replace()
2231 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_destroy()
2235 lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie, in efx_tc_flower_destroy()
2241 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act); in efx_tc_flower_destroy()
2242 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage, in efx_tc_flower_destroy()
2244 if (lhs_rule->match.encap) in efx_tc_flower_destroy()
2245 efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap); in efx_tc_flower_destroy()
2246 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n", in efx_tc_flower_destroy()
2247 lhs_rule->cookie); in efx_tc_flower_destroy()
2252 rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie, in efx_tc_flower_destroy()
2261 netif_warn(efx, drv, efx->net_dev, in efx_tc_flower_destroy()
2262 "Filter %lx not found to remove\n", tc->cookie); in efx_tc_flower_destroy()
2264 return -ENOENT; in efx_tc_flower_destroy()
2270 rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage, in efx_tc_flower_destroy()
2272 netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie); in efx_tc_flower_destroy()
2280 struct netlink_ext_ack *extack = tc->common.extack; in efx_tc_flower_stats()
2285 ctr = efx_tc_flower_find_counter_index(efx, tc->cookie); in efx_tc_flower_stats()
2290 netif_warn(efx, drv, efx->net_dev, in efx_tc_flower_stats()
2292 tc->cookie); in efx_tc_flower_stats()
2294 return -ENOENT; in efx_tc_flower_stats()
2296 if (WARN_ON(!ctr->cnt)) /* can't happen */ in efx_tc_flower_stats()
2297 return -EIO; in efx_tc_flower_stats()
2298 cnt = ctr->cnt; in efx_tc_flower_stats()
2300 spin_lock_bh(&cnt->lock); in efx_tc_flower_stats()
2302 packets = cnt->packets; in efx_tc_flower_stats()
2303 bytes = cnt->bytes; in efx_tc_flower_stats()
2304 flow_stats_update(&tc->stats, bytes - cnt->old_bytes, in efx_tc_flower_stats()
2305 packets - cnt->old_packets, 0, cnt->touched, in efx_tc_flower_stats()
2307 cnt->old_packets = packets; in efx_tc_flower_stats()
2308 cnt->old_bytes = bytes; in efx_tc_flower_stats()
2309 spin_unlock_bh(&cnt->lock); in efx_tc_flower_stats()
2318 if (!efx->tc) in efx_tc_flower()
2319 return -EOPNOTSUPP; in efx_tc_flower()
2321 mutex_lock(&efx->tc->mutex); in efx_tc_flower()
2322 switch (tc->command) { in efx_tc_flower()
2333 rc = -EOPNOTSUPP; in efx_tc_flower()
2336 mutex_unlock(&efx->tc->mutex); in efx_tc_flower()
2343 struct efx_tc_action_set_list *acts = &rule->acts; in efx_tc_configure_default_rule()
2344 struct efx_tc_match *match = &rule->match; in efx_tc_configure_default_rule()
2348 match->value.ingress_port = ing_port; in efx_tc_configure_default_rule()
2349 match->mask.ingress_port = ~0; in efx_tc_configure_default_rule()
2352 return -ENOMEM; in efx_tc_configure_default_rule()
2353 act->deliver = 1; in efx_tc_configure_default_rule()
2354 act->dest_mport = eg_port; in efx_tc_configure_default_rule()
2358 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); in efx_tc_configure_default_rule()
2359 list_add_tail(&act->list, &acts->list); in efx_tc_configure_default_rule()
2364 acts->fw_id, &rule->fw_id); in efx_tc_configure_default_rule()
2371 list_del(&act->list); in efx_tc_configure_default_rule()
2372 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_configure_default_rule()
2380 struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf; in efx_tc_configure_default_rule_pf()
2390 struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire; in efx_tc_configure_default_rule_wire()
2400 struct efx_tc_flow_rule *rule = &efv->dflt; in efx_tc_configure_default_rule_rep()
2401 struct efx_nic *efx = efv->parent; in efx_tc_configure_default_rule_rep()
2404 efx_mae_mport_mport(efx, efv->mport, &ing_port); in efx_tc_configure_default_rule_rep()
2405 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); in efx_tc_configure_default_rule_rep()
2412 if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL) in efx_tc_deconfigure_default_rule()
2414 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_tc_deconfigure_default_rule()
2425 return -ENOMEM; in efx_tc_configure_fallback_acts()
2426 act->deliver = 1; in efx_tc_configure_fallback_acts()
2427 act->dest_mport = eg_port; in efx_tc_configure_fallback_acts()
2431 EFX_WARN_ON_PARANOID(!list_empty(&acts->list)); in efx_tc_configure_fallback_acts()
2432 list_add_tail(&act->list, &acts->list); in efx_tc_configure_fallback_acts()
2438 list_del(&act->list); in efx_tc_configure_fallback_acts()
2439 efx_mae_free_action_set(efx, act->fw_id); in efx_tc_configure_fallback_acts()
2447 struct efx_tc_action_set_list *acts = &efx->tc->facts.pf; in efx_tc_configure_fallback_acts_pf()
2456 struct efx_tc_action_set_list *acts = &efx->tc->facts.reps; in efx_tc_configure_fallback_acts_reps()
2459 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port); in efx_tc_configure_fallback_acts_reps()
2474 rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label); in efx_tc_configure_rep_mport()
2477 pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n", in efx_tc_configure_rep_mport()
2478 efx->tc->reps_mport_id, rep_mport_label); in efx_tc_configure_rep_mport()
2480 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, in efx_tc_configure_rep_mport()
2481 &efx->tc->reps_mport_vport_id); in efx_tc_configure_rep_mport()
2487 efx_mae_free_mport(efx, efx->tc->reps_mport_id); in efx_tc_deconfigure_rep_mport()
2488 efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL; in efx_tc_deconfigure_rep_mport()
2496 if (efx->type->is_vf) in efx_tc_insert_rep_filters()
2498 if (!efx->tc) in efx_tc_insert_rep_filters()
2502 efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id); in efx_tc_insert_rep_filters()
2506 efx->tc->reps_filter_uc = rc; in efx_tc_insert_rep_filters()
2509 efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id); in efx_tc_insert_rep_filters()
2513 efx->tc->reps_filter_mc = rc; in efx_tc_insert_rep_filters()
2519 if (efx->type->is_vf) in efx_tc_remove_rep_filters()
2521 if (!efx->tc) in efx_tc_remove_rep_filters()
2523 if (efx->tc->reps_filter_mc >= 0) in efx_tc_remove_rep_filters()
2524 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc); in efx_tc_remove_rep_filters()
2525 efx->tc->reps_filter_mc = -1; in efx_tc_remove_rep_filters()
2526 if (efx->tc->reps_filter_uc >= 0) in efx_tc_remove_rep_filters()
2527 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc); in efx_tc_remove_rep_filters()
2528 efx->tc->reps_filter_uc = -1; in efx_tc_remove_rep_filters()
2535 rc = efx_mae_get_caps(efx, efx->tc->caps); in efx_init_tc()
2538 if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS) in efx_init_tc()
2543 netif_warn(efx, probe, efx->net_dev, in efx_init_tc()
2545 efx->tc->caps->match_field_count); in efx_init_tc()
2546 if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) { in efx_init_tc()
2547 netif_err(efx, probe, efx->net_dev, in efx_init_tc()
2549 efx->tc->caps->action_prios, EFX_TC_PRIO__NUM); in efx_init_tc()
2550 return -EIO; in efx_init_tc()
2573 efx->tc->up = true; in efx_init_tc()
2583 if (!efx->tc) in efx_fini_tc()
2585 if (efx->tc->up) in efx_fini_tc()
2588 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); in efx_fini_tc()
2589 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); in efx_fini_tc()
2590 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf); in efx_fini_tc()
2591 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps); in efx_fini_tc()
2592 efx->tc->up = false; in efx_fini_tc()
2604 WARN_ON(refcount_read(&encap->ref)); in efx_tc_encap_match_free()
2613 WARN_ON(refcount_read(&rid->ref)); in efx_tc_recirc_free()
2614 ida_free(&efx->tc->recirc_ida, rid->fw_id); in efx_tc_recirc_free()
2623 netif_err(efx, drv, efx->net_dev, in efx_tc_lhs_free()
2625 rule->cookie); in efx_tc_lhs_free()
2627 if (rule->lhs_act.zone) in efx_tc_lhs_free()
2628 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone); in efx_tc_lhs_free()
2629 if (rule->lhs_act.count) in efx_tc_lhs_free()
2630 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count); in efx_tc_lhs_free()
2640 WARN_ON(refcount_read(&ped->ref)); in efx_tc_mac_free()
2649 netif_err(efx, drv, efx->net_dev, in efx_tc_flow_free()
2651 rule->cookie); in efx_tc_flow_free()
2663 if (efx->type->is_vf) in efx_init_struct_tc()
2666 efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL); in efx_init_struct_tc()
2667 if (!efx->tc) in efx_init_struct_tc()
2668 return -ENOMEM; in efx_init_struct_tc()
2669 efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL); in efx_init_struct_tc()
2670 if (!efx->tc->caps) { in efx_init_struct_tc()
2671 rc = -ENOMEM; in efx_init_struct_tc()
2674 INIT_LIST_HEAD(&efx->tc->block_list); in efx_init_struct_tc()
2676 mutex_init(&efx->tc->mutex); in efx_init_struct_tc()
2677 init_waitqueue_head(&efx->tc->flush_wq); in efx_init_struct_tc()
2684 rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params); in efx_init_struct_tc()
2687 rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params); in efx_init_struct_tc()
2690 rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); in efx_init_struct_tc()
2693 rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params); in efx_init_struct_tc()
2699 rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params); in efx_init_struct_tc()
2702 ida_init(&efx->tc->recirc_ida); in efx_init_struct_tc()
2703 efx->tc->reps_filter_uc = -1; in efx_init_struct_tc()
2704 efx->tc->reps_filter_mc = -1; in efx_init_struct_tc()
2705 INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); in efx_init_struct_tc()
2706 efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_init_struct_tc()
2707 INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); in efx_init_struct_tc()
2708 efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; in efx_init_struct_tc()
2709 INIT_LIST_HEAD(&efx->tc->facts.pf.list); in efx_init_struct_tc()
2710 efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; in efx_init_struct_tc()
2711 INIT_LIST_HEAD(&efx->tc->facts.reps.list); in efx_init_struct_tc()
2712 efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL; in efx_init_struct_tc()
2713 efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type; in efx_init_struct_tc()
2718 rhashtable_destroy(&efx->tc->lhs_rule_ht); in efx_init_struct_tc()
2720 rhashtable_destroy(&efx->tc->match_action_ht); in efx_init_struct_tc()
2722 rhashtable_destroy(&efx->tc->encap_match_ht); in efx_init_struct_tc()
2724 rhashtable_destroy(&efx->tc->mac_ht); in efx_init_struct_tc()
2730 mutex_destroy(&efx->tc->mutex); in efx_init_struct_tc()
2731 kfree(efx->tc->caps); in efx_init_struct_tc()
2733 kfree(efx->tc); in efx_init_struct_tc()
2734 efx->tc = NULL; in efx_init_struct_tc()
2740 if (!efx->tc) in efx_fini_struct_tc()
2743 mutex_lock(&efx->tc->mutex); in efx_fini_struct_tc()
2744 EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != in efx_fini_struct_tc()
2746 EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != in efx_fini_struct_tc()
2748 EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id != in efx_fini_struct_tc()
2750 EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id != in efx_fini_struct_tc()
2752 rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx); in efx_fini_struct_tc()
2753 rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, in efx_fini_struct_tc()
2755 rhashtable_free_and_destroy(&efx->tc->encap_match_ht, in efx_fini_struct_tc()
2758 rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx); in efx_fini_struct_tc()
2759 WARN_ON(!ida_is_empty(&efx->tc->recirc_ida)); in efx_fini_struct_tc()
2760 ida_destroy(&efx->tc->recirc_ida); in efx_fini_struct_tc()
2761 rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL); in efx_fini_struct_tc()
2764 mutex_unlock(&efx->tc->mutex); in efx_fini_struct_tc()
2765 mutex_destroy(&efx->tc->mutex); in efx_fini_struct_tc()
2766 kfree(efx->tc->caps); in efx_fini_struct_tc()
2767 kfree(efx->tc); in efx_fini_struct_tc()
2768 efx->tc = NULL; in efx_fini_struct_tc()