1038 						      struct mlx5_ib_flow_prio *ft_prio,  in _create_flow_rule()  argument
1044 	struct mlx5_flow_table	*ft = ft_prio->flow_table;  in _create_flow_rule()
1151 	ft_prio->refcount++;  in _create_flow_rule()
1152 	handler->prio = ft_prio;  in _create_flow_rule()
1155 	ft_prio->flow_table = ft;  in _create_flow_rule()
1166 						     struct mlx5_ib_flow_prio *ft_prio,  in create_flow_rule()  argument
1170 	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);  in create_flow_rule()
1179 							  struct mlx5_ib_flow_prio *ft_prio,  in create_leftovers_rule()  argument
1216 	handler = create_flow_rule(dev, ft_prio,  in create_leftovers_rule()
1221 		handler_ucast = create_flow_rule(dev, ft_prio,  in create_leftovers_rule()
1226 			ft_prio->refcount--;  in create_leftovers_rule()
1284 	struct mlx5_ib_flow_prio *ft_prio;  in mlx5_ib_create_flow()  local
1347 	ft_prio = get_flow_table(dev, flow_attr,  in mlx5_ib_create_flow()
1349 	if (IS_ERR(ft_prio)) {  in mlx5_ib_create_flow()
1350 		err = PTR_ERR(ft_prio);  in mlx5_ib_create_flow()
1377 		handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,  in mlx5_ib_create_flow()
1382 		handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);  in mlx5_ib_create_flow()
1385 		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);  in mlx5_ib_create_flow()
1405 	put_flow_table(dev, ft_prio, false);  in mlx5_ib_create_flow()
1515 		      struct mlx5_ib_flow_prio *ft_prio,  in _create_raw_flow_rule()  argument
1525 	struct mlx5_flow_table *ft = ft_prio->flow_table;  in _create_raw_flow_rule()
1551 	ft_prio->refcount++;  in _create_raw_flow_rule()
1552 	handler->prio = ft_prio;  in _create_raw_flow_rule()
1554 	ft_prio->flow_table = ft;  in _create_raw_flow_rule()
1609 	struct mlx5_ib_flow_prio *ft_prio;  in raw_fs_rule_add()  local
1628 	ft_prio = _get_flow_table(dev, fs_matcher->priority,  in raw_fs_rule_add()
1630 	if (IS_ERR(ft_prio)) {  in raw_fs_rule_add()
1631 		err = PTR_ERR(ft_prio);  in raw_fs_rule_add()
1660 	handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,  in raw_fs_rule_add()
1678 	put_flow_table(dev, ft_prio, false);  in raw_fs_rule_add()
2027 				     struct mlx5_ib_flow_prio *ft_prio,  in steering_anchor_create_ft()  argument
2034 	if (ft_prio->anchor.ft)  in steering_anchor_create_ft()
2051 	ft_prio->anchor.ft = ft;  in steering_anchor_create_ft()
2056 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_destroy_ft()  argument
2058 	if (ft_prio->anchor.ft) {  in steering_anchor_destroy_ft()
2059 		mlx5_destroy_flow_table(ft_prio->anchor.ft);  in steering_anchor_destroy_ft()
2060 		ft_prio->anchor.ft = NULL;  in steering_anchor_destroy_ft()
2065 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_create_fg_drop()  argument
2072 	if (ft_prio->anchor.fg_drop)  in steering_anchor_create_fg_drop()
2082 	fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);  in steering_anchor_create_fg_drop()
2088 	ft_prio->anchor.fg_drop = fg;  in steering_anchor_create_fg_drop()
2097 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_destroy_fg_drop()  argument
2099 	if (ft_prio->anchor.fg_drop) {  in steering_anchor_destroy_fg_drop()
2100 		mlx5_destroy_flow_group(ft_prio->anchor.fg_drop);  in steering_anchor_destroy_fg_drop()
2101 		ft_prio->anchor.fg_drop = NULL;  in steering_anchor_destroy_fg_drop()
2106 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_create_fg_goto_table()  argument
2113 	if (ft_prio->anchor.fg_goto_table)  in steering_anchor_create_fg_goto_table()
2120 	fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);  in steering_anchor_create_fg_goto_table()
2125 	ft_prio->anchor.fg_goto_table = fg;  in steering_anchor_create_fg_goto_table()
2134 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_destroy_fg_goto_table()  argument
2136 	if (ft_prio->anchor.fg_goto_table) {  in steering_anchor_destroy_fg_goto_table()
2137 		mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table);  in steering_anchor_destroy_fg_goto_table()
2138 		ft_prio->anchor.fg_goto_table = NULL;  in steering_anchor_destroy_fg_goto_table()
2143 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_create_rule_drop()  argument
2148 	if (ft_prio->anchor.rule_drop)  in steering_anchor_create_rule_drop()
2151 	flow_act.fg = ft_prio->anchor.fg_drop;  in steering_anchor_create_rule_drop()
2154 	handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,  in steering_anchor_create_rule_drop()
2159 	ft_prio->anchor.rule_drop = handle;  in steering_anchor_create_rule_drop()
2164 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_destroy_rule_drop()  argument
2166 	if (ft_prio->anchor.rule_drop) {  in steering_anchor_destroy_rule_drop()
2167 		mlx5_del_flow_rules(ft_prio->anchor.rule_drop);  in steering_anchor_destroy_rule_drop()
2168 		ft_prio->anchor.rule_drop = NULL;  in steering_anchor_destroy_rule_drop()
2173 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_create_rule_goto_table()  argument
2179 	if (ft_prio->anchor.rule_goto_table)  in steering_anchor_create_rule_goto_table()
2184 	flow_act.fg = ft_prio->anchor.fg_goto_table;  in steering_anchor_create_rule_goto_table()
2187 	dest.ft = ft_prio->flow_table;  in steering_anchor_create_rule_goto_table()
2189 	handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,  in steering_anchor_create_rule_goto_table()
2194 	ft_prio->anchor.rule_goto_table = handle;  in steering_anchor_create_rule_goto_table()
2200 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)  in steering_anchor_destroy_rule_goto_table()  argument
2202 	if (ft_prio->anchor.rule_goto_table) {  in steering_anchor_destroy_rule_goto_table()
2203 		mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table);  in steering_anchor_destroy_rule_goto_table()
2204 		ft_prio->anchor.rule_goto_table = NULL;  in steering_anchor_destroy_rule_goto_table()
2209 				      struct mlx5_ib_flow_prio *ft_prio,  in steering_anchor_create_res()  argument
2214 	err = steering_anchor_create_ft(dev, ft_prio, ns_type);  in steering_anchor_create_res()
2218 	err = steering_anchor_create_fg_drop(ft_prio);  in steering_anchor_create_res()
2222 	err = steering_anchor_create_fg_goto_table(ft_prio);  in steering_anchor_create_res()
2226 	err = steering_anchor_create_rule_drop(ft_prio);  in steering_anchor_create_res()
2230 	err = steering_anchor_create_rule_goto_table(ft_prio);  in steering_anchor_create_res()
2237 	steering_anchor_destroy_rule_drop(ft_prio);  in steering_anchor_create_res()
2239 	steering_anchor_destroy_fg_goto_table(ft_prio);  in steering_anchor_create_res()
2241 	steering_anchor_destroy_fg_drop(ft_prio);  in steering_anchor_create_res()
2243 	steering_anchor_destroy_ft(ft_prio);  in steering_anchor_create_res()
2248 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)  in mlx5_steering_anchor_destroy_res()  argument
2250 	steering_anchor_destroy_rule_goto_table(ft_prio);  in mlx5_steering_anchor_destroy_res()
2251 	steering_anchor_destroy_rule_drop(ft_prio);  in mlx5_steering_anchor_destroy_res()
2252 	steering_anchor_destroy_fg_goto_table(ft_prio);  in mlx5_steering_anchor_destroy_res()
2253 	steering_anchor_destroy_fg_drop(ft_prio);  in mlx5_steering_anchor_destroy_res()
2254 	steering_anchor_destroy_ft(ft_prio);  in mlx5_steering_anchor_destroy_res()
2267 	if (!--obj->ft_prio->anchor.rule_goto_table_ref)  in steering_anchor_cleanup()
2268 		steering_anchor_destroy_rule_goto_table(obj->ft_prio);  in steering_anchor_cleanup()
2270 	put_flow_table(obj->dev, obj->ft_prio, true);  in steering_anchor_cleanup()
2408 	struct mlx5_ib_flow_prio *ft_prio;  in UVERBS_HANDLER()  local
2436 	ft_prio = _get_flow_table(dev, priority, ns_type, 0);  in UVERBS_HANDLER()
2437 	if (IS_ERR(ft_prio)) {  in UVERBS_HANDLER()
2438 		err = PTR_ERR(ft_prio);  in UVERBS_HANDLER()
2442 	ft_prio->refcount++;  in UVERBS_HANDLER()
2444 	if (!ft_prio->anchor.rule_goto_table_ref) {  in UVERBS_HANDLER()
2445 		err = steering_anchor_create_res(dev, ft_prio, ns_type);  in UVERBS_HANDLER()
2450 	ft_prio->anchor.rule_goto_table_ref++;  in UVERBS_HANDLER()
2452 	ft_id = mlx5_flow_table_id(ft_prio->anchor.ft);  in UVERBS_HANDLER()
2463 	obj->ft_prio = ft_prio;  in UVERBS_HANDLER()
2469 	--ft_prio->anchor.rule_goto_table_ref;  in UVERBS_HANDLER()
2470 	mlx5_steering_anchor_destroy_res(ft_prio);  in UVERBS_HANDLER()
2472 	put_flow_table(dev, ft_prio, true);  in UVERBS_HANDLER()