1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
8 #include "counters.h"
94 static int mlx5_ib_read_counters(struct ib_counters *counters,  in mlx5_ib_read_counters()  argument
98 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);  in mlx5_ib_read_counters()
103 	mutex_lock(&mcounters->mcntrs_mutex);  in mlx5_ib_read_counters()
104 	if (mcounters->cntrs_max_index > read_attr->ncounters) {  in mlx5_ib_read_counters()
105 		ret = -EINVAL;  in mlx5_ib_read_counters()
109 	mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),  in mlx5_ib_read_counters()
112 		ret = -ENOMEM;  in mlx5_ib_read_counters()
116 	mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;  in mlx5_ib_read_counters()
117 	mread_attr.flags = read_attr->flags;  in mlx5_ib_read_counters()
118 	ret = mcounters->read_counters(counters->device, &mread_attr);  in mlx5_ib_read_counters()
122 	/* do the pass over the counters data array to assign according to the  in mlx5_ib_read_counters()
125 	desc = mcounters->counters_data;  in mlx5_ib_read_counters()
126 	for (i = 0; i < mcounters->ncounters; i++)  in mlx5_ib_read_counters()
127 		read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];  in mlx5_ib_read_counters()
132 	mutex_unlock(&mcounters->mcntrs_mutex);  in mlx5_ib_read_counters()
136 static int mlx5_ib_destroy_counters(struct ib_counters *counters)  in mlx5_ib_destroy_counters()  argument
138 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);  in mlx5_ib_destroy_counters()
140 	mlx5_ib_counters_clear_description(counters);  in mlx5_ib_destroy_counters()
141 	if (mcounters->hw_cntrs_hndl)  in mlx5_ib_destroy_counters()
142 		mlx5_fc_destroy(to_mdev(counters->device)->mdev,  in mlx5_ib_destroy_counters()
143 				mcounters->hw_cntrs_hndl);  in mlx5_ib_destroy_counters()
147 static int mlx5_ib_create_counters(struct ib_counters *counters,  in mlx5_ib_create_counters()  argument
150 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);  in mlx5_ib_create_counters()
152 	mutex_init(&mcounters->mcntrs_mutex);  in mlx5_ib_create_counters()
160 	return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :  in get_counters()
161 						   &dev->port[port_num].cnts;  in get_counters()
165  * mlx5_ib_get_counters_id - Returns counters id to use for device+port
169  * mlx5_ib_get_counters_id() Returns counters set id to use for given
177 	return cnts->set_id;  in mlx5_ib_get_counters_id()
186 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +  in do_alloc_stats()
187 			  cnts->num_ext_ppcnt_counters;  in do_alloc_stats()
188 	stats = rdma_alloc_hw_stats_struct(cnts->descs,  in do_alloc_stats()
190 					   cnts->num_op_counters,  in do_alloc_stats()
195 	for (i = 0; i < cnts->num_op_counters; i++)  in do_alloc_stats()
196 		set_bit(num_hw_counters + i, stats->is_disabled);  in do_alloc_stats()
205 	const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;  in mlx5_ib_alloc_hw_device_stats()
214 	const struct mlx5_ib_counters *cnts = &dev->port[port_num - 1].cnts;  in mlx5_ib_alloc_hw_port_stats()
235 	for (i = 0; i < cnts->num_q_counters; i++) {  in mlx5_ib_query_q_counters()
236 		val = *(__be32 *)((void *)out + cnts->offsets[i]);  in mlx5_ib_query_q_counters()
237 		stats->value[i] = (u64)be32_to_cpu(val);  in mlx5_ib_query_q_counters()
247 	int offset = cnts->num_q_counters + cnts->num_cong_counters;  in mlx5_ib_query_ext_ppcnt_counters()
255 		return -ENOMEM;  in mlx5_ib_query_ext_ppcnt_counters()
259 	ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,  in mlx5_ib_query_ext_ppcnt_counters()
264 	for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)  in mlx5_ib_query_ext_ppcnt_counters()
265 		stats->value[i + offset] =  in mlx5_ib_query_ext_ppcnt_counters()
267 				    cnts->offsets[i + offset]));  in mlx5_ib_query_ext_ppcnt_counters()
278 	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);  in do_get_hw_stats()
284 		return -EINVAL;  in do_get_hw_stats()
286 	num_counters = cnts->num_q_counters +  in do_get_hw_stats()
287 		       cnts->num_cong_counters +  in do_get_hw_stats()
288 		       cnts->num_ext_ppcnt_counters;  in do_get_hw_stats()
291 	ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);  in do_get_hw_stats()
295 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {  in do_get_hw_stats()
301 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {  in do_get_hw_stats()
305 			/* If port is not affiliated yet, its in down state  in do_get_hw_stats()
306 			 * which doesn't have any counters yet, so it would be  in do_get_hw_stats()
311 		ret = mlx5_lag_query_cong_counters(dev->mdev,  in do_get_hw_stats()
312 						   stats->value +  in do_get_hw_stats()
313 						   cnts->num_q_counters,  in do_get_hw_stats()
314 						   cnts->num_cong_counters,  in do_get_hw_stats()
315 						   cnts->offsets +  in do_get_hw_stats()
316 						   cnts->num_q_counters);  in do_get_hw_stats()
338 	cnts = get_counters(dev, port_num - 1);  in do_get_op_stat()
339 	opfcs = cnts->opfcs;  in do_get_op_stat()
340 	type = *(u32 *)cnts->descs[index].priv;  in do_get_op_stat()
342 		return -EINVAL;  in do_get_op_stat()
347 	ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,  in do_get_op_stat()
353 	stats->value[index] = packets;  in do_get_op_stat()
365 	cnts = get_counters(dev, port_num - 1);  in do_get_op_stats()
366 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +  in do_get_op_stats()
367 			  cnts->num_ext_ppcnt_counters;  in do_get_op_stats()
369 	     index < (num_hw_counters + cnts->num_op_counters); index++) {  in do_get_op_stats()
375 	return cnts->num_op_counters;  in do_get_op_stats()
386 	cnts = get_counters(dev, port_num - 1);  in mlx5_ib_get_hw_stats()
387 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +  in mlx5_ib_get_hw_stats()
388 		cnts->num_ext_ppcnt_counters;  in mlx5_ib_get_hw_stats()
389 	num_counters = num_hw_counters + cnts->num_op_counters;  in mlx5_ib_get_hw_stats()
392 		return -EINVAL;  in mlx5_ib_get_hw_stats()
412 	struct mlx5_ib_dev *dev = to_mdev(counter->device);  in mlx5_ib_counter_alloc_stats()
414 		get_counters(dev, counter->port - 1);  in mlx5_ib_counter_alloc_stats()
421 	struct mlx5_ib_dev *dev = to_mdev(counter->device);  in mlx5_ib_counter_update_stats()
423 		get_counters(dev, counter->port - 1);  in mlx5_ib_counter_update_stats()
425 	return mlx5_ib_query_q_counters(dev->mdev, cnts,  in mlx5_ib_counter_update_stats()
426 					counter->stats, counter->id);  in mlx5_ib_counter_update_stats()
431 	struct mlx5_ib_dev *dev = to_mdev(counter->device);  in mlx5_ib_counter_dealloc()
434 	if (!counter->id)  in mlx5_ib_counter_dealloc()
439 	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);  in mlx5_ib_counter_dealloc()
440 	return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);  in mlx5_ib_counter_dealloc()
446 	struct mlx5_ib_dev *dev = to_mdev(qp->device);  in mlx5_ib_counter_bind_qp()
449 	if (!counter->id) {  in mlx5_ib_counter_bind_qp()
456 		err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);  in mlx5_ib_counter_bind_qp()
459 		counter->id =  in mlx5_ib_counter_bind_qp()
471 	counter->id = 0;  in mlx5_ib_counter_bind_qp()
492 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {  in mlx5_ib_fill_counters()
499 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {  in mlx5_ib_fill_counters()
506 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {  in mlx5_ib_fill_counters()
513 	if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {  in mlx5_ib_fill_counters()
520 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {  in mlx5_ib_fill_counters()
527 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {  in mlx5_ib_fill_counters()
540 	if (MLX5_CAP_FLOWTABLE(dev->mdev,  in mlx5_ib_fill_counters()
549 	if (MLX5_CAP_FLOWTABLE(dev->mdev,  in mlx5_ib_fill_counters()
567 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))  in __mlx5_ib_alloc_counters()
570 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))  in __mlx5_ib_alloc_counters()
573 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))  in __mlx5_ib_alloc_counters()
576 	if (MLX5_CAP_GEN(dev->mdev, roce_accl))  in __mlx5_ib_alloc_counters()
579 	cnts->num_q_counters = num_counters;  in __mlx5_ib_alloc_counters()
581 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {  in __mlx5_ib_alloc_counters()
582 		cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);  in __mlx5_ib_alloc_counters()
585 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {  in __mlx5_ib_alloc_counters()
586 		cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);  in __mlx5_ib_alloc_counters()
592 	if (MLX5_CAP_FLOWTABLE(dev->mdev,  in __mlx5_ib_alloc_counters()
596 	if (MLX5_CAP_FLOWTABLE(dev->mdev,  in __mlx5_ib_alloc_counters()
600 	cnts->num_op_counters = num_op_counters;  in __mlx5_ib_alloc_counters()
602 	cnts->descs = kcalloc(num_counters,  in __mlx5_ib_alloc_counters()
604 	if (!cnts->descs)  in __mlx5_ib_alloc_counters()
605 		return -ENOMEM;  in __mlx5_ib_alloc_counters()
607 	cnts->offsets = kcalloc(num_counters,  in __mlx5_ib_alloc_counters()
608 				sizeof(*cnts->offsets), GFP_KERNEL);  in __mlx5_ib_alloc_counters()
609 	if (!cnts->offsets)  in __mlx5_ib_alloc_counters()
615 	kfree(cnts->descs);  in __mlx5_ib_alloc_counters()
616 	cnts->descs = NULL;  in __mlx5_ib_alloc_counters()
617 	return -ENOMEM;  in __mlx5_ib_alloc_counters()
626 	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;  in mlx5_ib_dealloc_counters()
632 		if (dev->port[i].cnts.set_id) {  in mlx5_ib_dealloc_counters()
634 				 dev->port[i].cnts.set_id);  in mlx5_ib_dealloc_counters()
635 			mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);  in mlx5_ib_dealloc_counters()
637 		kfree(dev->port[i].cnts.descs);  in mlx5_ib_dealloc_counters()
638 		kfree(dev->port[i].cnts.offsets);  in mlx5_ib_dealloc_counters()
641 			if (!dev->port[i].cnts.opfcs[j].fc)  in mlx5_ib_dealloc_counters()
646 					&dev->port[i].cnts.opfcs[j], j);  in mlx5_ib_dealloc_counters()
647 			mlx5_fc_destroy(dev->mdev,  in mlx5_ib_dealloc_counters()
648 					dev->port[i].cnts.opfcs[j].fc);  in mlx5_ib_dealloc_counters()
649 			dev->port[i].cnts.opfcs[j].fc = NULL;  in mlx5_ib_dealloc_counters()
664 	is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;  in mlx5_ib_alloc_counters()
665 	num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;  in mlx5_ib_alloc_counters()
668 		err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);  in mlx5_ib_alloc_counters()
672 		mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,  in mlx5_ib_alloc_counters()
673 				      dev->port[i].cnts.offsets);  in mlx5_ib_alloc_counters()
678 		err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);  in mlx5_ib_alloc_counters()
686 		dev->port[i].cnts.set_id =  in mlx5_ib_alloc_counters()
699 	struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;  in read_flow_counters()
702 	return mlx5_fc_query(dev->mdev, fc,  in read_flow_counters()
703 			     &read_attr->out[IB_COUNTER_PACKETS],  in read_flow_counters()
704 			     &read_attr->out[IB_COUNTER_BYTES]);  in read_flow_counters()
707 /* flow counters currently expose two counters packets and bytes */
710 	struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,  in counters_set_description()  argument
713 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);  in counters_set_description()
718 		return -EINVAL;  in counters_set_description()
721 	mcounters->type = counters_type;  in counters_set_description()
722 	mcounters->read_counters = read_flow_counters;  in counters_set_description()
723 	mcounters->counters_num = FLOW_COUNTERS_NUM;  in counters_set_description()
724 	mcounters->ncounters = ncounters;  in counters_set_description()
728 			return -EINVAL;  in counters_set_description()
734 	mutex_lock(&mcounters->mcntrs_mutex);  in counters_set_description()
735 	mcounters->counters_data = desc_data;  in counters_set_description()
736 	mcounters->cntrs_max_index = cntrs_max_index;  in counters_set_description()
737 	mutex_unlock(&mcounters->mcntrs_mutex);  in counters_set_description()
752 	if (ucmd && ucmd->ncounters_data != 0) {  in mlx5_ib_flow_counters_set_data()
753 		cntrs_data = ucmd->data;  in mlx5_ib_flow_counters_set_data()
754 		if (cntrs_data->ncounters > MAX_COUNTERS_NUM)  in mlx5_ib_flow_counters_set_data()
755 			return -EINVAL;  in mlx5_ib_flow_counters_set_data()
757 		desc_data = kcalloc(cntrs_data->ncounters,  in mlx5_ib_flow_counters_set_data()
761 			return  -ENOMEM;  in mlx5_ib_flow_counters_set_data()
764 				   u64_to_user_ptr(cntrs_data->counters_data),  in mlx5_ib_flow_counters_set_data()
765 				   sizeof(*desc_data) * cntrs_data->ncounters)) {  in mlx5_ib_flow_counters_set_data()
766 			ret = -EFAULT;  in mlx5_ib_flow_counters_set_data()
771 	if (!mcounters->hw_cntrs_hndl) {  in mlx5_ib_flow_counters_set_data()
772 		mcounters->hw_cntrs_hndl = mlx5_fc_create(  in mlx5_ib_flow_counters_set_data()
773 			to_mdev(ibcounters->device)->mdev, false);  in mlx5_ib_flow_counters_set_data()
774 		if (IS_ERR(mcounters->hw_cntrs_hndl)) {  in mlx5_ib_flow_counters_set_data()
775 			ret = PTR_ERR(mcounters->hw_cntrs_hndl);  in mlx5_ib_flow_counters_set_data()
782 		/* counters already bound to at least one flow */  in mlx5_ib_flow_counters_set_data()
783 		if (mcounters->cntrs_max_index) {  in mlx5_ib_flow_counters_set_data()
784 			ret = -EINVAL;  in mlx5_ib_flow_counters_set_data()
791 					       cntrs_data->ncounters);  in mlx5_ib_flow_counters_set_data()
795 	} else if (!mcounters->cntrs_max_index) {  in mlx5_ib_flow_counters_set_data()
796 		/* counters not bound yet, must have udata passed */  in mlx5_ib_flow_counters_set_data()
797 		ret = -EINVAL;  in mlx5_ib_flow_counters_set_data()
805 		mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,  in mlx5_ib_flow_counters_set_data()
806 				mcounters->hw_cntrs_hndl);  in mlx5_ib_flow_counters_set_data()
807 		mcounters->hw_cntrs_hndl = NULL;  in mlx5_ib_flow_counters_set_data()
814 void mlx5_ib_counters_clear_description(struct ib_counters *counters)  in mlx5_ib_counters_clear_description()  argument
818 	if (!counters || atomic_read(&counters->usecnt) != 1)  in mlx5_ib_counters_clear_description()
821 	mcounters = to_mcounters(counters);  in mlx5_ib_counters_clear_description()
823 	mutex_lock(&mcounters->mcntrs_mutex);  in mlx5_ib_counters_clear_description()
824 	kfree(mcounters->counters_data);  in mlx5_ib_counters_clear_description()
825 	mcounters->counters_data = NULL;  in mlx5_ib_counters_clear_description()
826 	mcounters->cntrs_max_index = 0;  in mlx5_ib_counters_clear_description()
827 	mutex_unlock(&mcounters->mcntrs_mutex);  in mlx5_ib_counters_clear_description()
839 	cnts = &dev->port[port - 1].cnts;  in mlx5_ib_modify_stat()
840 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +  in mlx5_ib_modify_stat()
841 		cnts->num_ext_ppcnt_counters;  in mlx5_ib_modify_stat()
843 	    index >= (num_hw_counters + cnts->num_op_counters))  in mlx5_ib_modify_stat()
844 		return -EINVAL;  in mlx5_ib_modify_stat()
846 	if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))  in mlx5_ib_modify_stat()
847 		return -EINVAL;  in mlx5_ib_modify_stat()
849 	type = *(u32 *)cnts->descs[index].priv;  in mlx5_ib_modify_stat()
851 		return -EINVAL;  in mlx5_ib_modify_stat()
853 	opfc = &cnts->opfcs[type];  in mlx5_ib_modify_stat()
856 		if (opfc->fc)  in mlx5_ib_modify_stat()
857 			return -EEXIST;  in mlx5_ib_modify_stat()
859 		opfc->fc = mlx5_fc_create(dev->mdev, false);  in mlx5_ib_modify_stat()
860 		if (IS_ERR(opfc->fc))  in mlx5_ib_modify_stat()
861 			return PTR_ERR(opfc->fc);  in mlx5_ib_modify_stat()
865 			mlx5_fc_destroy(dev->mdev, opfc->fc);  in mlx5_ib_modify_stat()
866 			opfc->fc = NULL;  in mlx5_ib_modify_stat()
871 	if (!opfc->fc)  in mlx5_ib_modify_stat()
872 		return -EINVAL;  in mlx5_ib_modify_stat()
875 	mlx5_fc_destroy(dev->mdev, opfc->fc);  in mlx5_ib_modify_stat()
876 	opfc->fc = NULL;  in mlx5_ib_modify_stat()
912 	ib_set_device_ops(&dev->ib_dev, &counters_ops);  in mlx5_ib_counters_init()
914 	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))  in mlx5_ib_counters_init()
917 	if (is_mdev_switchdev_mode(dev->mdev))  in mlx5_ib_counters_init()
918 		ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);  in mlx5_ib_counters_init()
920 		ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);  in mlx5_ib_counters_init()
926 	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))  in mlx5_ib_counters_cleanup()