31  * into manual mode if there's any counter, so that the user is able to
71 	struct rdma_counter *counter;  in rdma_counter_alloc()  local
77 	counter = kzalloc(sizeof(*counter), GFP_KERNEL);  in rdma_counter_alloc()
78 	if (!counter)  in rdma_counter_alloc()
81 	counter->device    = dev;  in rdma_counter_alloc()
82 	counter->port      = port;  in rdma_counter_alloc()
83 	counter->res.type  = RDMA_RESTRACK_COUNTER;  in rdma_counter_alloc()
84 	counter->stats     = dev->ops.counter_alloc_stats(counter);  in rdma_counter_alloc()
85 	if (!counter->stats)  in rdma_counter_alloc()
100 	counter->mode.mode = mode;  in rdma_counter_alloc()
101 	kref_init(&counter->kref);  in rdma_counter_alloc()
102 	mutex_init(&counter->lock);  in rdma_counter_alloc()
104 	return counter;  in rdma_counter_alloc()
108 	kfree(counter->stats);  in rdma_counter_alloc()
110 	kfree(counter);  in rdma_counter_alloc()
114 static void rdma_counter_free(struct rdma_counter *counter)  in rdma_counter_free()  argument
118 	port_counter = &counter->device->port_data[counter->port].port_counter;  in rdma_counter_free()
128 	rdma_restrack_del(&counter->res);  in rdma_counter_free()
129 	kfree(counter->stats);  in rdma_counter_free()
130 	kfree(counter);  in rdma_counter_free()
133 static void auto_mode_init_counter(struct rdma_counter *counter,  in auto_mode_init_counter()  argument
137 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_init_counter()
139 	counter->mode.mode = RDMA_COUNTER_MODE_AUTO;  in auto_mode_init_counter()
140 	counter->mode.mask = new_mask;  in auto_mode_init_counter()
146 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,  in auto_mode_match()  argument
149 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_match()
155 	/* Ensure that counter belongs to the right PID */  in auto_mode_match()
156 	if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task))  in auto_mode_match()
165 static int __rdma_counter_bind_qp(struct rdma_counter *counter,  in __rdma_counter_bind_qp()  argument
170 	if (qp->counter)  in __rdma_counter_bind_qp()
176 	mutex_lock(&counter->lock);  in __rdma_counter_bind_qp()
177 	ret = qp->device->ops.counter_bind_qp(counter, qp);  in __rdma_counter_bind_qp()
178 	mutex_unlock(&counter->lock);  in __rdma_counter_bind_qp()
185 	struct rdma_counter *counter = qp->counter;  in __rdma_counter_unbind_qp()  local
191 	mutex_lock(&counter->lock);  in __rdma_counter_unbind_qp()
193 	mutex_unlock(&counter->lock);  in __rdma_counter_unbind_qp()
198 static void counter_history_stat_update(const struct rdma_counter *counter)  in counter_history_stat_update()  argument
200 	struct ib_device *dev = counter->device;  in counter_history_stat_update()
204 	port_counter = &dev->port_data[counter->port].port_counter;  in counter_history_stat_update()
208 	for (i = 0; i < counter->stats->num_counters; i++)  in counter_history_stat_update()
209 		port_counter->hstats->value[i] += counter->stats->value[i];  in counter_history_stat_update()
213  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
216  * Return: The counter (with ref-count increased) if found
222 	struct rdma_counter *counter = NULL;  in rdma_get_counter_auto_mode()  local
235 		counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_auto_mode()
236 		if ((counter->device != qp->device) || (counter->port != port))  in rdma_get_counter_auto_mode()
239 		if (auto_mode_match(qp, counter, port_counter->mode.mask))  in rdma_get_counter_auto_mode()
242 		counter = NULL;  in rdma_get_counter_auto_mode()
245 	if (counter && !kref_get_unless_zero(&counter->kref))  in rdma_get_counter_auto_mode()
246 		counter = NULL;  in rdma_get_counter_auto_mode()
249 	return counter;  in rdma_get_counter_auto_mode()
252 static void rdma_counter_res_add(struct rdma_counter *counter,  in rdma_counter_res_add()  argument
256 		rdma_restrack_set_task(&counter->res, qp->res.kern_name);  in rdma_counter_res_add()
257 		rdma_restrack_kadd(&counter->res);  in rdma_counter_res_add()
259 		rdma_restrack_attach_task(&counter->res, qp->res.task);  in rdma_counter_res_add()
260 		rdma_restrack_uadd(&counter->res);  in rdma_counter_res_add()
266 	struct rdma_counter *counter;  in counter_release()  local
268 	counter = container_of(kref, struct rdma_counter, kref);  in counter_release()
269 	counter_history_stat_update(counter);  in counter_release()
270 	counter->device->ops.counter_dealloc(counter);  in counter_release()
271 	rdma_counter_free(counter);  in counter_release()
275  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
282 	struct rdma_counter *counter;  in rdma_counter_bind_qp_auto()  local
292 	counter = rdma_get_counter_auto_mode(qp, port);  in rdma_counter_bind_qp_auto()
293 	if (counter) {  in rdma_counter_bind_qp_auto()
294 		ret = __rdma_counter_bind_qp(counter, qp);  in rdma_counter_bind_qp_auto()
296 			kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qp_auto()
300 		counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_AUTO);  in rdma_counter_bind_qp_auto()
301 		if (!counter)  in rdma_counter_bind_qp_auto()
304 		auto_mode_init_counter(counter, qp, port_counter->mode.mask);  in rdma_counter_bind_qp_auto()
306 		ret = __rdma_counter_bind_qp(counter, qp);  in rdma_counter_bind_qp_auto()
308 			rdma_counter_free(counter);  in rdma_counter_bind_qp_auto()
312 		rdma_counter_res_add(counter, qp);  in rdma_counter_bind_qp_auto()
319  * rdma_counter_unbind_qp - Unbind a qp from a counter
321  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
325 	struct rdma_counter *counter = qp->counter;  in rdma_counter_unbind_qp()  local
328 	if (!counter)  in rdma_counter_unbind_qp()
335 	kref_put(&counter->kref, counter_release);  in rdma_counter_unbind_qp()
339 int rdma_counter_query_stats(struct rdma_counter *counter)  in rdma_counter_query_stats()  argument
341 	struct ib_device *dev = counter->device;  in rdma_counter_query_stats()
347 	mutex_lock(&counter->lock);  in rdma_counter_query_stats()
348 	ret = dev->ops.counter_update_stats(counter);  in rdma_counter_query_stats()
349 	mutex_unlock(&counter->lock);  in rdma_counter_query_stats()
359 	struct rdma_counter *counter;  in get_running_counters_hwstat_sum()  local
371 		counter = container_of(res, struct rdma_counter, res);  in get_running_counters_hwstat_sum()
372 		if ((counter->device != dev) || (counter->port != port) ||  in get_running_counters_hwstat_sum()
373 		    rdma_counter_query_stats(counter))  in get_running_counters_hwstat_sum()
376 		sum += counter->stats->value[index];  in get_running_counters_hwstat_sum()
429 static int rdma_counter_bind_qp_manual(struct rdma_counter *counter,  in rdma_counter_bind_qp_manual()  argument
432 	if ((counter->device != qp->device) || (counter->port != qp->port))  in rdma_counter_bind_qp_manual()
435 	return __rdma_counter_bind_qp(counter, qp);  in rdma_counter_bind_qp_manual()
442 	struct rdma_counter *counter;  in rdma_get_counter_by_id()  local
453 	counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_by_id()
454 	kref_get(&counter->kref);  in rdma_get_counter_by_id()
457 	return counter;  in rdma_get_counter_by_id()
461  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
466 	struct rdma_counter *counter;  in rdma_counter_bind_qpn()  local
474 	counter = rdma_get_counter_by_id(dev, counter_id);  in rdma_counter_bind_qpn()
475 	if (!counter) {  in rdma_counter_bind_qpn()
480 	if (counter->res.task != qp->res.task) {  in rdma_counter_bind_qpn()
485 	ret = rdma_counter_bind_qp_manual(counter, qp);  in rdma_counter_bind_qpn()
493 	kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qpn()
500  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
501  *   The id of new counter is returned in @counter_id
506 	struct rdma_counter *counter;  in rdma_counter_bind_qpn_alloc()  local
525 	counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_MANUAL);  in rdma_counter_bind_qpn_alloc()
526 	if (!counter) {  in rdma_counter_bind_qpn_alloc()
531 	ret = rdma_counter_bind_qp_manual(counter, qp);  in rdma_counter_bind_qpn_alloc()
536 		*counter_id = counter->id;  in rdma_counter_bind_qpn_alloc()
538 	rdma_counter_res_add(counter, qp);  in rdma_counter_bind_qpn_alloc()
544 	rdma_counter_free(counter);  in rdma_counter_bind_qpn_alloc()
551  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
573 	if (!qp->counter || qp->counter->id != counter_id ||  in rdma_counter_unbind_qpn()