1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3  * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.
12 static int get_pas_size(struct mlx5_srq_attr *in)  in get_pas_size()  argument
14 	u32 log_page_size = in->log_page_size + 12;  in get_pas_size()
15 	u32 log_srq_size  = in->log_size;  in get_pas_size()
16 	u32 log_rq_stride = in->wqe_shift;  in get_pas_size()
17 	u32 page_offset   = in->page_offset;  in get_pas_size()
18 	u32 po_quanta	  = 1 << (log_page_size - 6);  in get_pas_size()
27 static void set_wq(void *wq, struct mlx5_srq_attr *in)  in set_wq()  argument
29 	MLX5_SET(wq,   wq, wq_signature,  !!(in->flags  in set_wq()
31 	MLX5_SET(wq,   wq, log_wq_pg_sz,  in->log_page_size);  in set_wq()
32 	MLX5_SET(wq,   wq, log_wq_stride, in->wqe_shift + 4);  in set_wq()
33 	MLX5_SET(wq,   wq, log_wq_sz,     in->log_size);  in set_wq()
34 	MLX5_SET(wq,   wq, page_offset,   in->page_offset);  in set_wq()
35 	MLX5_SET(wq,   wq, lwm,		  in->lwm);  in set_wq()
36 	MLX5_SET(wq,   wq, pd,		  in->pd);  in set_wq()
37 	MLX5_SET64(wq, wq, dbr_addr,	  in->db_record);  in set_wq()
40 static void set_srqc(void *srqc, struct mlx5_srq_attr *in)  in set_srqc()  argument
42 	MLX5_SET(srqc,   srqc, wq_signature,  !!(in->flags  in set_srqc()
44 	MLX5_SET(srqc,   srqc, log_page_size, in->log_page_size);  in set_srqc()
45 	MLX5_SET(srqc,   srqc, log_rq_stride, in->wqe_shift);  in set_srqc()
46 	MLX5_SET(srqc,   srqc, log_srq_size,  in->log_size);  in set_srqc()
47 	MLX5_SET(srqc,   srqc, page_offset,   in->page_offset);  in set_srqc()
48 	MLX5_SET(srqc,	 srqc, lwm,	      in->lwm);  in set_srqc()
49 	MLX5_SET(srqc,	 srqc, pd,	      in->pd);  in set_srqc()
50 	MLX5_SET64(srqc, srqc, dbr_addr,      in->db_record);  in set_srqc()
51 	MLX5_SET(srqc,	 srqc, xrcd,	      in->xrcd);  in set_srqc()
52 	MLX5_SET(srqc,	 srqc, cqn,	      in->cqn);  in set_srqc()
55 static void get_wq(void *wq, struct mlx5_srq_attr *in)  in get_wq()  argument
58 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;  in get_wq()
59 	in->log_page_size = MLX5_GET(wq,   wq, log_wq_pg_sz);  in get_wq()
60 	in->wqe_shift	  = MLX5_GET(wq,   wq, log_wq_stride) - 4;  in get_wq()
61 	in->log_size	  = MLX5_GET(wq,   wq, log_wq_sz);  in get_wq()
62 	in->page_offset   = MLX5_GET(wq,   wq, page_offset);  in get_wq()
63 	in->lwm		  = MLX5_GET(wq,   wq, lwm);  in get_wq()
64 	in->pd		  = MLX5_GET(wq,   wq, pd);  in get_wq()
65 	in->db_record	  = MLX5_GET64(wq, wq, dbr_addr);  in get_wq()
68 static void get_srqc(void *srqc, struct mlx5_srq_attr *in)  in get_srqc()  argument
71 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;  in get_srqc()
72 	in->log_page_size = MLX5_GET(srqc,   srqc, log_page_size);  in get_srqc()
73 	in->wqe_shift	  = MLX5_GET(srqc,   srqc, log_rq_stride);  in get_srqc()
74 	in->log_size	  = MLX5_GET(srqc,   srqc, log_srq_size);  in get_srqc()
75 	in->page_offset   = MLX5_GET(srqc,   srqc, page_offset);  in get_srqc()
76 	in->lwm		  = MLX5_GET(srqc,   srqc, lwm);  in get_srqc()
77 	in->pd		  = MLX5_GET(srqc,   srqc, pd);  in get_srqc()
78 	in->db_record	  = MLX5_GET64(srqc, srqc, dbr_addr);  in get_srqc()
83 	struct mlx5_srq_table *table = &dev->srq_table;  in mlx5_cmd_get_srq()
86 	xa_lock_irq(&table->array);  in mlx5_cmd_get_srq()
87 	srq = xa_load(&table->array, srqn);  in mlx5_cmd_get_srq()
89 		refcount_inc(&srq->common.refcount);  in mlx5_cmd_get_srq()
90 	xa_unlock_irq(&table->array);  in mlx5_cmd_get_srq()
96 			  struct mlx5_srq_attr *in)  in create_srq_cmd()  argument
106 	pas_size  = get_pas_size(in);  in create_srq_cmd()
110 		return -ENOMEM;  in create_srq_cmd()
112 	MLX5_SET(create_srq_in, create_in, uid, in->uid);  in create_srq_cmd()
116 	set_srqc(srqc, in);  in create_srq_cmd()
117 	memcpy(pas, in->pas, pas_size);  in create_srq_cmd()
122 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,  in create_srq_cmd()
126 		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);  in create_srq_cmd()
127 		srq->uid = in->uid;  in create_srq_cmd()
135 	u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {};  in destroy_srq_cmd()  local
137 	MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ);  in destroy_srq_cmd()
138 	MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);  in destroy_srq_cmd()
139 	MLX5_SET(destroy_srq_in, in, uid, srq->uid);  in destroy_srq_cmd()
141 	return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in);  in destroy_srq_cmd()
147 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};  in arm_srq_cmd()  local
149 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);  in arm_srq_cmd()
150 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);  in arm_srq_cmd()
151 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);  in arm_srq_cmd()
152 	MLX5_SET(arm_rq_in, in, lwm, lwm);  in arm_srq_cmd()
153 	MLX5_SET(arm_rq_in, in, uid, srq->uid);  in arm_srq_cmd()
155 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);  in arm_srq_cmd()
161 	u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {};  in query_srq_cmd()  local
168 		return -ENOMEM;  in query_srq_cmd()
170 	MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ);  in query_srq_cmd()
171 	MLX5_SET(query_srq_in, in, srqn, srq->srqn);  in query_srq_cmd()
172 	err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out);  in query_srq_cmd()
179 		out->flags |= MLX5_SRQ_FLAG_ERR;  in query_srq_cmd()
187 			      struct mlx5_srq_attr *in)  in create_xrc_srq_cmd()  argument
197 	pas_size  = get_pas_size(in);  in create_xrc_srq_cmd()
201 		return -ENOMEM;  in create_xrc_srq_cmd()
203 	MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);  in create_xrc_srq_cmd()
208 	set_srqc(xrc_srqc, in);  in create_xrc_srq_cmd()
209 	MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);  in create_xrc_srq_cmd()
210 	memcpy(pas, in->pas, pas_size);  in create_xrc_srq_cmd()
215 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,  in create_xrc_srq_cmd()
220 	srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);  in create_xrc_srq_cmd()
221 	srq->uid = in->uid;  in create_xrc_srq_cmd()
230 	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {};  in destroy_xrc_srq_cmd()  local
232 	MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ);  in destroy_xrc_srq_cmd()
233 	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);  in destroy_xrc_srq_cmd()
234 	MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);  in destroy_xrc_srq_cmd()
236 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in);  in destroy_xrc_srq_cmd()
242 	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {};  in arm_xrc_srq_cmd()  local
244 	MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);  in arm_xrc_srq_cmd()
245 	MLX5_SET(arm_xrc_srq_in, in, op_mod,  in arm_xrc_srq_cmd()
247 	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);  in arm_xrc_srq_cmd()
248 	MLX5_SET(arm_xrc_srq_in, in, lwm, lwm);  in arm_xrc_srq_cmd()
249 	MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);  in arm_xrc_srq_cmd()
251 	return  mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in);  in arm_xrc_srq_cmd()
258 	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {};  in query_xrc_srq_cmd()  local
265 		return -ENOMEM;  in query_xrc_srq_cmd()
267 	MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);  in query_xrc_srq_cmd()
268 	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);  in query_xrc_srq_cmd()
270 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out);  in query_xrc_srq_cmd()
278 		out->flags |= MLX5_SRQ_FLAG_ERR;  in query_xrc_srq_cmd()
286 			  struct mlx5_srq_attr *in)  in create_rmp_cmd()  argument
297 	pas_size = get_pas_size(in);  in create_rmp_cmd()
303 		err = -ENOMEM;  in create_rmp_cmd()
311 	MLX5_SET(create_rmp_in, create_in, uid, in->uid);  in create_rmp_cmd()
312 	set_wq(wq, in);  in create_rmp_cmd()
313 	memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);  in create_rmp_cmd()
316 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);  in create_rmp_cmd()
318 		srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn);  in create_rmp_cmd()
319 		srq->uid = in->uid;  in create_rmp_cmd()
330 	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};  in destroy_rmp_cmd()  local
332 	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);  in destroy_rmp_cmd()
333 	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);  in destroy_rmp_cmd()
334 	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);  in destroy_rmp_cmd()
335 	return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in);  in destroy_rmp_cmd()
342 	void *in = NULL;  in arm_rmp_cmd()  local
353 	in = kvzalloc(inlen, GFP_KERNEL);  in arm_rmp_cmd()
355 	if (!in || !out) {  in arm_rmp_cmd()
356 		err = -ENOMEM;  in arm_rmp_cmd()
360 	rmpc =	  MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);  in arm_rmp_cmd()
361 	bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);  in arm_rmp_cmd()
364 	MLX5_SET(modify_rmp_in, in,	 rmp_state, MLX5_RMPC_STATE_RDY);  in arm_rmp_cmd()
365 	MLX5_SET(modify_rmp_in, in,	 rmpn,      srq->srqn);  in arm_rmp_cmd()
366 	MLX5_SET(modify_rmp_in, in, uid, srq->uid);  in arm_rmp_cmd()
370 	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);  in arm_rmp_cmd()
372 	err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out);  in arm_rmp_cmd()
375 	kvfree(in);  in arm_rmp_cmd()
396 		err = -ENOMEM;  in query_rmp_cmd()
401 	MLX5_SET(query_rmp_in, rmp_in, rmpn,   srq->srqn);  in query_rmp_cmd()
402 	err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out);  in query_rmp_cmd()
409 		out->flags |= MLX5_SRQ_FLAG_ERR;  in query_rmp_cmd()
418 			  struct mlx5_srq_attr *in)  in create_xrq_cmd()  argument
428 	pas_size = get_pas_size(in);  in create_xrq_cmd()
432 		return -ENOMEM;  in create_xrq_cmd()
437 	set_wq(wq, in);  in create_xrq_cmd()
438 	memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);  in create_xrq_cmd()
440 	if (in->type == IB_SRQT_TM) {  in create_xrq_cmd()
442 		if (in->flags & MLX5_SRQ_FLAG_RNDV)  in create_xrq_cmd()
446 			 in->tm_log_list_size);  in create_xrq_cmd()
448 	MLX5_SET(xrqc, xrqc, user_index, in->user_index);  in create_xrq_cmd()
449 	MLX5_SET(xrqc, xrqc, cqn, in->cqn);  in create_xrq_cmd()
451 	MLX5_SET(create_xrq_in, create_in, uid, in->uid);  in create_xrq_cmd()
452 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,  in create_xrq_cmd()
456 		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);  in create_xrq_cmd()
457 		srq->uid = in->uid;  in create_xrq_cmd()
465 	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {};  in destroy_xrq_cmd()  local
467 	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);  in destroy_xrq_cmd()
468 	MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);  in destroy_xrq_cmd()
469 	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);  in destroy_xrq_cmd()
471 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in);  in destroy_xrq_cmd()
478 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};  in arm_xrq_cmd()  local
480 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);  in arm_xrq_cmd()
481 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);  in arm_xrq_cmd()
482 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);  in arm_xrq_cmd()
483 	MLX5_SET(arm_rq_in, in, lwm, lwm);  in arm_xrq_cmd()
484 	MLX5_SET(arm_rq_in, in, uid, srq->uid);  in arm_xrq_cmd()
486 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);  in arm_xrq_cmd()
492 	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {};  in query_xrq_cmd()  local
500 		return -ENOMEM;  in query_xrq_cmd()
502 	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);  in query_xrq_cmd()
503 	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);  in query_xrq_cmd()
505 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out);  in query_xrq_cmd()
512 		out->flags |= MLX5_SRQ_FLAG_ERR;  in query_xrq_cmd()
513 	out->tm_next_tag =  in query_xrq_cmd()
516 	out->tm_hw_phase_cnt =  in query_xrq_cmd()
519 	out->tm_sw_phase_cnt =  in query_xrq_cmd()
529 			    struct mlx5_srq_attr *in)  in create_srq_split()  argument
531 	if (!dev->mdev->issi)  in create_srq_split()
532 		return create_srq_cmd(dev, srq, in);  in create_srq_split()
533 	switch (srq->common.res) {  in create_srq_split()
535 		return create_xrc_srq_cmd(dev, srq, in);  in create_srq_split()
537 		return create_xrq_cmd(dev, srq, in);  in create_srq_split()
539 		return create_rmp_cmd(dev, srq, in);  in create_srq_split()
545 	if (!dev->mdev->issi)  in destroy_srq_split()
547 	switch (srq->common.res) {  in destroy_srq_split()
558 			struct mlx5_srq_attr *in)  in mlx5_cmd_create_srq()  argument
560 	struct mlx5_srq_table *table = &dev->srq_table;  in mlx5_cmd_create_srq()
563 	switch (in->type) {  in mlx5_cmd_create_srq()
565 		srq->common.res = MLX5_RES_XSRQ;  in mlx5_cmd_create_srq()
568 		srq->common.res = MLX5_RES_XRQ;  in mlx5_cmd_create_srq()
571 		srq->common.res = MLX5_RES_SRQ;  in mlx5_cmd_create_srq()
574 	err = create_srq_split(dev, srq, in);  in mlx5_cmd_create_srq()
578 	refcount_set(&srq->common.refcount, 1);  in mlx5_cmd_create_srq()
579 	init_completion(&srq->common.free);  in mlx5_cmd_create_srq()
581 	err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));  in mlx5_cmd_create_srq()
595 	struct mlx5_srq_table *table = &dev->srq_table;  in mlx5_cmd_destroy_srq()
600 	tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0);  in mlx5_cmd_destroy_srq()
602 		return xa_err(tmp) ?: -EINVAL;  in mlx5_cmd_destroy_srq()
608 		 * because  we are storing in pre-allocated space xarray  in mlx5_cmd_destroy_srq()
611 		xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0);  in mlx5_cmd_destroy_srq()
614 	xa_erase_irq(&table->array, srq->srqn);  in mlx5_cmd_destroy_srq()
616 	mlx5_core_res_put(&srq->common);  in mlx5_cmd_destroy_srq()
617 	wait_for_completion(&srq->common.free);  in mlx5_cmd_destroy_srq()
624 	if (!dev->mdev->issi)  in mlx5_cmd_query_srq()
626 	switch (srq->common.res) {  in mlx5_cmd_query_srq()
639 	if (!dev->mdev->issi)  in mlx5_cmd_arm_srq()
641 	switch (srq->common.res) {  in mlx5_cmd_arm_srq()
666 	srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;  in srq_event_notifier()
668 	xa_lock(&table->array);  in srq_event_notifier()
669 	srq = xa_load(&table->array, srqn);  in srq_event_notifier()
671 		refcount_inc(&srq->common.refcount);  in srq_event_notifier()
672 	xa_unlock(&table->array);  in srq_event_notifier()
677 	srq->event(srq, eqe->type);  in srq_event_notifier()
679 	mlx5_core_res_put(&srq->common);  in srq_event_notifier()
686 	struct mlx5_srq_table *table = &dev->srq_table;  in mlx5_init_srq_table()
689 	xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ);  in mlx5_init_srq_table()
691 	table->nb.notifier_call = srq_event_notifier;  in mlx5_init_srq_table()
692 	mlx5_notifier_register(dev->mdev, &table->nb);  in mlx5_init_srq_table()
699 	struct mlx5_srq_table *table = &dev->srq_table;  in mlx5_cleanup_srq_table()
701 	mlx5_notifier_unregister(dev->mdev, &table->nb);  in mlx5_cleanup_srq_table()