62 	u32 wqe_idx;  in irdma_nop_1()  local
68 	wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);  in irdma_nop_1()
69 	wqe = qp->sq_base[wqe_idx].elem;  in irdma_nop_1()
71 	qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;  in irdma_nop_1()
97 	u32 wqe_idx;  in irdma_clr_wqes()  local
100 		wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;  in irdma_clr_wqes()
101 		wqe = qp->sq_base[wqe_idx].elem;  in irdma_clr_wqes()
102 		if (wqe_idx)  in irdma_clr_wqes()
152 static void irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx)  in irdma_qp_ring_push_db()  argument
155 		      FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);  in irdma_qp_ring_push_db()
162 		       u32 wqe_idx, bool post_sq)  in irdma_qp_push_wqe()  argument
173 				  (wqe_idx & 0x7) * 0x20);  in irdma_qp_push_wqe()
175 		irdma_qp_ring_push_db(qp, wqe_idx);  in irdma_qp_push_wqe()
187 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,  in irdma_qp_get_next_send_wqe()  argument
220 	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);  in irdma_qp_get_next_send_wqe()
221 	if (!*wqe_idx)  in irdma_qp_get_next_send_wqe()
226 	wqe = qp->sq_base[*wqe_idx].elem;  in irdma_qp_get_next_send_wqe()
232 	qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;  in irdma_qp_get_next_send_wqe()
233 	qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;  in irdma_qp_get_next_send_wqe()
234 	qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;  in irdma_qp_get_next_send_wqe()
244 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)  in irdma_qp_get_next_recv_wqe()  argument
252 	IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);  in irdma_qp_get_next_recv_wqe()
256 	if (!*wqe_idx)  in irdma_qp_get_next_recv_wqe()
259 	wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;  in irdma_qp_get_next_recv_wqe()
276 	u32 i, wqe_idx;  in irdma_uk_rdma_write()  local
303 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,  in irdma_uk_rdma_write()
308 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_rdma_write()
355 		irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);  in irdma_uk_rdma_write()
380 	u32 wqe_idx;  in irdma_uk_rdma_read()  local
397 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,  in irdma_uk_rdma_read()
402 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_rdma_read()
442 		irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);  in irdma_uk_rdma_read()
463 	u32 i, wqe_idx, total_size = 0, byte_off;  in irdma_uk_send()  local
486 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,  in irdma_uk_send()
491 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_send()
543 		irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);  in irdma_uk_send()
684 	u32 wqe_idx;  in irdma_uk_inline_rdma_write()  local
695 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,  in irdma_uk_inline_rdma_write()
700 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_inline_rdma_write()
729 		irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);  in irdma_uk_inline_rdma_write()
750 	u32 wqe_idx;  in irdma_uk_inline_send()  local
761 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,  in irdma_uk_inline_send()
766 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_inline_send()
800 		irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);  in irdma_uk_inline_send()
822 	u32 wqe_idx;  in irdma_uk_stag_local_invalidate()  local
830 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,  in irdma_uk_stag_local_invalidate()
835 	irdma_clr_wqes(qp, wqe_idx);  in irdma_uk_stag_local_invalidate()
854 		irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,  in irdma_uk_stag_local_invalidate()
872 	u32 wqe_idx, i, byte_off;  in irdma_uk_post_receive()  local
880 	wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);  in irdma_uk_post_receive()
884 	qp->rq_wrid_array[wqe_idx] = info->wr_id;  in irdma_uk_post_receive()
1005 	u32 wqe_idx, q_type;  in irdma_uk_cq_poll_cmpl()  local
1114 	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);  in irdma_uk_cq_poll_cmpl()
1120 		array_idx = wqe_idx / qp->rq_wqe_size_multiplier;  in irdma_uk_cq_poll_cmpl()
1158 			if (wqe_idx + 1 >= qp->conn_wqes)  in irdma_uk_cq_poll_cmpl()
1161 			if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {  in irdma_uk_cq_poll_cmpl()
1177 			info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;  in irdma_uk_cq_poll_cmpl()
1179 				info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;  in irdma_uk_cq_poll_cmpl()
1182 					    wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);  in irdma_uk_cq_poll_cmpl()
1505 	u32 wqe_idx;  in irdma_nop()  local
1510 	wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,  in irdma_nop()
1515 	irdma_clr_wqes(qp, wqe_idx);  in irdma_nop()