2503 	struct irdma_sc_ceq *ceq;  in irdma_sc_cq_create()  local
2513 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_create()
2514 	if (ceq && ceq->reg_cq)  in irdma_sc_cq_create()
2515 		ret_code = irdma_sc_add_cq_ctx(ceq, cq);  in irdma_sc_cq_create()
2522 		if (ceq && ceq->reg_cq)  in irdma_sc_cq_create()
2523 			irdma_sc_remove_cq_ctx(ceq, cq);  in irdma_sc_cq_create()
2575 	struct irdma_sc_ceq *ceq;  in irdma_sc_cq_destroy()  local
2582 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_destroy()
2583 	if (ceq && ceq->reg_cq)  in irdma_sc_cq_destroy()
2584 		irdma_sc_remove_cq_ctx(ceq, cq);  in irdma_sc_cq_destroy()
3002 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,  in irdma_sc_find_reg_cq()  argument
3007 	for (i = 0; i < ceq->reg_cq_size; i++) {  in irdma_sc_find_reg_cq()
3008 		if (cq == ceq->reg_cq[i])  in irdma_sc_find_reg_cq()
3020 int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)  in irdma_sc_add_cq_ctx()  argument
3024 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3026 	if (ceq->reg_cq_size == ceq->elem_cnt) {  in irdma_sc_add_cq_ctx()
3027 		spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3031 	ceq->reg_cq[ceq->reg_cq_size++] = cq;  in irdma_sc_add_cq_ctx()
3033 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3043 void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)  in irdma_sc_remove_cq_ctx()  argument
3048 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3049 	cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);  in irdma_sc_remove_cq_ctx()
3053 	ceq->reg_cq_size--;  in irdma_sc_remove_cq_ctx()
3054 	if (cq_ctx_idx != ceq->reg_cq_size)  in irdma_sc_remove_cq_ctx()
3055 		ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];  in irdma_sc_remove_cq_ctx()
3056 	ceq->reg_cq[ceq->reg_cq_size] = NULL;  in irdma_sc_remove_cq_ctx()
3059 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3596 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,  in irdma_sc_ceq_init()  argument
3612 	ceq->size = sizeof(*ceq);  in irdma_sc_ceq_init()
3613 	ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;  in irdma_sc_ceq_init()
3614 	ceq->ceq_id = info->ceq_id;  in irdma_sc_ceq_init()
3615 	ceq->dev = info->dev;  in irdma_sc_ceq_init()
3616 	ceq->elem_cnt = info->elem_cnt;  in irdma_sc_ceq_init()
3617 	ceq->ceq_elem_pa = info->ceqe_pa;  in irdma_sc_ceq_init()
3618 	ceq->virtual_map = info->virtual_map;  in irdma_sc_ceq_init()
3619 	ceq->itr_no_expire = info->itr_no_expire;  in irdma_sc_ceq_init()
3620 	ceq->reg_cq = info->reg_cq;  in irdma_sc_ceq_init()
3621 	ceq->reg_cq_size = 0;  in irdma_sc_ceq_init()
3622 	spin_lock_init(&ceq->req_cq_lock);  in irdma_sc_ceq_init()
3623 	ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);  in irdma_sc_ceq_init()
3624 	ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);  in irdma_sc_ceq_init()
3625 	ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);  in irdma_sc_ceq_init()
3626 	ceq->tph_en = info->tph_en;  in irdma_sc_ceq_init()
3627 	ceq->tph_val = info->tph_val;  in irdma_sc_ceq_init()
3628 	ceq->vsi = info->vsi;  in irdma_sc_ceq_init()
3629 	ceq->polarity = 1;  in irdma_sc_ceq_init()
3630 	IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);  in irdma_sc_ceq_init()
3631 	ceq->dev->ceq[info->ceq_id] = ceq;  in irdma_sc_ceq_init()
3643 static int irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,  in irdma_sc_ceq_create()  argument
3650 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_create()
3654 	set_64bit_val(wqe, 16, ceq->elem_cnt);  in irdma_sc_ceq_create()
3656 		      (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));  in irdma_sc_ceq_create()
3658 		      (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));  in irdma_sc_ceq_create()
3660 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |  in irdma_sc_ceq_create()
3661 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));  in irdma_sc_ceq_create()
3662 	hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |  in irdma_sc_ceq_create()
3664 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_create()
3665 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_create()
3666 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |  in irdma_sc_ceq_create()
3667 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_create()
3685 static int irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)  in irdma_sc_cceq_create_done()  argument
3689 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_create_done()
3698 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)  in irdma_sc_cceq_destroy_done()  argument
3702 	if (ceq->reg_cq)  in irdma_sc_cceq_destroy_done()
3703 		irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_destroy_done()
3705 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_destroy_done()
3717 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)  in irdma_sc_cceq_create()  argument
3720 	struct irdma_sc_dev *dev = ceq->dev;  in irdma_sc_cceq_create()
3722 	dev->ccq->vsi = ceq->vsi;  in irdma_sc_cceq_create()
3723 	if (ceq->reg_cq) {  in irdma_sc_cceq_create()
3724 		ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_create()
3729 	ret_code = irdma_sc_ceq_create(ceq, scratch, true);  in irdma_sc_cceq_create()
3731 		return irdma_sc_cceq_create_done(ceq);  in irdma_sc_cceq_create()
3742 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq)  in irdma_sc_ceq_destroy()  argument
3748 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_destroy()
3753 	set_64bit_val(wqe, 16, ceq->elem_cnt);  in irdma_sc_ceq_destroy()
3754 	set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);  in irdma_sc_ceq_destroy()
3755 	hdr = ceq->ceq_id |  in irdma_sc_ceq_destroy()
3757 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_destroy()
3758 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_destroy()
3759 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_destroy()
3781 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)  in irdma_sc_process_ceq()  argument
3793 		ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);  in irdma_sc_process_ceq()
3796 		if (polarity != ceq->polarity)  in irdma_sc_process_ceq()
3802 			IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3804 			if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3805 				ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3810 		if (ceq->reg_cq) {  in irdma_sc_process_ceq()
3811 			spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3812 			cq_idx = irdma_sc_find_reg_cq(ceq, cq);  in irdma_sc_process_ceq()
3813 			spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3816 		IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3817 		if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3818 			ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3835 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)  in irdma_sc_cleanup_ceqes()  argument
3838 	u8 ceq_polarity = ceq->polarity;  in irdma_sc_cleanup_ceqes()
3845 	next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);  in irdma_sc_cleanup_ceqes()
3847 	for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) {  in irdma_sc_cleanup_ceqes()
3848 		ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);  in irdma_sc_cleanup_ceqes()
3859 		next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);  in irdma_sc_cleanup_ceqes()
4968 		status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,  in irdma_exec_cqp_cmd()
4979 		status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,  in irdma_exec_cqp_cmd()