76 		dev_dbg(&scq->hwq.pdev->dev,  in __bnxt_qplib_add_flush_qp()
84 			dev_dbg(&rcq->hwq.pdev->dev,  in __bnxt_qplib_add_flush_qp()
143 	qp->sq.hwq.prod = 0;  in bnxt_qplib_clean_qp()
144 	qp->sq.hwq.cons = 0;  in bnxt_qplib_clean_qp()
146 	qp->rq.hwq.prod = 0;  in bnxt_qplib_clean_qp()
147 	qp->rq.hwq.cons = 0;  in bnxt_qplib_clean_qp()
182 				  rq->hwq.max_elements * qp->rq_hdr_buf_size,  in bnxt_qplib_free_qp_hdr_buf()
186 				  sq->hwq.max_elements * qp->sq_hdr_buf_size,  in bnxt_qplib_free_qp_hdr_buf()
203 	if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {  in bnxt_qplib_alloc_qp_hdr_buf()
205 					sq->hwq.max_elements *  in bnxt_qplib_alloc_qp_hdr_buf()
216 	if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {  in bnxt_qplib_alloc_qp_hdr_buf()
218 						    rq->hwq.max_elements *  in bnxt_qplib_alloc_qp_hdr_buf()
239 	struct bnxt_qplib_hwq *hwq = &nq->hwq;  in bnxt_qplib_service_nq()  local
251 	raw_cons = hwq->cons;  in bnxt_qplib_service_nq()
253 		sw_cons = HWQ_CMP(raw_cons, hwq);  in bnxt_qplib_service_nq()
254 		nq_ptr = (struct nq_base **)hwq->pbl_ptr;  in bnxt_qplib_service_nq()
256 		if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))  in bnxt_qplib_service_nq()
315 	if (hwq->cons != raw_cons) {  in bnxt_qplib_service_nq()
316 		hwq->cons = raw_cons;  in bnxt_qplib_service_nq()
317 		bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, hwq->cons,  in bnxt_qplib_service_nq()
318 					    hwq->max_elements, nq->ring_id,  in bnxt_qplib_service_nq()
326 	struct bnxt_qplib_hwq *hwq = &nq->hwq;  in bnxt_qplib_nq_irq()  local
331 	sw_cons = HWQ_CMP(hwq->cons, hwq);  in bnxt_qplib_nq_irq()
332 	nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;  in bnxt_qplib_nq_irq()
346 	bnxt_qplib_ring_nq_db(nq->bar_reg_iomem, nq->hwq.cons,  in bnxt_qplib_nq_stop_irq()
347 			      nq->hwq.max_elements, nq->ring_id, gen_p5);  in bnxt_qplib_nq_stop_irq()
409 	bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, nq->hwq.cons,  in bnxt_qplib_nq_start_irq()
410 				    nq->hwq.max_elements, nq->ring_id, gen_p5);  in bnxt_qplib_nq_start_irq()
466 	if (nq->hwq.max_elements) {  in bnxt_qplib_free_nq()
467 		bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);  in bnxt_qplib_free_nq()
468 		nq->hwq.max_elements = 0;  in bnxt_qplib_free_nq()
477 	if (!nq->hwq.max_elements ||  in bnxt_qplib_alloc_nq()
478 	    nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)  in bnxt_qplib_alloc_nq()
479 		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;  in bnxt_qplib_alloc_nq()
481 	if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL,  in bnxt_qplib_alloc_nq()
482 				      &nq->hwq.max_elements,  in bnxt_qplib_alloc_nq()
494 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;  in bnxt_qplib_arm_srq()
529 	bnxt_qplib_free_hwq(res->pdev, &srq->hwq);  in bnxt_qplib_destroy_srq()
542 	srq->hwq.max_elements = srq->max_wqe;  in bnxt_qplib_create_srq()
543 	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, &srq->sg_info,  in bnxt_qplib_create_srq()
544 				       &srq->hwq.max_elements,  in bnxt_qplib_create_srq()
550 	srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),  in bnxt_qplib_create_srq()
563 	req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);  in bnxt_qplib_create_srq()
564 	pbl = &srq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_srq()
565 	req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &  in bnxt_qplib_create_srq()
592 	srq->last_idx = srq->hwq.max_elements - 1;  in bnxt_qplib_create_srq()
593 	for (idx = 0; idx < srq->hwq.max_elements; idx++)  in bnxt_qplib_create_srq()
605 	bnxt_qplib_free_hwq(res->pdev, &srq->hwq);  in bnxt_qplib_create_srq()
614 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;  in bnxt_qplib_modify_srq()
663 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;  in bnxt_qplib_post_srq_recv()
742 	sq->hwq.max_elements = sq->max_wqe;  in bnxt_qplib_create_qp1()
743 	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL,  in bnxt_qplib_create_qp1()
744 				       &sq->hwq.max_elements,  in bnxt_qplib_create_qp1()
750 	sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);  in bnxt_qplib_create_qp1()
755 	pbl = &sq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_qp1()
758 		((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)  in bnxt_qplib_create_qp1()
781 		rq->hwq.max_elements = qp->rq.max_wqe;  in bnxt_qplib_create_qp1()
782 		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL,  in bnxt_qplib_create_qp1()
783 					       &rq->hwq.max_elements,  in bnxt_qplib_create_qp1()
789 		rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),  in bnxt_qplib_create_qp1()
795 		pbl = &rq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_qp1()
798 			((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<  in bnxt_qplib_create_qp1()
824 	req.sq_size = cpu_to_le32(sq->hwq.max_elements);  in bnxt_qplib_create_qp1()
825 	req.rq_size = cpu_to_le32(rq->hwq.max_elements);  in bnxt_qplib_create_qp1()
851 	bnxt_qplib_free_hwq(res->pdev, &rq->hwq);  in bnxt_qplib_create_qp1()
854 	bnxt_qplib_free_hwq(res->pdev, &sq->hwq);  in bnxt_qplib_create_qp1()
890 	sq->hwq.max_elements = sq->max_wqe;  in bnxt_qplib_create_qp()
891 	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, &sq->sg_info,  in bnxt_qplib_create_qp()
892 				       &sq->hwq.max_elements,  in bnxt_qplib_create_qp()
899 	sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);  in bnxt_qplib_create_qp()
904 	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;  in bnxt_qplib_create_qp()
908 					(sq->hwq.max_elements)];  in bnxt_qplib_create_qp()
910 			      &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]  in bnxt_qplib_create_qp()
911 			      [get_sqe_idx(sq->hwq.max_elements)];  in bnxt_qplib_create_qp()
919 		for (i = 0; i < sq->hwq.max_elements; i++) {  in bnxt_qplib_create_qp()
930 	pbl = &sq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_qp()
933 		((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)  in bnxt_qplib_create_qp()
959 		rq->hwq.max_elements = rq->max_wqe;  in bnxt_qplib_create_qp()
960 		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq,  in bnxt_qplib_create_qp()
962 					       &rq->hwq.max_elements,  in bnxt_qplib_create_qp()
968 		rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),  in bnxt_qplib_create_qp()
974 		pbl = &rq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_qp()
977 			((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<  in bnxt_qplib_create_qp()
1003 	req.sq_size = cpu_to_le32(sq->hwq.max_elements);  in bnxt_qplib_create_qp()
1004 	req.rq_size = cpu_to_le32(rq->hwq.max_elements);  in bnxt_qplib_create_qp()
1084 	bnxt_qplib_free_hwq(res->pdev, &rq->hwq);  in bnxt_qplib_create_qp()
1087 	bnxt_qplib_free_hwq(res->pdev, &sq->hwq);  in bnxt_qplib_create_qp()
1270 	req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);  in bnxt_qplib_modify_qp()
1271 	req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);  in bnxt_qplib_modify_qp()
1358 	qp->sq.max_wqe = qp->sq.hwq.max_elements;  in bnxt_qplib_query_qp()
1359 	qp->rq.max_wqe = qp->rq.hwq.max_elements;  in bnxt_qplib_query_qp()
1373 	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;  in __clean_cq()
1443 	bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);  in bnxt_qplib_free_qp_res()
1446 	bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);  in bnxt_qplib_free_qp_res()
1465 		sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);  in bnxt_qplib_get_qp1_sq_buf()
1479 	return HWQ_CMP(rq->hwq.prod, &rq->hwq);  in bnxt_qplib_get_rq_prod_index()
1496 		sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);  in bnxt_qplib_get_qp1_rq_buf()
1515 	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);  in bnxt_qplib_post_send_db()
1538 			dev_dbg(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
1546 		dev_err(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
1548 			sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,  in bnxt_qplib_post_send()
1553 	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);  in bnxt_qplib_post_send()
1562 	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;  in bnxt_qplib_post_send()
1571 			dev_warn(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
1778 		sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);  in bnxt_qplib_post_send()
1787 	sq->hwq.prod++;  in bnxt_qplib_post_send()
1799 			dev_err(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
1816 	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);  in bnxt_qplib_post_recv_db()
1835 		dev_dbg(&rq->hwq.pdev->dev,  in bnxt_qplib_post_recv()
1840 		dev_err(&rq->hwq.pdev->dev,  in bnxt_qplib_post_recv()
1845 	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);  in bnxt_qplib_post_recv()
1848 	rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;  in bnxt_qplib_post_recv()
1876 		sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);  in bnxt_qplib_post_recv()
1880 	rq->hwq.prod++;  in bnxt_qplib_post_recv()
1889 			dev_err(&rq->hwq.pdev->dev,  in bnxt_qplib_post_recv()
1914 	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;  in bnxt_qplib_arm_cq()
1936 	cq->hwq.max_elements = cq->max_wqe;  in bnxt_qplib_create_cq()
1937 	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, &cq->sg_info,  in bnxt_qplib_create_cq()
1938 				       &cq->hwq.max_elements,  in bnxt_qplib_create_cq()
1954 	req.cq_size = cpu_to_le32(cq->hwq.max_elements);  in bnxt_qplib_create_cq()
1955 	pbl = &cq->hwq.pbl[PBL_LVL_0];  in bnxt_qplib_create_cq()
1957 	    ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<  in bnxt_qplib_create_cq()
1991 	bnxt_qplib_free_hwq(res->pdev, &cq->hwq);  in bnxt_qplib_create_cq()
2011 	bnxt_qplib_free_hwq(res->pdev, &cq->hwq);  in bnxt_qplib_destroy_cq()
2023 	sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);  in __flush_sq()
2026 		sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);  in __flush_sq()
2045 		sq->hwq.cons++;  in __flush_sq()
2048 	if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)  in __flush_sq()
2077 	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);  in __flush_rq()
2080 		sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);  in __flush_rq()
2091 		rq->hwq.cons++;  in __flush_rq()
2094 	if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)  in __flush_rq()
2137 		dev_dbg(&cq->hwq.pdev->dev,  in do_wa9060()
2151 		peek_raw_cq_cons = cq->hwq.cons;  in do_wa9060()
2153 		i = cq->hwq.max_elements;  in do_wa9060()
2155 			peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);  in do_wa9060()
2156 			peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;  in do_wa9060()
2161 					  cq->hwq.max_elements)) {  in do_wa9060()
2180 						, &sq->hwq);  in do_wa9060()
2189 						dev_dbg(&cq->hwq.pdev->dev,  in do_wa9060()
2206 		dev_err(&cq->hwq.pdev->dev,  in do_wa9060()
2230 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2236 	cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);  in bnxt_qplib_cq_process_req()
2237 	if (cqe_sq_cons > sq->hwq.max_elements) {  in bnxt_qplib_cq_process_req()
2238 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2240 			cqe_sq_cons, sq->hwq.max_elements);  in bnxt_qplib_cq_process_req()
2245 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2255 		sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);  in bnxt_qplib_cq_process_req()
2274 		if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&  in bnxt_qplib_cq_process_req()
2277 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2299 		sq->hwq.cons++;  in bnxt_qplib_cq_process_req()
2305 	if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {  in bnxt_qplib_cq_process_req()
2321 	spin_lock(&srq->hwq.lock);  in bnxt_qplib_release_srqe()
2325 	srq->hwq.cons++; /* Support for SRQE counter */  in bnxt_qplib_release_srqe()
2326 	spin_unlock(&srq->hwq.lock);  in bnxt_qplib_release_srqe()
2344 		dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n");  in bnxt_qplib_cq_process_res_rc()
2348 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
2368 		if (wr_id_idx >= srq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_rc()
2369 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
2371 				wr_id_idx, srq->hwq.max_elements);  in bnxt_qplib_cq_process_res_rc()
2381 		if (wr_id_idx >= rq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_rc()
2382 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
2384 				wr_id_idx, rq->hwq.max_elements);  in bnxt_qplib_cq_process_res_rc()
2390 		rq->hwq.cons++;  in bnxt_qplib_cq_process_res_rc()
2419 		dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n");  in bnxt_qplib_cq_process_res_ud()
2423 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
2449 		if (wr_id_idx >= srq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_ud()
2450 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
2452 				wr_id_idx, srq->hwq.max_elements);  in bnxt_qplib_cq_process_res_ud()
2462 		if (wr_id_idx >= rq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_ud()
2463 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
2465 				wr_id_idx, rq->hwq.max_elements);  in bnxt_qplib_cq_process_res_ud()
2472 		rq->hwq.cons++;  in bnxt_qplib_cq_process_res_ud()
2491 	raw_cons = cq->hwq.cons;  in bnxt_qplib_is_cq_empty()
2492 	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);  in bnxt_qplib_is_cq_empty()
2493 	hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;  in bnxt_qplib_is_cq_empty()
2497 	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);  in bnxt_qplib_is_cq_empty()
2516 		dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n");  in bnxt_qplib_cq_process_res_raweth_qp1()
2520 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
2549 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
2553 		if (wr_id_idx >= srq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_raweth_qp1()
2554 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
2556 				wr_id_idx, srq->hwq.max_elements);  in bnxt_qplib_cq_process_res_raweth_qp1()
2566 		if (wr_id_idx >= rq->hwq.max_elements) {  in bnxt_qplib_cq_process_res_raweth_qp1()
2567 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
2569 				wr_id_idx, rq->hwq.max_elements);  in bnxt_qplib_cq_process_res_raweth_qp1()
2575 		rq->hwq.cons++;  in bnxt_qplib_cq_process_res_raweth_qp1()
2602 		dev_warn(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2609 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2624 	if (cqe_cons > sq->hwq.max_elements) {  in bnxt_qplib_cq_process_terminal()
2625 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2627 			cqe_cons, sq->hwq.max_elements);  in bnxt_qplib_cq_process_terminal()
2632 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2643 		sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);  in bnxt_qplib_cq_process_terminal()
2657 		sq->hwq.cons++;  in bnxt_qplib_cq_process_terminal()
2672 	} else if (cqe_cons > rq->hwq.max_elements) {  in bnxt_qplib_cq_process_terminal()
2673 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2675 			cqe_cons, rq->hwq.max_elements);  in bnxt_qplib_cq_process_terminal()
2680 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
2702 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_cutoff()
2723 		dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp);  in bnxt_qplib_process_flush_list()
2728 		dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp);  in bnxt_qplib_process_flush_list()
2743 	raw_cons = cq->hwq.cons;  in bnxt_qplib_poll_cq()
2747 		sw_cons = HWQ_CMP(raw_cons, &cq->hwq);  in bnxt_qplib_poll_cq()
2748 		hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;  in bnxt_qplib_poll_cq()
2752 		if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))  in bnxt_qplib_poll_cq()
2795 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_poll_cq()
2808 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_poll_cq()
2813 	if (cq->hwq.cons != raw_cons) {  in bnxt_qplib_poll_cq()
2814 		cq->hwq.cons = raw_cons;  in bnxt_qplib_poll_cq()