Lines Matching refs:hwq
75 dev_dbg(&scq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
83 dev_dbg(&rcq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
142 qp->sq.hwq.prod = 0; in bnxt_qplib_clean_qp()
143 qp->sq.hwq.cons = 0; in bnxt_qplib_clean_qp()
145 qp->rq.hwq.prod = 0; in bnxt_qplib_clean_qp()
146 qp->rq.hwq.cons = 0; in bnxt_qplib_clean_qp()
236 struct bnxt_qplib_hwq *hwq = &nq->hwq; in clean_nq() local
243 spin_lock_bh(&hwq->lock); in clean_nq()
245 raw_cons = hwq->cons; in clean_nq()
247 sw_cons = HWQ_CMP(raw_cons, hwq); in clean_nq()
248 nq_ptr = (struct nq_base **)hwq->pbl_ptr; in clean_nq()
250 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) in clean_nq()
280 spin_unlock_bh(&hwq->lock); in clean_nq()
301 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_service_nq() local
311 spin_lock_bh(&hwq->lock); in bnxt_qplib_service_nq()
313 raw_cons = hwq->cons; in bnxt_qplib_service_nq()
315 sw_cons = HWQ_CMP(raw_cons, hwq); in bnxt_qplib_service_nq()
316 nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL); in bnxt_qplib_service_nq()
317 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) in bnxt_qplib_service_nq()
382 if (hwq->cons != raw_cons) { in bnxt_qplib_service_nq()
383 hwq->cons = raw_cons; in bnxt_qplib_service_nq()
386 spin_unlock_bh(&hwq->lock); in bnxt_qplib_service_nq()
392 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_nq_irq() local
396 sw_cons = HWQ_CMP(hwq->cons, hwq); in bnxt_qplib_nq_irq()
397 prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL)); in bnxt_qplib_nq_irq()
505 nq_db->dbinfo.hwq = &nq->hwq; in bnxt_qplib_map_nq_db()
546 if (nq->hwq.max_elements) { in bnxt_qplib_free_nq()
547 bnxt_qplib_free_hwq(nq->res, &nq->hwq); in bnxt_qplib_free_nq()
548 nq->hwq.max_elements = 0; in bnxt_qplib_free_nq()
559 if (!nq->hwq.max_elements || in bnxt_qplib_alloc_nq()
560 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) in bnxt_qplib_alloc_nq()
561 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; in bnxt_qplib_alloc_nq()
567 hwq_attr.depth = nq->hwq.max_elements; in bnxt_qplib_alloc_nq()
570 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { in bnxt_qplib_alloc_nq()
598 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_destroy_srq()
618 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); in bnxt_qplib_create_srq()
622 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), in bnxt_qplib_create_srq()
635 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); in bnxt_qplib_create_srq()
636 pbl = &srq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_srq()
637 pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) << in bnxt_qplib_create_srq()
639 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) << in bnxt_qplib_create_srq()
653 srq->last_idx = srq->hwq.max_elements - 1; in bnxt_qplib_create_srq()
654 for (idx = 0; idx < srq->hwq.max_elements; idx++) in bnxt_qplib_create_srq()
659 srq->dbinfo.hwq = &srq->hwq; in bnxt_qplib_create_srq()
670 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_create_srq()
679 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_modify_srq()
728 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_post_srq_recv()
836 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
845 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
847 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp1()
849 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); in bnxt_qplib_create_qp1()
863 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
870 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
872 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp1()
874 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); in bnxt_qplib_create_qp1()
900 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp1()
905 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp1()
921 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp1()
925 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp1()
932 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_init_psn_ptr() local
938 hwq = &sq->hwq; in bnxt_qplib_init_psn_ptr()
940 fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg); in bnxt_qplib_init_psn_ptr()
943 hwq->pad_pgofft = indx_pad; in bnxt_qplib_init_psn_ptr()
944 hwq->pad_pg = (u64 *)psn_pg; in bnxt_qplib_init_psn_ptr()
945 hwq->pad_stride = size; in bnxt_qplib_init_psn_ptr()
987 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
999 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1001 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp()
1003 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); in bnxt_qplib_create_qp()
1019 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
1027 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1029 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp()
1031 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); in bnxt_qplib_create_qp()
1106 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp()
1111 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp()
1128 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp()
1132 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp()
1314 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); in bnxt_qplib_modify_qp()
1315 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); in bnxt_qplib_modify_qp()
1402 qp->sq.max_wqe = qp->sq.hwq.max_elements; in bnxt_qplib_query_qp()
1403 qp->rq.max_wqe = qp->rq.hwq.max_elements; in bnxt_qplib_query_qp()
1417 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; in __clean_cq()
1488 bnxt_qplib_free_hwq(res, &qp->sq.hwq); in bnxt_qplib_free_qp_res()
1491 bnxt_qplib_free_hwq(res, &qp->rq.hwq); in bnxt_qplib_free_qp_res()
1586 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_put_inline() local
1594 hwq = &qp->sq.hwq; in bnxt_qplib_put_inline()
1605 il_dst = bnxt_qplib_get_prod_qe(hwq, *idx); in bnxt_qplib_put_inline()
1629 static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq, in bnxt_qplib_put_sges() argument
1637 dsge = bnxt_qplib_get_prod_qe(hwq, *idx); in bnxt_qplib_put_sges()
1675 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_pull_psn_buff() local
1680 hwq = &sq->hwq; in bnxt_qplib_pull_psn_buff()
1681 if (!hwq->pad_pg) in bnxt_qplib_pull_psn_buff()
1684 pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1685 pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1686 buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1704 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_post_send() local
1715 hwq = &sq->hwq; in bnxt_qplib_post_send()
1718 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1727 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1729 hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta); in bnxt_qplib_post_send()
1738 swq->slot_idx = hwq->prod; in bnxt_qplib_post_send()
1749 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_send()
1754 base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_send()
1755 ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_send()
1763 data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, in bnxt_qplib_post_send()
1924 bnxt_qplib_hwq_incr_prod(hwq, swq->slots); in bnxt_qplib_post_send()
1935 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1957 struct bnxt_qplib_hwq *hwq; in bnxt_qplib_post_recv() local
1964 hwq = &rq->hwq; in bnxt_qplib_post_recv()
1966 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1974 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1986 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_recv()
1992 base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
1993 ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
1998 bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx); in bnxt_qplib_post_recv()
2002 sge = bnxt_qplib_get_prod_qe(hwq, idx++); in bnxt_qplib_post_recv()
2012 bnxt_qplib_hwq_incr_prod(hwq, swq->slots); in bnxt_qplib_post_recv()
2022 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2048 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); in bnxt_qplib_create_cq()
2061 req.cq_size = cpu_to_le32(cq->hwq.max_elements); in bnxt_qplib_create_cq()
2062 pbl = &cq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_cq()
2063 pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) << in bnxt_qplib_create_cq()
2065 pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK); in bnxt_qplib_create_cq()
2085 cq->dbinfo.hwq = &cq->hwq; in bnxt_qplib_create_cq()
2095 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_create_cq()
2118 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_destroy_cq()
2151 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots); in __flush_sq()
2198 bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots); in __flush_rq()
2245 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2258 peek_raw_cq_cons = cq->hwq.cons; in do_wa9060()
2260 i = cq->hwq.max_elements; in do_wa9060()
2262 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); in do_wa9060()
2263 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, in do_wa9060()
2267 cq->hwq.max_elements)) { in do_wa9060()
2296 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2313 dev_err(&cq->hwq.pdev->dev, in do_wa9060()
2337 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2345 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2376 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2398 bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots); in bnxt_qplib_cq_process_req()
2421 spin_lock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2425 srq->hwq.cons++; /* Support for SRQE counter */ in bnxt_qplib_release_srqe()
2426 spin_unlock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2444 dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n"); in bnxt_qplib_cq_process_res_rc()
2448 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2468 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_rc()
2469 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2471 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_rc()
2484 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2495 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_rc()
2525 dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n"); in bnxt_qplib_cq_process_res_ud()
2529 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2555 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_ud()
2556 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2558 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_ud()
2571 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2583 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_ud()
2603 raw_cons = cq->hwq.cons; in bnxt_qplib_is_cq_empty()
2604 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); in bnxt_qplib_is_cq_empty()
2605 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); in bnxt_qplib_is_cq_empty()
2607 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements); in bnxt_qplib_is_cq_empty()
2626 dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n"); in bnxt_qplib_cq_process_res_raweth_qp1()
2630 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2659 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2663 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_raweth_qp1()
2664 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2666 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_raweth_qp1()
2679 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2690 bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots); in bnxt_qplib_cq_process_res_raweth_qp1()
2718 dev_warn(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2725 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2742 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2767 bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots); in bnxt_qplib_cq_process_terminal()
2784 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2791 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2813 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_cutoff()
2834 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2839 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2854 raw_cons = cq->hwq.cons; in bnxt_qplib_poll_cq()
2858 sw_cons = HWQ_CMP(raw_cons, &cq->hwq); in bnxt_qplib_poll_cq()
2859 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL); in bnxt_qplib_poll_cq()
2862 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) in bnxt_qplib_poll_cq()
2905 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
2918 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
2923 if (cq->hwq.cons != raw_cons) { in bnxt_qplib_poll_cq()
2924 cq->hwq.cons = raw_cons; in bnxt_qplib_poll_cq()