Lines Matching refs:cqp
156 irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_arp_cache_entry() argument
163 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_arp_cache_entry()
173 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_arp_cache_entry()
181 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_arp_cache_entry()
194 irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_arp_cache_entry() argument
200 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_arp_cache_entry()
206 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_del_arp_cache_entry()
215 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_arp_cache_entry()
228 irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_apbvt_entry() argument
235 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_apbvt_entry()
243 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_apbvt_entry()
251 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_apbvt_entry()
276 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_qhash_table_entry() argument
286 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_qhash_table_entry()
327 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | in irdma_sc_manage_qhash_table_entry()
341 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_qhash_table_entry()
428 struct irdma_sc_cqp *cqp; in irdma_sc_qp_create() local
432 cqp = qp->dev->cqp; in irdma_sc_qp_create()
433 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || in irdma_sc_qp_create()
434 qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1)) in irdma_sc_qp_create()
437 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_create()
456 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_create()
464 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_create()
481 struct irdma_sc_cqp *cqp; in irdma_sc_qp_modify() local
486 cqp = qp->dev->cqp; in irdma_sc_qp_modify()
487 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_modify()
526 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_modify()
534 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_modify()
552 struct irdma_sc_cqp *cqp; in irdma_sc_qp_destroy() local
555 cqp = qp->dev->cqp; in irdma_sc_qp_destroy()
556 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_destroy()
568 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_destroy()
576 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_destroy()
741 irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_alloc_local_mac_entry() argument
747 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_local_mac_entry()
753 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_local_mac_entry()
764 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_local_mac_entry()
776 irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_local_mac_entry() argument
783 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_local_mac_entry()
792 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_local_mac_entry()
802 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_local_mac_entry()
815 irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_local_mac_entry() argument
821 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_local_mac_entry()
828 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_del_local_mac_entry()
840 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_local_mac_entry()
1042 struct irdma_sc_cqp *cqp; in irdma_sc_alloc_stag() local
1053 cqp = dev->cqp; in irdma_sc_alloc_stag()
1054 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_stag()
1078 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_stag()
1086 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_stag()
1105 struct irdma_sc_cqp *cqp; in irdma_sc_mr_reg_non_shared() local
1131 cqp = dev->cqp; in irdma_sc_mr_reg_non_shared()
1132 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mr_reg_non_shared()
1167 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mr_reg_non_shared()
1175 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mr_reg_non_shared()
1194 struct irdma_sc_cqp *cqp; in irdma_sc_dealloc_stag() local
1196 cqp = dev->cqp; in irdma_sc_dealloc_stag()
1197 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_dealloc_stag()
1208 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_dealloc_stag()
1216 irdma_sc_cqp_post_sq(cqp); in irdma_sc_dealloc_stag()
1233 struct irdma_sc_cqp *cqp; in irdma_sc_mw_alloc() local
1236 cqp = dev->cqp; in irdma_sc_mw_alloc()
1237 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mw_alloc()
1250 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mw_alloc()
1258 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mw_alloc()
2063 irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, in irdma_sc_gather_stats() argument
2072 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gather_stats()
2080 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_gather_stats()
2094 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gather_stats()
2095 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_gather_stats()
2097 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_gather_stats()
2110 irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, in irdma_sc_manage_stats_inst() argument
2117 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_stats_inst()
2123 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_manage_stats_inst()
2137 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_stats_inst()
2147 static enum irdma_status_code irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, in irdma_sc_set_up_map() argument
2155 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_set_up_map()
2167 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | in irdma_sc_set_up_map()
2178 irdma_sc_cqp_post_sq(cqp); in irdma_sc_set_up_map()
2191 irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, in irdma_sc_manage_ws_node() argument
2198 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_ws_node()
2206 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | in irdma_sc_manage_ws_node()
2221 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_ws_node()
2239 struct irdma_sc_cqp *cqp; in irdma_sc_qp_flush_wqes() local
2257 cqp = qp->pd->dev->cqp; in irdma_sc_qp_flush_wqes()
2258 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_flush_wqes()
2287 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_flush_wqes()
2295 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_flush_wqes()
2313 struct irdma_sc_cqp *cqp; in irdma_sc_gen_ae() local
2316 cqp = qp->pd->dev->cqp; in irdma_sc_gen_ae()
2317 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gen_ae()
2328 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_gen_ae()
2336 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gen_ae()
2353 struct irdma_sc_cqp *cqp; in irdma_sc_qp_upload_context() local
2356 cqp = dev->cqp; in irdma_sc_qp_upload_context()
2357 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_upload_context()
2368 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_upload_context()
2376 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_upload_context()
2389 irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, in irdma_sc_manage_push_page() argument
2397 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) in irdma_sc_manage_push_page()
2400 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_push_page()
2408 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_manage_push_page()
2417 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_push_page()
2428 static enum irdma_status_code irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, in irdma_sc_suspend_qp() argument
2435 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_suspend_qp()
2441 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_suspend_qp()
2448 irdma_sc_cqp_post_sq(cqp); in irdma_sc_suspend_qp()
2459 static enum irdma_status_code irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, in irdma_sc_resume_qp() argument
2466 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_resume_qp()
2475 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_resume_qp()
2482 irdma_sc_cqp_post_sq(cqp); in irdma_sc_resume_qp()
2548 struct irdma_sc_cqp *cqp; in irdma_sc_cq_create() local
2553 cqp = cq->dev->cqp; in irdma_sc_cq_create()
2554 if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1)) in irdma_sc_cq_create()
2567 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_create()
2598 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_create()
2606 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_create()
2620 struct irdma_sc_cqp *cqp; in irdma_sc_cq_destroy() local
2625 cqp = cq->dev->cqp; in irdma_sc_cq_destroy()
2626 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_destroy()
2650 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_destroy()
2658 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_destroy()
2688 struct irdma_sc_cqp *cqp; in irdma_sc_cq_modify() local
2698 cqp = cq->dev->cqp; in irdma_sc_cq_modify()
2699 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_modify()
2724 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_modify()
2732 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_modify()
2761 static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, in irdma_get_cqp_reg_info() argument
2764 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_get_cqp_reg_info()
2775 static enum irdma_status_code irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, in irdma_cqp_poll_registers() argument
2782 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error); in irdma_cqp_poll_registers()
2784 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_cqp_poll_registers()
2785 ibdev_dbg(to_ibdev(cqp->dev), in irdma_cqp_poll_registers()
2792 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_cqp_poll_registers()
2793 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++; in irdma_cqp_poll_registers()
2796 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_cqp_poll_registers()
3119 enum irdma_status_code irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, in irdma_sc_cqp_init() argument
3131 cqp->size = sizeof(*cqp); in irdma_sc_cqp_init()
3132 cqp->sq_size = info->sq_size; in irdma_sc_cqp_init()
3133 cqp->hw_sq_size = hw_sq_size; in irdma_sc_cqp_init()
3134 cqp->sq_base = info->sq; in irdma_sc_cqp_init()
3135 cqp->host_ctx = info->host_ctx; in irdma_sc_cqp_init()
3136 cqp->sq_pa = info->sq_pa; in irdma_sc_cqp_init()
3137 cqp->host_ctx_pa = info->host_ctx_pa; in irdma_sc_cqp_init()
3138 cqp->dev = info->dev; in irdma_sc_cqp_init()
3139 cqp->struct_ver = info->struct_ver; in irdma_sc_cqp_init()
3140 cqp->hw_maj_ver = info->hw_maj_ver; in irdma_sc_cqp_init()
3141 cqp->hw_min_ver = info->hw_min_ver; in irdma_sc_cqp_init()
3142 cqp->scratch_array = info->scratch_array; in irdma_sc_cqp_init()
3143 cqp->polarity = 0; in irdma_sc_cqp_init()
3144 cqp->en_datacenter_tcp = info->en_datacenter_tcp; in irdma_sc_cqp_init()
3145 cqp->ena_vf_count = info->ena_vf_count; in irdma_sc_cqp_init()
3146 cqp->hmc_profile = info->hmc_profile; in irdma_sc_cqp_init()
3147 cqp->ceqs_per_vf = info->ceqs_per_vf; in irdma_sc_cqp_init()
3148 cqp->disable_packed = info->disable_packed; in irdma_sc_cqp_init()
3149 cqp->rocev2_rto_policy = info->rocev2_rto_policy; in irdma_sc_cqp_init()
3150 cqp->protocol_used = info->protocol_used; in irdma_sc_cqp_init()
3151 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); in irdma_sc_cqp_init()
3152 info->dev->cqp = cqp; in irdma_sc_cqp_init()
3154 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); in irdma_sc_cqp_init()
3155 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0; in irdma_sc_cqp_init()
3156 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0; in irdma_sc_cqp_init()
3158 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); in irdma_sc_cqp_init()
3160 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_sc_cqp_init()
3161 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); in irdma_sc_cqp_init()
3162 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_init()
3164 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_init()
3166 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, in irdma_sc_cqp_init()
3167 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); in irdma_sc_cqp_init()
3177 enum irdma_status_code irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, in irdma_sc_cqp_create() argument
3185 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; in irdma_sc_cqp_create()
3186 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size, in irdma_sc_cqp_create()
3188 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device, in irdma_sc_cqp_create()
3189 cqp->sdbuf.size, &cqp->sdbuf.pa, in irdma_sc_cqp_create()
3191 if (!cqp->sdbuf.va) in irdma_sc_cqp_create()
3194 spin_lock_init(&cqp->dev->cqp_lock); in irdma_sc_cqp_create()
3196 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | in irdma_sc_cqp_create()
3197 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | in irdma_sc_cqp_create()
3198 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | in irdma_sc_cqp_create()
3199 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); in irdma_sc_cqp_create()
3202 cqp->rocev2_rto_policy) | in irdma_sc_cqp_create()
3204 cqp->protocol_used); in irdma_sc_cqp_create()
3207 set_64bit_val(cqp->host_ctx, 0, temp); in irdma_sc_cqp_create()
3208 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa); in irdma_sc_cqp_create()
3210 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | in irdma_sc_cqp_create()
3211 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); in irdma_sc_cqp_create()
3212 set_64bit_val(cqp->host_ctx, 16, temp); in irdma_sc_cqp_create()
3213 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp); in irdma_sc_cqp_create()
3214 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | in irdma_sc_cqp_create()
3215 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); in irdma_sc_cqp_create()
3217 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | in irdma_sc_cqp_create()
3218 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); in irdma_sc_cqp_create()
3220 set_64bit_val(cqp->host_ctx, 32, temp); in irdma_sc_cqp_create()
3221 set_64bit_val(cqp->host_ctx, 40, 0); in irdma_sc_cqp_create()
3224 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | in irdma_sc_cqp_create()
3225 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | in irdma_sc_cqp_create()
3226 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); in irdma_sc_cqp_create()
3228 set_64bit_val(cqp->host_ctx, 48, temp); in irdma_sc_cqp_create()
3231 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | in irdma_sc_cqp_create()
3232 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | in irdma_sc_cqp_create()
3233 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | in irdma_sc_cqp_create()
3234 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); in irdma_sc_cqp_create()
3236 set_64bit_val(cqp->host_ctx, 56, temp); in irdma_sc_cqp_create()
3238 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false); in irdma_sc_cqp_create()
3239 p1 = cqp->host_ctx_pa >> 32; in irdma_sc_cqp_create()
3240 p2 = (u32)cqp->host_ctx_pa; in irdma_sc_cqp_create()
3242 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_create()
3243 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_create()
3246 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_create()
3250 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_create()
3251 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_create()
3254 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { in irdma_sc_cqp_create()
3259 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cqp_create()
3263 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_create()
3264 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_create()
3265 cqp->sdbuf.va = NULL; in irdma_sc_cqp_create()
3266 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_cqp_create()
3276 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_post_sq() argument
3278 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); in irdma_sc_cqp_post_sq()
3280 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_post_sq()
3282 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_cqp_post_sq()
3292 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_cqp_get_next_send_wqe_idx() argument
3298 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { in irdma_sc_cqp_get_next_send_wqe_idx()
3299 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_get_next_send_wqe_idx()
3301 cqp->sq_ring.head, cqp->sq_ring.tail, in irdma_sc_cqp_get_next_send_wqe_idx()
3302 cqp->sq_ring.size); in irdma_sc_cqp_get_next_send_wqe_idx()
3305 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); in irdma_sc_cqp_get_next_send_wqe_idx()
3309 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++; in irdma_sc_cqp_get_next_send_wqe_idx()
3311 cqp->polarity = !cqp->polarity; in irdma_sc_cqp_get_next_send_wqe_idx()
3312 wqe = cqp->sq_base[*wqe_idx].elem; in irdma_sc_cqp_get_next_send_wqe_idx()
3313 cqp->scratch_array[*wqe_idx] = scratch; in irdma_sc_cqp_get_next_send_wqe_idx()
3323 enum irdma_status_code irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_destroy() argument
3328 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_destroy()
3329 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_destroy()
3331 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_destroy()
3335 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_destroy()
3336 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_destroy()
3337 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); in irdma_sc_cqp_destroy()
3339 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_destroy()
3340 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_destroy()
3341 cqp->sdbuf.va = NULL; in irdma_sc_cqp_destroy()
3382 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_get_cqe_info() local
3399 cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx; in irdma_sc_ccq_get_cqe_info()
3405 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_ccq_get_cqe_info()
3406 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_ccq_get_cqe_info()
3411 info->scratch = cqp->scratch_array[wqe_idx]; in irdma_sc_ccq_get_cqe_info()
3415 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1); in irdma_sc_ccq_get_cqe_info()
3417 info->cqp = cqp; in irdma_sc_ccq_get_cqe_info()
3431 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_sc_ccq_get_cqe_info()
3443 enum irdma_status_code irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, in irdma_sc_poll_for_cqp_op_done() argument
3451 ccq = cqp->dev->ccq; in irdma_sc_poll_for_cqp_op_done()
3453 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) in irdma_sc_poll_for_cqp_op_done()
3457 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_poll_for_cqp_op_done()
3467 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_poll_for_cqp_op_done()
3486 irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, in irdma_sc_manage_hmc_pm_func_table() argument
3493 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_hmc_pm_func_table()
3509 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_hmc_pm_func_table()
3518 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_hmc_pm_func_table()
3529 irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_commit_fpm_val_done() argument
3531 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, in irdma_sc_commit_fpm_val_done()
3545 irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id, in irdma_sc_commit_fpm_val() argument
3554 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_commit_fpm_val()
3563 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_commit_fpm_val()
3571 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_commit_fpm_val()
3574 irdma_sc_cqp_post_sq(cqp); in irdma_sc_commit_fpm_val()
3576 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_commit_fpm_val()
3577 cqp->dev->hw_attrs.max_done_count); in irdma_sc_commit_fpm_val()
3579 ret_code = irdma_sc_commit_fpm_val_done(cqp); in irdma_sc_commit_fpm_val()
3591 irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_fpm_val_done() argument
3593 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, in irdma_sc_query_fpm_val_done()
3607 irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id, in irdma_sc_query_fpm_val() argument
3616 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_fpm_val()
3624 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_query_fpm_val()
3631 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_query_fpm_val()
3634 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_fpm_val()
3636 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_query_fpm_val()
3637 cqp->dev->hw_attrs.max_done_count); in irdma_sc_query_fpm_val()
3639 ret_code = irdma_sc_query_fpm_val_done(cqp); in irdma_sc_query_fpm_val()
3700 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_create() local
3704 cqp = ceq->dev->cqp; in irdma_sc_ceq_create()
3705 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_create()
3722 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_create()
3730 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_create()
3742 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_create_done() local
3744 cqp = ceq->dev->cqp; in irdma_sc_cceq_create_done()
3745 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, in irdma_sc_cceq_create_done()
3755 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_destroy_done() local
3760 cqp = ceq->dev->cqp; in irdma_sc_cceq_destroy_done()
3761 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cceq_destroy_done()
3763 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, in irdma_sc_cceq_destroy_done()
3800 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_destroy() local
3804 cqp = ceq->dev->cqp; in irdma_sc_ceq_destroy()
3805 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_destroy()
3816 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_destroy()
3824 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_destroy()
3967 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_create() local
3970 cqp = aeq->dev->cqp; in irdma_sc_aeq_create()
3971 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_create()
3983 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_create()
3991 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_create()
4006 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_destroy() local
4013 cqp = dev->cqp; in irdma_sc_aeq_destroy()
4014 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_destroy()
4022 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_destroy()
4030 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_destroy()
4253 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_create_done() local
4255 cqp = ccq->dev->cqp; in irdma_sc_ccq_create_done()
4257 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); in irdma_sc_ccq_create_done()
4281 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; in irdma_sc_ccq_create()
4295 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_destroy() local
4301 cqp = ccq->dev->cqp; in irdma_sc_ccq_destroy()
4302 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ccq_destroy()
4318 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ccq_destroy()
4325 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_ccq_destroy()
4328 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ccq_destroy()
4329 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_ccq_destroy()
4330 cqp->dev->hw_attrs.max_done_count); in irdma_sc_ccq_destroy()
4333 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_ccq_destroy()
4359 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_init_iw_hmc()
4431 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_cfg_iw_fpm()
4451 cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, struct irdma_update_sds_info *info, in cqp_sds_wqe_fill() argument
4458 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; in cqp_sds_wqe_fill()
4462 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); in cqp_sds_wqe_fill()
4506 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in cqp_sds_wqe_fill()
4534 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_pe_sds() local
4537 ret_code = cqp_sds_wqe_fill(cqp, info, scratch); in irdma_update_pe_sds()
4539 irdma_sc_cqp_post_sq(cqp); in irdma_update_pe_sds()
4554 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_sds_noccq() local
4557 ret_code = cqp_sds_wqe_fill(cqp, info, 0); in irdma_update_sds_noccq()
4561 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_update_sds_noccq()
4563 irdma_sc_cqp_post_sq(cqp); in irdma_update_sds_noccq()
4564 return irdma_cqp_poll_registers(cqp, tail, in irdma_update_sds_noccq()
4565 cqp->dev->hw_attrs.max_done_count); in irdma_update_sds_noccq()
4577 irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_static_hmc_pages_allocated() argument
4585 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_static_hmc_pages_allocated()
4594 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_static_hmc_pages_allocated()
4602 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_static_hmc_pages_allocated()
4605 irdma_sc_cqp_post_sq(cqp); in irdma_sc_static_hmc_pages_allocated()
4608 return irdma_cqp_poll_registers(cqp, tail, in irdma_sc_static_hmc_pages_allocated()
4609 cqp->dev->hw_attrs.max_done_count); in irdma_sc_static_hmc_pages_allocated()
4611 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_static_hmc_pages_allocated()
4623 static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) in irdma_cqp_ring_full() argument
4625 return IRDMA_RING_FULL_ERR(cqp->sq_ring); in irdma_cqp_ring_full()
4663 irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_rdma_features_done() argument
4665 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_query_rdma_features_done()
4677 irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, in irdma_sc_query_rdma_features() argument
4683 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_rdma_features()
4691 cqp->polarity) | in irdma_sc_query_rdma_features()
4700 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_rdma_features()
4723 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4725 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4748 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4750 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4792 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) in irdma_q1_cnt()
4947 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && in irdma_cfg_fpm_val()
5088 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, in irdma_exec_cqp_cmd()
5101 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, in irdma_exec_cqp_cmd()
5107 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5112 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5119 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, in irdma_exec_cqp_cmd()
5127 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, in irdma_exec_cqp_cmd()
5138 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, in irdma_exec_cqp_cmd()
5144 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, in irdma_exec_cqp_cmd()
5149 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5155 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5161 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5167 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, in irdma_exec_cqp_cmd()
5172 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, in irdma_exec_cqp_cmd()
5177 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5183 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, in irdma_exec_cqp_cmd()
5189 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, in irdma_exec_cqp_cmd()
5238 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5244 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5249 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5255 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5262 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, in irdma_exec_cqp_cmd()
5267 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, in irdma_exec_cqp_cmd()
5272 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, in irdma_exec_cqp_cmd()
5277 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, in irdma_exec_cqp_cmd()
5282 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, in irdma_exec_cqp_cmd()
5306 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) in irdma_process_cqp_cmd()
5326 !irdma_cqp_ring_full(dev->cqp)) { in irdma_process_bh()