1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
12  * irdma_get_qp_from_list - get next qp from a list
26 		entry = head->next;  in irdma_get_qp_from_list()
28 		lastentry = &qp->list;  in irdma_get_qp_from_list()
29 		entry = lastentry->next;  in irdma_get_qp_from_list()
38  * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI
48 		mutex_lock(&vsi->qos[i].qos_mutex);  in irdma_sc_suspend_resume_qps()
49 		qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);  in irdma_sc_suspend_resume_qps()
52 				if (!qp->dev->ws_add(vsi, i)) {  in irdma_sc_suspend_resume_qps()
53 					qp->qs_handle =  in irdma_sc_suspend_resume_qps()
54 						vsi->qos[qp->user_pri].qs_handle;  in irdma_sc_suspend_resume_qps()
63 					atomic_inc(&vsi->qp_suspend_reqs);  in irdma_sc_suspend_resume_qps()
65 			qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);  in irdma_sc_suspend_resume_qps()
67 		mutex_unlock(&vsi->qos[i].qos_mutex);  in irdma_sc_suspend_resume_qps()
72  * irdma_change_l2params - given the new l2 parameters, change all qp
79 	if (l2params->mtu_changed) {  in irdma_change_l2params()
80 		vsi->mtu = l2params->mtu;  in irdma_change_l2params()
81 		if (vsi->ieq)  in irdma_change_l2params()
85 	if (!l2params->tc_changed)  in irdma_change_l2params()
88 	vsi->tc_change_pending = false;  in irdma_change_l2params()
93  * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
98 	struct irdma_sc_vsi *vsi = qp->vsi;  in irdma_qp_rem_qos()
100 	ibdev_dbg(to_ibdev(qp->dev),  in irdma_qp_rem_qos()
101 		  "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",  in irdma_qp_rem_qos()
102 		  qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,  in irdma_qp_rem_qos()
103 		  qp->on_qoslist);  in irdma_qp_rem_qos()
104 	mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);  in irdma_qp_rem_qos()
105 	if (qp->on_qoslist) {  in irdma_qp_rem_qos()
106 		qp->on_qoslist = false;  in irdma_qp_rem_qos()
107 		list_del(&qp->list);  in irdma_qp_rem_qos()
109 	mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);  in irdma_qp_rem_qos()
113  * irdma_qp_add_qos - called during setctx for qp to be added to qos
118 	struct irdma_sc_vsi *vsi = qp->vsi;  in irdma_qp_add_qos()
120 	ibdev_dbg(to_ibdev(qp->dev),  in irdma_qp_add_qos()
121 		  "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",  in irdma_qp_add_qos()
122 		  qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,  in irdma_qp_add_qos()
123 		  qp->on_qoslist);  in irdma_qp_add_qos()
124 	mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);  in irdma_qp_add_qos()
125 	if (!qp->on_qoslist) {  in irdma_qp_add_qos()
126 		list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);  in irdma_qp_add_qos()
127 		qp->on_qoslist = true;  in irdma_qp_add_qos()
128 		qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;  in irdma_qp_add_qos()
130 	mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);  in irdma_qp_add_qos()
134  * irdma_sc_pd_init - initialize sc pd struct
143 	pd->pd_id = pd_id;  in irdma_sc_pd_init()
144 	pd->abi_ver = abi_ver;  in irdma_sc_pd_init()
145 	pd->dev = dev;  in irdma_sc_pd_init()
149  * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
166 	set_64bit_val(wqe, 8, info->reach_max);  in irdma_sc_add_arp_cache_entry()
167 	set_64bit_val(wqe, 16, ether_addr_to_u64(info->mac_addr));  in irdma_sc_add_arp_cache_entry()
169 	hdr = info->arp_index |  in irdma_sc_add_arp_cache_entry()
171 	      FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) |  in irdma_sc_add_arp_cache_entry()
173 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_add_arp_cache_entry()
187  * irdma_sc_del_arp_cache_entry - dele arp cache entry
206 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_del_arp_cache_entry()
221  * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
239 	set_64bit_val(wqe, 16, info->port);  in irdma_sc_manage_apbvt_entry()
242 	      FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) |  in irdma_sc_manage_apbvt_entry()
243 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_manage_apbvt_entry()
257  * irdma_sc_manage_qhash_table_entry - manage quad hash entries
284 	struct irdma_sc_vsi *vsi = info->vsi;  in irdma_sc_manage_qhash_table_entry()
290 	set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr));  in irdma_sc_manage_qhash_table_entry()
292 	qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |  in irdma_sc_manage_qhash_table_entry()
293 	      FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);  in irdma_sc_manage_qhash_table_entry()
294 	if (info->ipv4_valid) {  in irdma_sc_manage_qhash_table_entry()
296 			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0]));  in irdma_sc_manage_qhash_table_entry()
299 			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) |  in irdma_sc_manage_qhash_table_entry()
300 			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1]));  in irdma_sc_manage_qhash_table_entry()
303 			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) |  in irdma_sc_manage_qhash_table_entry()
304 			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3]));  in irdma_sc_manage_qhash_table_entry()
307 			 vsi->qos[info->user_pri].qs_handle);  in irdma_sc_manage_qhash_table_entry()
308 	if (info->vlan_valid)  in irdma_sc_manage_qhash_table_entry()
309 		qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id);  in irdma_sc_manage_qhash_table_entry()
311 	if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {  in irdma_sc_manage_qhash_table_entry()
312 		qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port);  in irdma_sc_manage_qhash_table_entry()
313 		if (!info->ipv4_valid) {  in irdma_sc_manage_qhash_table_entry()
315 				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) |  in irdma_sc_manage_qhash_table_entry()
316 				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1]));  in irdma_sc_manage_qhash_table_entry()
318 				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) |  in irdma_sc_manage_qhash_table_entry()
319 				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3]));  in irdma_sc_manage_qhash_table_entry()
322 				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0]));  in irdma_sc_manage_qhash_table_entry()
327 	temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) |  in irdma_sc_manage_qhash_table_entry()
330 	       FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) |  in irdma_sc_manage_qhash_table_entry()
331 	       FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) |  in irdma_sc_manage_qhash_table_entry()
332 	       FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) |  in irdma_sc_manage_qhash_table_entry()
333 	       FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type);  in irdma_sc_manage_qhash_table_entry()
347  * irdma_sc_qp_init - initialize qp
358 	if (info->qp_uk_init_info.max_sq_frag_cnt >  in irdma_sc_qp_init()
359 	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||  in irdma_sc_qp_init()
360 	    info->qp_uk_init_info.max_rq_frag_cnt >  in irdma_sc_qp_init()
361 	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)  in irdma_sc_qp_init()
364 	qp->dev = info->pd->dev;  in irdma_sc_qp_init()
365 	qp->vsi = info->vsi;  in irdma_sc_qp_init()
366 	qp->ieq_qp = info->vsi->exception_lan_q;  in irdma_sc_qp_init()
367 	qp->sq_pa = info->sq_pa;  in irdma_sc_qp_init()
368 	qp->rq_pa = info->rq_pa;  in irdma_sc_qp_init()
369 	qp->hw_host_ctx_pa = info->host_ctx_pa;  in irdma_sc_qp_init()
370 	qp->q2_pa = info->q2_pa;  in irdma_sc_qp_init()
371 	qp->shadow_area_pa = info->shadow_area_pa;  in irdma_sc_qp_init()
372 	qp->q2_buf = info->q2;  in irdma_sc_qp_init()
373 	qp->pd = info->pd;  in irdma_sc_qp_init()
374 	qp->hw_host_ctx = info->host_ctx;  in irdma_sc_qp_init()
375 	info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;  in irdma_sc_qp_init()
376 	ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info);  in irdma_sc_qp_init()
380 	qp->virtual_map = info->virtual_map;  in irdma_sc_qp_init()
381 	pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_qp_init()
383 	if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||  in irdma_sc_qp_init()
384 	    (info->virtual_map && info->rq_pa >= pble_obj_cnt))  in irdma_sc_qp_init()
387 	qp->llp_stream_handle = (void *)(-1);  in irdma_sc_qp_init()
388 	qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,  in irdma_sc_qp_init()
390 	ibdev_dbg(to_ibdev(qp->dev),  in irdma_sc_qp_init()
392 		  qp->hw_sq_size, qp->qp_uk.sq_ring.size);  in irdma_sc_qp_init()
393 	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)  in irdma_sc_qp_init()
396 		ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,  in irdma_sc_qp_init()
401 	qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *  in irdma_sc_qp_init()
403 	ibdev_dbg(to_ibdev(qp->dev),  in irdma_sc_qp_init()
405 		  qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);  in irdma_sc_qp_init()
406 	qp->sq_tph_val = info->sq_tph_val;  in irdma_sc_qp_init()
407 	qp->rq_tph_val = info->rq_tph_val;  in irdma_sc_qp_init()
408 	qp->sq_tph_en = info->sq_tph_en;  in irdma_sc_qp_init()
409 	qp->rq_tph_en = info->rq_tph_en;  in irdma_sc_qp_init()
410 	qp->rcv_tph_en = info->rcv_tph_en;  in irdma_sc_qp_init()
411 	qp->xmit_tph_en = info->xmit_tph_en;  in irdma_sc_qp_init()
412 	qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;  in irdma_sc_qp_init()
413 	qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;  in irdma_sc_qp_init()
419  * irdma_sc_qp_create - create qp
432 	cqp = qp->dev->cqp;  in irdma_sc_qp_create()
433 	if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||  in irdma_sc_qp_create()
434 	    qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))  in irdma_sc_qp_create()
441 	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);  in irdma_sc_qp_create()
442 	set_64bit_val(wqe, 40, qp->shadow_area_pa);  in irdma_sc_qp_create()
444 	hdr = qp->qp_uk.qp_id |  in irdma_sc_qp_create()
446 	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) |  in irdma_sc_qp_create()
447 	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |  in irdma_sc_qp_create()
448 	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |  in irdma_sc_qp_create()
449 	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |  in irdma_sc_qp_create()
450 	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |  in irdma_sc_qp_create()
451 	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |  in irdma_sc_qp_create()
452 	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |  in irdma_sc_qp_create()
454 			 info->arp_cache_idx_valid) |  in irdma_sc_qp_create()
455 	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |  in irdma_sc_qp_create()
456 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_qp_create()
470  * irdma_sc_qp_modify - modify qp cqp wqe
486 	cqp = qp->dev->cqp;  in irdma_sc_qp_modify()
491 	if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {  in irdma_sc_qp_modify()
492 		if (info->dont_send_fin)  in irdma_sc_qp_modify()
494 		if (info->dont_send_term)  in irdma_sc_qp_modify()
498 			term_len = info->termlen;  in irdma_sc_qp_modify()
502 		      FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) |  in irdma_sc_qp_modify()
504 	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);  in irdma_sc_qp_modify()
505 	set_64bit_val(wqe, 40, qp->shadow_area_pa);  in irdma_sc_qp_modify()
507 	hdr = qp->qp_uk.qp_id |  in irdma_sc_qp_modify()
509 	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |  in irdma_sc_qp_modify()
510 	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |  in irdma_sc_qp_modify()
512 			 info->cached_var_valid) |  in irdma_sc_qp_modify()
513 	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |  in irdma_sc_qp_modify()
514 	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |  in irdma_sc_qp_modify()
515 	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |  in irdma_sc_qp_modify()
516 	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |  in irdma_sc_qp_modify()
517 	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |  in irdma_sc_qp_modify()
518 	      FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) |  in irdma_sc_qp_modify()
520 			 info->remove_hash_idx) |  in irdma_sc_qp_modify()
522 	      FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) |  in irdma_sc_qp_modify()
524 			 info->arp_cache_idx_valid) |  in irdma_sc_qp_modify()
525 	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |  in irdma_sc_qp_modify()
526 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_qp_modify()
540  * irdma_sc_qp_destroy - cqp destroy qp
555 	cqp = qp->dev->cqp;  in irdma_sc_qp_destroy()
560 	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);  in irdma_sc_qp_destroy()
561 	set_64bit_val(wqe, 40, qp->shadow_area_pa);  in irdma_sc_qp_destroy()
563 	hdr = qp->qp_uk.qp_id |  in irdma_sc_qp_destroy()
565 	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |  in irdma_sc_qp_destroy()
568 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_qp_destroy()
582  * irdma_sc_get_encoded_ird_size -
611  * irdma_sc_qp_setctx_roce - set qp's context
624 	roce_info = info->roce_info;  in irdma_sc_qp_setctx_roce()
625 	udp = info->udp_info;  in irdma_sc_qp_setctx_roce()
626 	qp->user_pri = info->user_pri;  in irdma_sc_qp_setctx_roce()
627 	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {  in irdma_sc_qp_setctx_roce()
632 		push_idx = qp->push_idx;  in irdma_sc_qp_setctx_roce()
635 		      FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |  in irdma_sc_qp_setctx_roce()
636 		      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |  in irdma_sc_qp_setctx_roce()
637 		      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |  in irdma_sc_qp_setctx_roce()
638 		      FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |  in irdma_sc_qp_setctx_roce()
639 		      FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |  in irdma_sc_qp_setctx_roce()
642 		      FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) |  in irdma_sc_qp_setctx_roce()
643 		      FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) |  in irdma_sc_qp_setctx_roce()
644 		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) |  in irdma_sc_qp_setctx_roce()
645 		      FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |  in irdma_sc_qp_setctx_roce()
646 		      FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |  in irdma_sc_qp_setctx_roce()
647 		      FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |  in irdma_sc_qp_setctx_roce()
648 		      FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag));  in irdma_sc_qp_setctx_roce()
649 	set_64bit_val(qp_ctx, 8, qp->sq_pa);  in irdma_sc_qp_setctx_roce()
650 	set_64bit_val(qp_ctx, 16, qp->rq_pa);  in irdma_sc_qp_setctx_roce()
651 	if ((roce_info->dcqcn_en || roce_info->dctcp_en) &&  in irdma_sc_qp_setctx_roce()
652 	    !(udp->tos & 0x03))  in irdma_sc_qp_setctx_roce()
653 		udp->tos |= ECN_CODE_PT_VAL;  in irdma_sc_qp_setctx_roce()
655 		      FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |  in irdma_sc_qp_setctx_roce()
656 		      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) |  in irdma_sc_qp_setctx_roce()
657 		      FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) |  in irdma_sc_qp_setctx_roce()
658 		      FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) |  in irdma_sc_qp_setctx_roce()
659 		      FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port));  in irdma_sc_qp_setctx_roce()
661 		      FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) |  in irdma_sc_qp_setctx_roce()
662 		      FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3]));  in irdma_sc_qp_setctx_roce()
664 		      FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) |  in irdma_sc_qp_setctx_roce()
665 		      FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1]));  in irdma_sc_qp_setctx_roce()
667 		      FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) |  in irdma_sc_qp_setctx_roce()
668 		      FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) |  in irdma_sc_qp_setctx_roce()
669 		      FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx));  in irdma_sc_qp_setctx_roce()
671 		      FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) |  in irdma_sc_qp_setctx_roce()
672 		      FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) |  in irdma_sc_qp_setctx_roce()
673 		      FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) |  in irdma_sc_qp_setctx_roce()
674 		      FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label));  in irdma_sc_qp_setctx_roce()
676 		      FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) |  in irdma_sc_qp_setctx_roce()
677 		      FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp));  in irdma_sc_qp_setctx_roce()
679 		      FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) |  in irdma_sc_qp_setctx_roce()
680 		      FIELD_PREP(IRDMAQPC_LSN, udp->lsn));  in irdma_sc_qp_setctx_roce()
682 		      FIELD_PREP(IRDMAQPC_EPSN, udp->epsn));  in irdma_sc_qp_setctx_roce()
684 		      FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) |  in irdma_sc_qp_setctx_roce()
685 		      FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una));  in irdma_sc_qp_setctx_roce()
687 		      FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd));  in irdma_sc_qp_setctx_roce()
689 		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) |  in irdma_sc_qp_setctx_roce()
690 		      FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) |  in irdma_sc_qp_setctx_roce()
691 		      FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) |  in irdma_sc_qp_setctx_roce()
692 		      FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin));  in irdma_sc_qp_setctx_roce()
694 		      FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |  in irdma_sc_qp_setctx_roce()
695 		      FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));  in irdma_sc_qp_setctx_roce()
697 		      FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));  in irdma_sc_qp_setctx_roce()
698 	set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info->mac_addr) << 16);  in irdma_sc_qp_setctx_roce()
700 		      FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |  in irdma_sc_qp_setctx_roce()
701 		      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |  in irdma_sc_qp_setctx_roce()
702 		      FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |  in irdma_sc_qp_setctx_roce()
703 		      FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |  in irdma_sc_qp_setctx_roce()
704 		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |  in irdma_sc_qp_setctx_roce()
705 		      FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |  in irdma_sc_qp_setctx_roce()
706 		      FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |  in irdma_sc_qp_setctx_roce()
707 		      FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |  in irdma_sc_qp_setctx_roce()
708 		      FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |  in irdma_sc_qp_setctx_roce()
709 		      FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) |  in irdma_sc_qp_setctx_roce()
710 		      FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) |  in irdma_sc_qp_setctx_roce()
711 		      FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |  in irdma_sc_qp_setctx_roce()
712 		      FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));  in irdma_sc_qp_setctx_roce()
714 		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));  in irdma_sc_qp_setctx_roce()
716 		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |  in irdma_sc_qp_setctx_roce()
717 		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |  in irdma_sc_qp_setctx_roce()
718 		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));  in irdma_sc_qp_setctx_roce()
720 		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) |  in irdma_sc_qp_setctx_roce()
721 		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2]));  in irdma_sc_qp_setctx_roce()
723 		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) |  in irdma_sc_qp_setctx_roce()
724 		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));  in irdma_sc_qp_setctx_roce()
726 		      FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |  in irdma_sc_qp_setctx_roce()
727 		      FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));  in irdma_sc_qp_setctx_roce()
729 		      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));  in irdma_sc_qp_setctx_roce()
735 /* irdma_sc_alloc_local_mac_entry - allocate a mac entry
753 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_alloc_local_mac_entry()
769  * irdma_sc_add_local_mac_entry - add mac enry
787 	set_64bit_val(wqe, 32, ether_addr_to_u64(info->mac_addr));  in irdma_sc_add_local_mac_entry()
789 	header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |  in irdma_sc_add_local_mac_entry()
792 		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_add_local_mac_entry()
807  * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
828 		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |  in irdma_sc_del_local_mac_entry()
845  * irdma_sc_qp_setctx - set qp's context
861 	iw = info->iwarp_info;  in irdma_sc_qp_setctx()
862 	tcp = info->tcp_info;  in irdma_sc_qp_setctx()
863 	dev = qp->dev;  in irdma_sc_qp_setctx()
864 	if (iw->rcv_mark_en) {  in irdma_sc_qp_setctx()
865 		qp->pfpdu.marker_len = 4;  in irdma_sc_qp_setctx()
866 		qp->pfpdu.rcv_start_seq = tcp->rcv_nxt;  in irdma_sc_qp_setctx()
868 	qp->user_pri = info->user_pri;  in irdma_sc_qp_setctx()
869 	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {  in irdma_sc_qp_setctx()
874 		push_idx = qp->push_idx;  in irdma_sc_qp_setctx()
876 	qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |  in irdma_sc_qp_setctx()
877 	      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |  in irdma_sc_qp_setctx()
878 	      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |  in irdma_sc_qp_setctx()
879 	      FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |  in irdma_sc_qp_setctx()
880 	      FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |  in irdma_sc_qp_setctx()
884 	set_64bit_val(qp_ctx, 8, qp->sq_pa);  in irdma_sc_qp_setctx()
885 	set_64bit_val(qp_ctx, 16, qp->rq_pa);  in irdma_sc_qp_setctx()
887 	qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |  in irdma_sc_qp_setctx()
888 	      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size);  in irdma_sc_qp_setctx()
889 	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_sc_qp_setctx()
891 				  qp->src_mac_addr_idx);  in irdma_sc_qp_setctx()
893 		      FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |  in irdma_sc_qp_setctx()
894 		      FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));  in irdma_sc_qp_setctx()
896 		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));  in irdma_sc_qp_setctx()
898 		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |  in irdma_sc_qp_setctx()
899 		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |  in irdma_sc_qp_setctx()
900 		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) |  in irdma_sc_qp_setctx()
901 		      FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp));  in irdma_sc_qp_setctx()
902 	if (info->iwarp_info_valid) {  in irdma_sc_qp_setctx()
903 		qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) |  in irdma_sc_qp_setctx()
904 		       FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) |  in irdma_sc_qp_setctx()
905 		       FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) |  in irdma_sc_qp_setctx()
906 		       FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) |  in irdma_sc_qp_setctx()
907 		       FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) |  in irdma_sc_qp_setctx()
908 		       FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) |  in irdma_sc_qp_setctx()
910 				  iw->err_rq_idx_valid);  in irdma_sc_qp_setctx()
911 		qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id);  in irdma_sc_qp_setctx()
912 		qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) |  in irdma_sc_qp_setctx()
913 			FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin);  in irdma_sc_qp_setctx()
915 			      FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |  in irdma_sc_qp_setctx()
916 			      FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));  in irdma_sc_qp_setctx()
918 		if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)  in irdma_sc_qp_setctx()
919 			mac = ether_addr_to_u64(iw->mac_addr);  in irdma_sc_qp_setctx()
922 			      mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));  in irdma_sc_qp_setctx()
924 			      FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) |  in irdma_sc_qp_setctx()
925 			      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) |  in irdma_sc_qp_setctx()
926 			      FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |  in irdma_sc_qp_setctx()
927 			      FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |  in irdma_sc_qp_setctx()
928 			      FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |  in irdma_sc_qp_setctx()
929 			      FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |  in irdma_sc_qp_setctx()
930 			      FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |  in irdma_sc_qp_setctx()
931 			      FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |  in irdma_sc_qp_setctx()
932 			      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |  in irdma_sc_qp_setctx()
934 			      FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) |  in irdma_sc_qp_setctx()
935 			      FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) |  in irdma_sc_qp_setctx()
936 			      FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) |  in irdma_sc_qp_setctx()
937 …FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nx…  in irdma_sc_qp_setctx()
938 …FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nx…  in irdma_sc_qp_setctx()
939 			      FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en));  in irdma_sc_qp_setctx()
941 	if (info->tcp_info_valid) {  in irdma_sc_qp_setctx()
942 		qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) |  in irdma_sc_qp_setctx()
943 		       FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) |  in irdma_sc_qp_setctx()
945 				  tcp->insert_vlan_tag) |  in irdma_sc_qp_setctx()
946 		       FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) |  in irdma_sc_qp_setctx()
947 		       FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) |  in irdma_sc_qp_setctx()
948 		       FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) |  in irdma_sc_qp_setctx()
949 		       FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh);  in irdma_sc_qp_setctx()
951 		if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))  in irdma_sc_qp_setctx()
952 			tcp->tos |= ECN_CODE_PT_VAL;  in irdma_sc_qp_setctx()
954 		qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) |  in irdma_sc_qp_setctx()
955 		       FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) |  in irdma_sc_qp_setctx()
956 		       FIELD_PREP(IRDMAQPC_TOS, tcp->tos) |  in irdma_sc_qp_setctx()
957 		       FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) |  in irdma_sc_qp_setctx()
958 		       FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port);  in irdma_sc_qp_setctx()
959 		if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {  in irdma_sc_qp_setctx()
960 			qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx);  in irdma_sc_qp_setctx()
962 			qp->src_mac_addr_idx = tcp->src_mac_addr_idx;  in irdma_sc_qp_setctx()
965 			      FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) |  in irdma_sc_qp_setctx()
966 			      FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3]));  in irdma_sc_qp_setctx()
968 			      FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) |  in irdma_sc_qp_setctx()
969 			      FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1]));  in irdma_sc_qp_setctx()
971 			      FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) |  in irdma_sc_qp_setctx()
972 			      FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) |  in irdma_sc_qp_setctx()
973 			      FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) |  in irdma_sc_qp_setctx()
974 			      FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx));  in irdma_sc_qp_setctx()
975 		qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) |  in irdma_sc_qp_setctx()
976 		       FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) |  in irdma_sc_qp_setctx()
978 				  tcp->ignore_tcp_opt) |  in irdma_sc_qp_setctx()
980 				  tcp->ignore_tcp_uns_opt) |  in irdma_sc_qp_setctx()
981 		       FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) |  in irdma_sc_qp_setctx()
982 		       FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) |  in irdma_sc_qp_setctx()
983 		       FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale);  in irdma_sc_qp_setctx()
985 			      FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) |  in irdma_sc_qp_setctx()
986 			      FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age));  in irdma_sc_qp_setctx()
988 			      FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) |  in irdma_sc_qp_setctx()
989 			      FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd));  in irdma_sc_qp_setctx()
991 			      FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) |  in irdma_sc_qp_setctx()
992 			      FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd));  in irdma_sc_qp_setctx()
994 			      FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) |  in irdma_sc_qp_setctx()
995 			      FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una));  in irdma_sc_qp_setctx()
997 			      FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) |  in irdma_sc_qp_setctx()
998 			      FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var));  in irdma_sc_qp_setctx()
1000 			      FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) |  in irdma_sc_qp_setctx()
1001 			      FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd));  in irdma_sc_qp_setctx()
1003 			      FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) |  in irdma_sc_qp_setctx()
1004 			      FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2));  in irdma_sc_qp_setctx()
1005 		qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) |  in irdma_sc_qp_setctx()
1006 			FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh);  in irdma_sc_qp_setctx()
1008 			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) |  in irdma_sc_qp_setctx()
1009 			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2]));  in irdma_sc_qp_setctx()
1011 			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) |  in irdma_sc_qp_setctx()
1012 			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0]));  in irdma_sc_qp_setctx()
1014 			      FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) |  in irdma_sc_qp_setctx()
1015 			      FIELD_PREP(IRDMAQPC_TLOW, iw->t_low));  in irdma_sc_qp_setctx()
1017 			      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));  in irdma_sc_qp_setctx()
1030  * irdma_sc_alloc_stag - mr stag alloc
1046 	if (info->page_size == 0x40000000)  in irdma_sc_alloc_stag()
1048 	else if (info->page_size == 0x200000)  in irdma_sc_alloc_stag()
1053 	cqp = dev->cqp;  in irdma_sc_alloc_stag()
1059 		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |  in irdma_sc_alloc_stag()
1060 		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len));  in irdma_sc_alloc_stag()
1062 		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));  in irdma_sc_alloc_stag()
1064 		      FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index));  in irdma_sc_alloc_stag()
1066 	if (info->chunk_size)  in irdma_sc_alloc_stag()
1068 			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx));  in irdma_sc_alloc_stag()
1072 	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |  in irdma_sc_alloc_stag()
1073 	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |  in irdma_sc_alloc_stag()
1075 	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |  in irdma_sc_alloc_stag()
1076 	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |  in irdma_sc_alloc_stag()
1077 	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |  in irdma_sc_alloc_stag()
1078 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_alloc_stag()
1092  * irdma_sc_mr_reg_non_shared - non-shared mr registration
1112 	if (info->page_size == 0x40000000)  in irdma_sc_mr_reg_non_shared()
1114 	else if (info->page_size == 0x200000)  in irdma_sc_mr_reg_non_shared()
1116 	else if (info->page_size == 0x1000)  in irdma_sc_mr_reg_non_shared()
1121 	if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |  in irdma_sc_mr_reg_non_shared()
1127 	pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_mr_reg_non_shared()
1128 	if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)  in irdma_sc_mr_reg_non_shared()
1131 	cqp = dev->cqp;  in irdma_sc_mr_reg_non_shared()
1135 	fbo = info->va & (info->page_size - 1);  in irdma_sc_mr_reg_non_shared()
1138 		      (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?  in irdma_sc_mr_reg_non_shared()
1139 		      info->va : fbo));  in irdma_sc_mr_reg_non_shared()
1141 		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) |  in irdma_sc_mr_reg_non_shared()
1142 		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));  in irdma_sc_mr_reg_non_shared()
1144 		      FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) |  in irdma_sc_mr_reg_non_shared()
1145 		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));  in irdma_sc_mr_reg_non_shared()
1146 	if (!info->chunk_size) {  in irdma_sc_mr_reg_non_shared()
1147 		set_64bit_val(wqe, 32, info->reg_addr_pa);  in irdma_sc_mr_reg_non_shared()
1152 			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index));  in irdma_sc_mr_reg_non_shared()
1154 	set_64bit_val(wqe, 40, info->hmc_fcn_index);  in irdma_sc_mr_reg_non_shared()
1157 	addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;  in irdma_sc_mr_reg_non_shared()
1160 	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |  in irdma_sc_mr_reg_non_shared()
1162 	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |  in irdma_sc_mr_reg_non_shared()
1165 	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |  in irdma_sc_mr_reg_non_shared()
1166 	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |  in irdma_sc_mr_reg_non_shared()
1167 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_mr_reg_non_shared()
1181  * irdma_sc_dealloc_stag - deallocate stag
1196 	cqp = dev->cqp;  in irdma_sc_dealloc_stag()
1202 		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));  in irdma_sc_dealloc_stag()
1204 		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));  in irdma_sc_dealloc_stag()
1207 	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) |  in irdma_sc_dealloc_stag()
1208 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_dealloc_stag()
1222  * irdma_sc_mw_alloc - mw allocate
1236 	cqp = dev->cqp;  in irdma_sc_mw_alloc()
1242 		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));  in irdma_sc_mw_alloc()
1244 		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index));  in irdma_sc_mw_alloc()
1247 	      FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) |  in irdma_sc_mw_alloc()
1249 			 info->mw1_bind_dont_vldt_key) |  in irdma_sc_mw_alloc()
1250 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_mw_alloc()
1264  * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
1279 	if (info->page_size == 0x40000000)  in irdma_sc_mr_fast_register()
1281 	else if (info->page_size == 0x200000)  in irdma_sc_mr_fast_register()
1286 	sq_info.wr_id = info->wr_id;  in irdma_sc_mr_fast_register()
1287 	sq_info.signaled = info->signaled;  in irdma_sc_mr_fast_register()
1288 	sq_info.push_wqe = info->push_wqe;  in irdma_sc_mr_fast_register()
1290 	wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,  in irdma_sc_mr_fast_register()
1295 	irdma_clr_wqes(&qp->qp_uk, wqe_idx);  in irdma_sc_mr_fast_register()
1297 	ibdev_dbg(to_ibdev(qp->dev),  in irdma_sc_mr_fast_register()
1299 		  info->wr_id, wqe_idx,  in irdma_sc_mr_fast_register()
1300 		  &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);  in irdma_sc_mr_fast_register()
1302 	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?  in irdma_sc_mr_fast_register()
1303 		(uintptr_t)info->va : info->fbo;  in irdma_sc_mr_fast_register()
1307 			 info->first_pm_pbl_index >> 16);  in irdma_sc_mr_fast_register()
1310 		      FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa));  in irdma_sc_mr_fast_register()
1312 		      info->total_len |  in irdma_sc_mr_fast_register()
1313 		      FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));  in irdma_sc_mr_fast_register()
1315 	hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) |  in irdma_sc_mr_fast_register()
1316 	      FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) |  in irdma_sc_mr_fast_register()
1318 	      FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |  in irdma_sc_mr_fast_register()
1320 	      FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |  in irdma_sc_mr_fast_register()
1321 	      FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |  in irdma_sc_mr_fast_register()
1323 	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |  in irdma_sc_mr_fast_register()
1324 	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |  in irdma_sc_mr_fast_register()
1325 	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |  in irdma_sc_mr_fast_register()
1326 	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_mr_fast_register()
1334 		irdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,  in irdma_sc_mr_fast_register()
1338 			irdma_uk_qp_post_wr(&qp->qp_uk);  in irdma_sc_mr_fast_register()
1345  * irdma_sc_gen_rts_ae - request AE generated after RTS
1354 	qp_uk = &qp->qp_uk;  in irdma_sc_gen_rts_ae()
1356 	wqe = qp_uk->sq_base[1].elem;  in irdma_sc_gen_rts_ae()
1360 	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_gen_rts_ae()
1367 	wqe = qp_uk->sq_base[2].elem;  in irdma_sc_gen_rts_ae()
1369 	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_gen_rts_ae()
1378  * irdma_sc_send_lsmm - send last streaming mode message
1391 	qp_uk = &qp->qp_uk;  in irdma_sc_send_lsmm()
1392 	wqe = qp_uk->sq_base->elem;  in irdma_sc_send_lsmm()
1395 	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {  in irdma_sc_send_lsmm()
1403 			      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));  in irdma_sc_send_lsmm()
1410 	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_send_lsmm()
1418 	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)  in irdma_sc_send_lsmm()
1423  * irdma_sc_send_lsmm_nostag - for privilege qp
1434 	qp_uk = &qp->qp_uk;  in irdma_sc_send_lsmm_nostag()
1435 	wqe = qp_uk->sq_base->elem;  in irdma_sc_send_lsmm_nostag()
1439 	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)  in irdma_sc_send_lsmm_nostag()
1445 			      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));  in irdma_sc_send_lsmm_nostag()
1451 	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_send_lsmm_nostag()
1461  * irdma_sc_send_rtt - send last read0 or write0
1471 	qp_uk = &qp->qp_uk;  in irdma_sc_send_rtt()
1472 	wqe = qp_uk->sq_base->elem;  in irdma_sc_send_rtt()
1477 		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {  in irdma_sc_send_rtt()
1482 				      (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));  in irdma_sc_send_rtt()
1486 		      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_send_rtt()
1489 		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {  in irdma_sc_send_rtt()
1493 				      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));  in irdma_sc_send_rtt()
1496 		      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);  in irdma_sc_send_rtt()
1506 	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)  in irdma_sc_send_rtt()
1511  * irdma_iwarp_opcode - determine if incoming is rdma layer
1520 	if (info->q2_data_written) {  in irdma_iwarp_opcode()
1529  * irdma_locate_mpa - return pointer to mpa in the pkt
1545  * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
1557 	qp->flush_code = opcode;  in irdma_bld_termhdr_ctrl()
1558 	hdr->layer_etype = layer_etype;  in irdma_bld_termhdr_ctrl()
1559 	hdr->error_code = err;  in irdma_bld_termhdr_ctrl()
1563  * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
1577 		hdr->hdrct = DDP_LEN_FLAG;  in irdma_bld_termhdr_ddp_rdma()
1582 				hdr->hdrct |= DDP_HDR_FLAG;  in irdma_bld_termhdr_ddp_rdma()
1587 				hdr->hdrct |= DDP_HDR_FLAG;  in irdma_bld_termhdr_ddp_rdma()
1592 				hdr->hdrct |= RDMA_HDR_FLAG;  in irdma_bld_termhdr_ddp_rdma()
1599  * irdma_bld_terminate_hdr - build terminate message header
1606 	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;  in irdma_bld_terminate_hdr()
1612 	termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;  in irdma_bld_terminate_hdr()
1615 	if (info->q2_data_written) {  in irdma_bld_terminate_hdr()
1621 	qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;  in irdma_bld_terminate_hdr()
1622 	qp->sq_flush_code = info->sq;  in irdma_bld_terminate_hdr()
1623 	qp->rq_flush_code = info->rq;  in irdma_bld_terminate_hdr()
1625 	switch (info->ae_id) {  in irdma_bld_terminate_hdr()
1627 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1638 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1639 		if (info->q2_data_written)  in irdma_bld_terminate_hdr()
1668 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1680 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1697 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1703 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1729 		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;  in irdma_bld_terminate_hdr()
1778  * irdma_terminate_send_fin() - Send fin for terminate message
1788  * irdma_terminate_connection() - Bad AE and send terminate to remote QP
1797 	if (qp->term_flags & IRDMA_TERM_SENT)  in irdma_terminate_connection()
1802 	qp->term_flags |= IRDMA_TERM_SENT;  in irdma_terminate_connection()
1808  * irdma_terminate_received - handle terminate received AE
1815 	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;  in irdma_terminate_received()
1823 	if (info->q2_data_written) {  in irdma_terminate_received()
1824 		/* did not validate the frame - do it now */  in irdma_terminate_received()
1840 		info->ae_id = aeq_id;  in irdma_terminate_received()
1841 		if (info->ae_id) {  in irdma_terminate_received()
1842 			/* Bad terminate recvd - send back a terminate */  in irdma_terminate_received()
1848 	qp->term_flags |= IRDMA_TERM_RCVD;  in irdma_terminate_received()
1849 	qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;  in irdma_terminate_received()
1851 	if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||  in irdma_terminate_received()
1852 	    termhdr->layer_etype == RDMAP_REMOTE_OP) {  in irdma_terminate_received()
1877  * irdma_sc_vsi_init - Init the vsi structure
1887 	vsi->dev = info->dev;  in irdma_sc_vsi_init()
1888 	vsi->back_vsi = info->back_vsi;  in irdma_sc_vsi_init()
1889 	vsi->register_qset = info->register_qset;  in irdma_sc_vsi_init()
1890 	vsi->unregister_qset = info->unregister_qset;  in irdma_sc_vsi_init()
1891 	vsi->mtu = info->params->mtu;  in irdma_sc_vsi_init()
1892 	vsi->exception_lan_q = info->exception_lan_q;  in irdma_sc_vsi_init()
1893 	vsi->vsi_idx = info->pf_data_vsi_num;  in irdma_sc_vsi_init()
1894 	if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_sc_vsi_init()
1895 		vsi->fcn_id = info->dev->hmc_fn_id;  in irdma_sc_vsi_init()
1897 	l2p = info->params;  in irdma_sc_vsi_init()
1898 	vsi->qos_rel_bw = l2p->vsi_rel_bw;  in irdma_sc_vsi_init()
1899 	vsi->qos_prio_type = l2p->vsi_prio_type;  in irdma_sc_vsi_init()
1901 		if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_sc_vsi_init()
1902 			vsi->qos[i].qs_handle = l2p->qs_handle_list[i];  in irdma_sc_vsi_init()
1903 		vsi->qos[i].traffic_class = info->params->up2tc[i];  in irdma_sc_vsi_init()
1904 		vsi->qos[i].rel_bw =  in irdma_sc_vsi_init()
1905 			l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;  in irdma_sc_vsi_init()
1906 		vsi->qos[i].prio_type =  in irdma_sc_vsi_init()
1907 			l2p->tc_info[vsi->qos[i].traffic_class].prio_type;  in irdma_sc_vsi_init()
1908 		vsi->qos[i].valid = false;  in irdma_sc_vsi_init()
1909 		mutex_init(&vsi->qos[i].qos_mutex);  in irdma_sc_vsi_init()
1910 		INIT_LIST_HEAD(&vsi->qos[i].qplist);  in irdma_sc_vsi_init()
1912 	if (vsi->register_qset) {  in irdma_sc_vsi_init()
1913 		vsi->dev->ws_add = irdma_ws_add;  in irdma_sc_vsi_init()
1914 		vsi->dev->ws_remove = irdma_ws_remove;  in irdma_sc_vsi_init()
1915 		vsi->dev->ws_reset = irdma_ws_reset;  in irdma_sc_vsi_init()
1917 		vsi->dev->ws_add = irdma_null_ws_add;  in irdma_sc_vsi_init()
1918 		vsi->dev->ws_remove = irdma_null_ws_remove;  in irdma_sc_vsi_init()
1919 		vsi->dev->ws_reset = irdma_null_ws_reset;  in irdma_sc_vsi_init()
1924  * irdma_get_fcn_id - Return the function id
1930 	struct irdma_sc_dev *dev = vsi->dev;  in irdma_get_fcn_id()
1934 	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {  in irdma_get_fcn_id()
1943 		if (!dev->fcn_id_array[i]) {  in irdma_get_fcn_id()
1945 			dev->fcn_id_array[i] = true;  in irdma_get_fcn_id()
1953  * irdma_vsi_stats_init - Initialize the vsi statistics
1960 	u8 fcn_id = info->fcn_id;  in irdma_vsi_stats_init()
1963 	vsi->pestat = info->pestat;  in irdma_vsi_stats_init()
1964 	vsi->pestat->hw = vsi->dev->hw;  in irdma_vsi_stats_init()
1965 	vsi->pestat->vsi = vsi;  in irdma_vsi_stats_init()
1966 	stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;  in irdma_vsi_stats_init()
1967 	stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);  in irdma_vsi_stats_init()
1968 	stats_buff_mem->va = dma_alloc_coherent(vsi->pestat->hw->device,  in irdma_vsi_stats_init()
1969 						stats_buff_mem->size,  in irdma_vsi_stats_init()
1970 						&stats_buff_mem->pa,  in irdma_vsi_stats_init()
1972 	if (!stats_buff_mem->va)  in irdma_vsi_stats_init()
1975 	vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va;  in irdma_vsi_stats_init()
1976 	vsi->pestat->gather_info.last_gather_stats_va =  in irdma_vsi_stats_init()
1977 		(void *)((uintptr_t)stats_buff_mem->va +  in irdma_vsi_stats_init()
1981 	if (info->alloc_fcn_id)  in irdma_vsi_stats_init()
1986 	vsi->stats_fcn_id_alloc = info->alloc_fcn_id;  in irdma_vsi_stats_init()
1987 	vsi->fcn_id = fcn_id;  in irdma_vsi_stats_init()
1988 	if (info->alloc_fcn_id) {  in irdma_vsi_stats_init()
1989 		vsi->pestat->gather_info.use_stats_inst = true;  in irdma_vsi_stats_init()
1990 		vsi->pestat->gather_info.stats_inst_index = fcn_id;  in irdma_vsi_stats_init()
1996 	dma_free_coherent(vsi->pestat->hw->device, stats_buff_mem->size,  in irdma_vsi_stats_init()
1997 			  stats_buff_mem->va, stats_buff_mem->pa);  in irdma_vsi_stats_init()
1998 	stats_buff_mem->va = NULL;  in irdma_vsi_stats_init()
2004  * irdma_vsi_stats_free - Free the vsi stats
2010 	u8 fcn_id = vsi->fcn_id;  in irdma_vsi_stats_free()
2011 	struct irdma_sc_dev *dev = vsi->dev;  in irdma_vsi_stats_free()
2013 	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {  in irdma_vsi_stats_free()
2014 		if (vsi->stats_fcn_id_alloc) {  in irdma_vsi_stats_free()
2015 			stats_info.stats_idx = vsi->fcn_id;  in irdma_vsi_stats_free()
2020 		if (vsi->stats_fcn_id_alloc &&  in irdma_vsi_stats_free()
2021 		    fcn_id < vsi->dev->hw_attrs.max_stat_inst)  in irdma_vsi_stats_free()
2022 			vsi->dev->fcn_id_array[fcn_id] = false;  in irdma_vsi_stats_free()
2025 	if (!vsi->pestat)  in irdma_vsi_stats_free()
2028 	dma_free_coherent(vsi->pestat->hw->device,  in irdma_vsi_stats_free()
2029 			  vsi->pestat->gather_info.stats_buff_mem.size,  in irdma_vsi_stats_free()
2030 			  vsi->pestat->gather_info.stats_buff_mem.va,  in irdma_vsi_stats_free()
2031 			  vsi->pestat->gather_info.stats_buff_mem.pa);  in irdma_vsi_stats_free()
2032 	vsi->pestat->gather_info.stats_buff_mem.va = NULL;  in irdma_vsi_stats_free()
2036  * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
2038  * @queue_type: queue type selected for the calculation algorithm
2057  * irdma_sc_gather_stats - collect the statistics
2069 	if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)  in irdma_sc_gather_stats()
2077 		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index));  in irdma_sc_gather_stats()
2078 	set_64bit_val(wqe, 32, info->stats_buff_mem.pa);  in irdma_sc_gather_stats()
2080 	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |  in irdma_sc_gather_stats()
2081 	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) |  in irdma_sc_gather_stats()
2083 			  info->stats_inst_index) |  in irdma_sc_gather_stats()
2085 			  info->use_hmc_fcn_index) |  in irdma_sc_gather_stats()
2095 	ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_gather_stats()
2097 		  cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);  in irdma_sc_gather_stats()
2103  * irdma_sc_manage_stats_inst - allocate or free stats instance
2122 		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id));  in irdma_sc_manage_stats_inst()
2123 	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |  in irdma_sc_manage_stats_inst()
2126 			  info->use_hmc_fcn_index) |  in irdma_sc_manage_stats_inst()
2127 	       FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) |  in irdma_sc_manage_stats_inst()
2142  * irdma_sc_set_up_map - set the up map table
2160 		temp |= (u64)info->map[i] << (i * 8);  in irdma_sc_set_up_map()
2164 		      FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) |  in irdma_sc_set_up_map()
2165 		      FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx));  in irdma_sc_set_up_map()
2167 	temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) |  in irdma_sc_set_up_map()
2168 	       FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) |  in irdma_sc_set_up_map()
2170 			  info->use_cnp_up_override) |  in irdma_sc_set_up_map()
2184  * irdma_sc_manage_ws_node - create/modify/destroy WS node
2203 		      FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) |  in irdma_sc_manage_ws_node()
2204 		      FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight));  in irdma_sc_manage_ws_node()
2206 	temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) |  in irdma_sc_manage_ws_node()
2208 	       FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) |  in irdma_sc_manage_ws_node()
2209 	       FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) |  in irdma_sc_manage_ws_node()
2210 	       FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) |  in irdma_sc_manage_ws_node()
2211 	       FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) |  in irdma_sc_manage_ws_node()
2213 	       FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) |  in irdma_sc_manage_ws_node()
2214 	       FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id);  in irdma_sc_manage_ws_node()
2227  * irdma_sc_qp_flush_wqes - flush qp's wqe
2243 	if (info->rq && !qp->flush_rq)  in irdma_sc_qp_flush_wqes()
2245 	if (info->sq && !qp->flush_sq)  in irdma_sc_qp_flush_wqes()
2247 	qp->flush_sq |= flush_sq;  in irdma_sc_qp_flush_wqes()
2248 	qp->flush_rq |= flush_rq;  in irdma_sc_qp_flush_wqes()
2251 		ibdev_dbg(to_ibdev(qp->dev),  in irdma_sc_qp_flush_wqes()
2253 			  qp->qp_uk.qp_id);  in irdma_sc_qp_flush_wqes()
2257 	cqp = qp->pd->dev->cqp;  in irdma_sc_qp_flush_wqes()
2262 	if (info->userflushcode) {  in irdma_sc_qp_flush_wqes()
2265 					   info->rq_minor_code) |  in irdma_sc_qp_flush_wqes()
2267 					   info->rq_major_code);  in irdma_sc_qp_flush_wqes()
2270 					   info->sq_minor_code) |  in irdma_sc_qp_flush_wqes()
2272 					   info->sq_major_code);  in irdma_sc_qp_flush_wqes()
2276 	temp = (info->generate_ae) ?  in irdma_sc_qp_flush_wqes()
2277 		info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,  in irdma_sc_qp_flush_wqes()
2278 					   info->ae_src) : 0;  in irdma_sc_qp_flush_wqes()
2281 	hdr = qp->qp_uk.qp_id |  in irdma_sc_qp_flush_wqes()
2283 	      FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) |  in irdma_sc_qp_flush_wqes()
2284 	      FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) |  in irdma_sc_qp_flush_wqes()
2287 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_qp_flush_wqes()
2301  * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
2316 	cqp = qp->pd->dev->cqp;  in irdma_sc_gen_ae()
2321 	temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,  in irdma_sc_gen_ae()
2322 					  info->ae_src);  in irdma_sc_gen_ae()
2325 	hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,  in irdma_sc_gen_ae()
2328 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_gen_ae()
2341 /*** irdma_sc_qp_upload_context - upload qp's context
2356 	cqp = dev->cqp;  in irdma_sc_qp_upload_context()
2361 	set_64bit_val(wqe, 16, info->buf_pa);  in irdma_sc_qp_upload_context()
2363 	hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) |  in irdma_sc_qp_upload_context()
2365 	      FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) |  in irdma_sc_qp_upload_context()
2366 	      FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) |  in irdma_sc_qp_upload_context()
2367 	      FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) |  in irdma_sc_qp_upload_context()
2368 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_qp_upload_context()
2382  * irdma_sc_manage_push_page - Handle push page
2396 	if (info->free_page &&  in irdma_sc_manage_push_page()
2397 	    info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)  in irdma_sc_manage_push_page()
2404 	set_64bit_val(wqe, 16, info->qs_handle);  in irdma_sc_manage_push_page()
2405 	hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) |  in irdma_sc_manage_push_page()
2406 	      FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) |  in irdma_sc_manage_push_page()
2408 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |  in irdma_sc_manage_push_page()
2409 	      FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page);  in irdma_sc_manage_push_page()
2423  * irdma_sc_suspend_qp - suspend qp for param change
2439 	hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) |  in irdma_sc_suspend_qp()
2441 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_suspend_qp()
2454  * irdma_sc_resume_qp - resume qp after suspend
2471 		      FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle));  in irdma_sc_resume_qp()
2473 	hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) |  in irdma_sc_resume_qp()
2475 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_resume_qp()
2488  * irdma_sc_cq_ack - acknowledge completion q
2493 	writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);  in irdma_sc_cq_ack()
2497  * irdma_sc_cq_init - initialize completion q
2507 	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_cq_init()
2508 	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)  in irdma_sc_cq_init()
2511 	cq->cq_pa = info->cq_base_pa;  in irdma_sc_cq_init()
2512 	cq->dev = info->dev;  in irdma_sc_cq_init()
2513 	cq->ceq_id = info->ceq_id;  in irdma_sc_cq_init()
2514 	info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;  in irdma_sc_cq_init()
2515 	info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;  in irdma_sc_cq_init()
2516 	ret_code = irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info);  in irdma_sc_cq_init()
2520 	cq->virtual_map = info->virtual_map;  in irdma_sc_cq_init()
2521 	cq->pbl_chunk_size = info->pbl_chunk_size;  in irdma_sc_cq_init()
2522 	cq->ceqe_mask = info->ceqe_mask;  in irdma_sc_cq_init()
2523 	cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;  in irdma_sc_cq_init()
2524 	cq->shadow_area_pa = info->shadow_area_pa;  in irdma_sc_cq_init()
2525 	cq->shadow_read_threshold = info->shadow_read_threshold;  in irdma_sc_cq_init()
2526 	cq->ceq_id_valid = info->ceq_id_valid;  in irdma_sc_cq_init()
2527 	cq->tph_en = info->tph_en;  in irdma_sc_cq_init()
2528 	cq->tph_val = info->tph_val;  in irdma_sc_cq_init()
2529 	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;  in irdma_sc_cq_init()
2530 	cq->vsi = info->vsi;  in irdma_sc_cq_init()
2536  * irdma_sc_cq_create - create completion q
2553 	cqp = cq->dev->cqp;  in irdma_sc_cq_create()
2554 	if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))  in irdma_sc_cq_create()
2557 	if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))  in irdma_sc_cq_create()
2560 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_create()
2561 	if (ceq && ceq->reg_cq)  in irdma_sc_cq_create()
2569 		if (ceq && ceq->reg_cq)  in irdma_sc_cq_create()
2574 	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);  in irdma_sc_cq_create()
2577 		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));  in irdma_sc_cq_create()
2578 	set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));  in irdma_sc_cq_create()
2579 	set_64bit_val(wqe, 40, cq->shadow_area_pa);  in irdma_sc_cq_create()
2581 		      FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));  in irdma_sc_cq_create()
2583 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |  in irdma_sc_cq_create()
2584 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));  in irdma_sc_cq_create()
2586 	hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |  in irdma_sc_cq_create()
2587 	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),  in irdma_sc_cq_create()
2590 	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |  in irdma_sc_cq_create()
2592 	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |  in irdma_sc_cq_create()
2593 	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |  in irdma_sc_cq_create()
2594 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |  in irdma_sc_cq_create()
2595 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |  in irdma_sc_cq_create()
2597 			 cq->cq_uk.avoid_mem_cflct) |  in irdma_sc_cq_create()
2598 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_cq_create()
2612  * irdma_sc_cq_destroy - destroy completion q
2625 	cqp = cq->dev->cqp;  in irdma_sc_cq_destroy()
2630 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_destroy()
2631 	if (ceq && ceq->reg_cq)  in irdma_sc_cq_destroy()
2634 	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);  in irdma_sc_cq_destroy()
2636 	set_64bit_val(wqe, 40, cq->shadow_area_pa);  in irdma_sc_cq_destroy()
2638 		      (cq->virtual_map ? cq->first_pm_pbl_idx : 0));  in irdma_sc_cq_destroy()
2640 	hdr = cq->cq_uk.cq_id |  in irdma_sc_cq_destroy()
2641 	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),  in irdma_sc_cq_destroy()
2644 	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |  in irdma_sc_cq_destroy()
2645 	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |  in irdma_sc_cq_destroy()
2646 	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |  in irdma_sc_cq_destroy()
2647 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |  in irdma_sc_cq_destroy()
2648 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |  in irdma_sc_cq_destroy()
2649 	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) |  in irdma_sc_cq_destroy()
2650 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_cq_destroy()
2664  * irdma_sc_cq_resize - set resized cq buffer info
2670 	cq->virtual_map = info->virtual_map;  in irdma_sc_cq_resize()
2671 	cq->cq_pa = info->cq_pa;  in irdma_sc_cq_resize()
2672 	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;  in irdma_sc_cq_resize()
2673 	cq->pbl_chunk_size = info->pbl_chunk_size;  in irdma_sc_cq_resize()
2674 	irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);  in irdma_sc_cq_resize()
2678  * irdma_sc_cq_modify - modify a Completion Queue
2693 	pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_cq_modify()
2694 	if (info->cq_resize && info->virtual_map &&  in irdma_sc_cq_modify()
2695 	    info->first_pm_pbl_idx >= pble_obj_cnt)  in irdma_sc_cq_modify()
2698 	cqp = cq->dev->cqp;  in irdma_sc_cq_modify()
2703 	set_64bit_val(wqe, 0, info->cq_size);  in irdma_sc_cq_modify()
2706 		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold));  in irdma_sc_cq_modify()
2707 	set_64bit_val(wqe, 32, info->cq_pa);  in irdma_sc_cq_modify()
2708 	set_64bit_val(wqe, 40, cq->shadow_area_pa);  in irdma_sc_cq_modify()
2709 	set_64bit_val(wqe, 48, info->first_pm_pbl_idx);  in irdma_sc_cq_modify()
2711 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |  in irdma_sc_cq_modify()
2712 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));  in irdma_sc_cq_modify()
2714 	hdr = cq->cq_uk.cq_id |  in irdma_sc_cq_modify()
2716 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) |  in irdma_sc_cq_modify()
2717 	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) |  in irdma_sc_cq_modify()
2718 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) |  in irdma_sc_cq_modify()
2719 	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) |  in irdma_sc_cq_modify()
2720 	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |  in irdma_sc_cq_modify()
2721 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |  in irdma_sc_cq_modify()
2723 			 cq->cq_uk.avoid_mem_cflct) |  in irdma_sc_cq_modify()
2724 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_cq_modify()
2738  * irdma_check_cqp_progress - check cqp processing progress
2744 	if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {  in irdma_check_cqp_progress()
2745 		timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];  in irdma_check_cqp_progress()
2746 		timeout->count = 0;  in irdma_check_cqp_progress()
2748 		if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=  in irdma_check_cqp_progress()
2749 		    timeout->compl_cqp_cmds)  in irdma_check_cqp_progress()
2750 			timeout->count++;  in irdma_check_cqp_progress()
2755  * irdma_get_cqp_reg_info - get head and tail for cqp using registers
2764 	*val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]);  in irdma_get_cqp_reg_info()
2770  * irdma_cqp_poll_registers - poll cqp registers
2784 			error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);  in irdma_cqp_poll_registers()
2785 			ibdev_dbg(to_ibdev(cqp->dev),  in irdma_cqp_poll_registers()
2792 			IRDMA_RING_MOVE_TAIL(cqp->sq_ring);  in irdma_cqp_poll_registers()
2793 			cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;  in irdma_cqp_poll_registers()
2796 		udelay(cqp->dev->hw_attrs.max_sleep_count);  in irdma_cqp_poll_registers()
2803  * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
2839  * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
2883 	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {  in irdma_sc_parse_fpm_commit_buf()
2921  * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
2945  * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
2964 	obj_info = hmc_info->hmc_obj;  in irdma_sc_parse_fpm_query_buf()
2967 	hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);  in irdma_sc_parse_fpm_query_buf()
2970 	hmc_fpm_misc->max_sds = max_pe_sds;  in irdma_sc_parse_fpm_query_buf()
2971 	hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;  in irdma_sc_parse_fpm_query_buf()
2994 	hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);  in irdma_sc_parse_fpm_query_buf()
2995 	if (!hmc_fpm_misc->xf_block_size)  in irdma_sc_parse_fpm_query_buf()
3003 	hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);  in irdma_sc_parse_fpm_query_buf()
3004 	if (!hmc_fpm_misc->q1_block_size)  in irdma_sc_parse_fpm_query_buf()
3014 	hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);  in irdma_sc_parse_fpm_query_buf()
3015 	hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);  in irdma_sc_parse_fpm_query_buf()
3016 	hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);  in irdma_sc_parse_fpm_query_buf()
3017 	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_sc_parse_fpm_query_buf()
3026 	hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);  in irdma_sc_parse_fpm_query_buf()
3027 	if (!hmc_fpm_misc->rrf_block_size &&  in irdma_sc_parse_fpm_query_buf()
3038 	hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);  in irdma_sc_parse_fpm_query_buf()
3039 	if (!hmc_fpm_misc->ooiscf_block_size &&  in irdma_sc_parse_fpm_query_buf()
3047  * irdma_sc_find_reg_cq - find cq ctx index
3056 	for (i = 0; i < ceq->reg_cq_size; i++) {  in irdma_sc_find_reg_cq()
3057 		if (cq == ceq->reg_cq[i])  in irdma_sc_find_reg_cq()
3065  * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
3074 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3076 	if (ceq->reg_cq_size == ceq->elem_cnt) {  in irdma_sc_add_cq_ctx()
3077 		spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3081 	ceq->reg_cq[ceq->reg_cq_size++] = cq;  in irdma_sc_add_cq_ctx()
3083 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3089  * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
3098 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3103 	ceq->reg_cq_size--;  in irdma_sc_remove_cq_ctx()
3104 	if (cq_ctx_idx != ceq->reg_cq_size)  in irdma_sc_remove_cq_ctx()
3105 		ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];  in irdma_sc_remove_cq_ctx()
3106 	ceq->reg_cq[ceq->reg_cq_size] = NULL;  in irdma_sc_remove_cq_ctx()
3109 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3113  * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
3124 	if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||  in irdma_sc_cqp_init()
3125 	    info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||  in irdma_sc_cqp_init()
3126 	    ((info->sq_size & (info->sq_size - 1))))  in irdma_sc_cqp_init()
3129 	hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size,  in irdma_sc_cqp_init()
3131 	cqp->size = sizeof(*cqp);  in irdma_sc_cqp_init()
3132 	cqp->sq_size = info->sq_size;  in irdma_sc_cqp_init()
3133 	cqp->hw_sq_size = hw_sq_size;  in irdma_sc_cqp_init()
3134 	cqp->sq_base = info->sq;  in irdma_sc_cqp_init()
3135 	cqp->host_ctx = info->host_ctx;  in irdma_sc_cqp_init()
3136 	cqp->sq_pa = info->sq_pa;  in irdma_sc_cqp_init()
3137 	cqp->host_ctx_pa = info->host_ctx_pa;  in irdma_sc_cqp_init()
3138 	cqp->dev = info->dev;  in irdma_sc_cqp_init()
3139 	cqp->struct_ver = info->struct_ver;  in irdma_sc_cqp_init()
3140 	cqp->hw_maj_ver = info->hw_maj_ver;  in irdma_sc_cqp_init()
3141 	cqp->hw_min_ver = info->hw_min_ver;  in irdma_sc_cqp_init()
3142 	cqp->scratch_array = info->scratch_array;  in irdma_sc_cqp_init()
3143 	cqp->polarity = 0;  in irdma_sc_cqp_init()
3144 	cqp->en_datacenter_tcp = info->en_datacenter_tcp;  in irdma_sc_cqp_init()
3145 	cqp->ena_vf_count = info->ena_vf_count;  in irdma_sc_cqp_init()
3146 	cqp->hmc_profile = info->hmc_profile;  in irdma_sc_cqp_init()
3147 	cqp->ceqs_per_vf = info->ceqs_per_vf;  in irdma_sc_cqp_init()
3148 	cqp->disable_packed = info->disable_packed;  in irdma_sc_cqp_init()
3149 	cqp->rocev2_rto_policy = info->rocev2_rto_policy;  in irdma_sc_cqp_init()
3150 	cqp->protocol_used = info->protocol_used;  in irdma_sc_cqp_init()
3151 	memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params));  in irdma_sc_cqp_init()
3152 	info->dev->cqp = cqp;  in irdma_sc_cqp_init()
3154 	IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);  in irdma_sc_cqp_init()
3155 	cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;  in irdma_sc_cqp_init()
3156 	cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;  in irdma_sc_cqp_init()
3158 	INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);  in irdma_sc_cqp_init()
3160 	writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]);  in irdma_sc_cqp_init()
3161 	writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]);  in irdma_sc_cqp_init()
3162 	writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);  in irdma_sc_cqp_init()
3164 	ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_cqp_init()
3166 		  cqp->sq_size, cqp->hw_sq_size, cqp->sq_base,  in irdma_sc_cqp_init()
3167 		  (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity);  in irdma_sc_cqp_init()
3172  * irdma_sc_cqp_create - create cqp during bringup
3185 	hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev;  in irdma_sc_cqp_create()
3186 	cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,  in irdma_sc_cqp_create()
3188 	cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device,  in irdma_sc_cqp_create()
3189 					   cqp->sdbuf.size, &cqp->sdbuf.pa,  in irdma_sc_cqp_create()
3191 	if (!cqp->sdbuf.va)  in irdma_sc_cqp_create()
3194 	spin_lock_init(&cqp->dev->cqp_lock);  in irdma_sc_cqp_create()
3196 	temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) |  in irdma_sc_cqp_create()
3197 	       FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) |  in irdma_sc_cqp_create()
3198 	       FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) |  in irdma_sc_cqp_create()
3199 	       FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf);  in irdma_sc_cqp_create()
3202 				   cqp->rocev2_rto_policy) |  in irdma_sc_cqp_create()
3204 				   cqp->protocol_used);  in irdma_sc_cqp_create()
3207 	set_64bit_val(cqp->host_ctx, 0, temp);  in irdma_sc_cqp_create()
3208 	set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);  in irdma_sc_cqp_create()
3210 	temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |  in irdma_sc_cqp_create()
3211 	       FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);  in irdma_sc_cqp_create()
3212 	set_64bit_val(cqp->host_ctx, 16, temp);  in irdma_sc_cqp_create()
3213 	set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);  in irdma_sc_cqp_create()
3214 	temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) |  in irdma_sc_cqp_create()
3215 	       FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver);  in irdma_sc_cqp_create()
3217 		temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) |  in irdma_sc_cqp_create()
3218 			FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor);  in irdma_sc_cqp_create()
3220 	set_64bit_val(cqp->host_ctx, 32, temp);  in irdma_sc_cqp_create()
3221 	set_64bit_val(cqp->host_ctx, 40, 0);  in irdma_sc_cqp_create()
3224 		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) |  in irdma_sc_cqp_create()
3225 			FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) |  in irdma_sc_cqp_create()
3226 			FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor);  in irdma_sc_cqp_create()
3228 	set_64bit_val(cqp->host_ctx, 48, temp);  in irdma_sc_cqp_create()
3231 		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) |  in irdma_sc_cqp_create()
3232 			FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) |  in irdma_sc_cqp_create()
3233 			FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) |  in irdma_sc_cqp_create()
3234 			FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod);  in irdma_sc_cqp_create()
3236 	set_64bit_val(cqp->host_ctx, 56, temp);  in irdma_sc_cqp_create()
3238 			     8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false);  in irdma_sc_cqp_create()
3239 	p1 = cqp->host_ctx_pa >> 32;  in irdma_sc_cqp_create()
3240 	p2 = (u32)cqp->host_ctx_pa;  in irdma_sc_cqp_create()
3242 	writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);  in irdma_sc_cqp_create()
3243 	writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]);  in irdma_sc_cqp_create()
3246 		if (cnt++ > cqp->dev->hw_attrs.max_done_count) {  in irdma_sc_cqp_create()
3250 		udelay(cqp->dev->hw_attrs.max_sleep_count);  in irdma_sc_cqp_create()
3251 		val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);  in irdma_sc_cqp_create()
3254 	if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {  in irdma_sc_cqp_create()
3259 	cqp->process_cqp_sds = irdma_update_sds_noccq;  in irdma_sc_cqp_create()
3263 	dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,  in irdma_sc_cqp_create()
3264 			  cqp->sdbuf.va, cqp->sdbuf.pa);  in irdma_sc_cqp_create()
3265 	cqp->sdbuf.va = NULL;  in irdma_sc_cqp_create()
3266 	err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);  in irdma_sc_cqp_create()
3273  * irdma_sc_cqp_post_sq - post of cqp's sq
3278 	writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);  in irdma_sc_cqp_post_sq()
3280 	ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_cqp_post_sq()
3282 		  cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);  in irdma_sc_cqp_post_sq()
3286  * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
3298 	if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {  in irdma_sc_cqp_get_next_send_wqe_idx()
3299 		ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_cqp_get_next_send_wqe_idx()
3301 			  cqp->sq_ring.head, cqp->sq_ring.tail,  in irdma_sc_cqp_get_next_send_wqe_idx()
3302 			  cqp->sq_ring.size);  in irdma_sc_cqp_get_next_send_wqe_idx()
3305 	IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);  in irdma_sc_cqp_get_next_send_wqe_idx()
3309 	cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;  in irdma_sc_cqp_get_next_send_wqe_idx()
3311 		cqp->polarity = !cqp->polarity;  in irdma_sc_cqp_get_next_send_wqe_idx()
3312 	wqe = cqp->sq_base[*wqe_idx].elem;  in irdma_sc_cqp_get_next_send_wqe_idx()
3313 	cqp->scratch_array[*wqe_idx] = scratch;  in irdma_sc_cqp_get_next_send_wqe_idx()
3320  * irdma_sc_cqp_destroy - destroy cqp during close
3328 	writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);  in irdma_sc_cqp_destroy()
3329 	writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);  in irdma_sc_cqp_destroy()
3331 		if (cnt++ > cqp->dev->hw_attrs.max_done_count) {  in irdma_sc_cqp_destroy()
3335 		udelay(cqp->dev->hw_attrs.max_sleep_count);  in irdma_sc_cqp_destroy()
3336 		val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);  in irdma_sc_cqp_destroy()
3337 	} while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));  in irdma_sc_cqp_destroy()
3339 	dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,  in irdma_sc_cqp_destroy()
3340 			  cqp->sdbuf.va, cqp->sdbuf.pa);  in irdma_sc_cqp_destroy()
3341 	cqp->sdbuf.va = NULL;  in irdma_sc_cqp_destroy()
3346  * irdma_sc_ccq_arm - enable intr for control cq
3356 	get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);  in irdma_sc_ccq_arm()
3365 	set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);  in irdma_sc_ccq_arm()
3369 	writel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db);  in irdma_sc_ccq_arm()
3373  * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
3388 	if (ccq->cq_uk.avoid_mem_cflct)  in irdma_sc_ccq_get_cqe_info()
3389 		cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);  in irdma_sc_ccq_get_cqe_info()
3391 		cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);  in irdma_sc_ccq_get_cqe_info()
3395 	if (polarity != ccq->cq_uk.polarity)  in irdma_sc_ccq_get_cqe_info()
3400 	info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);  in irdma_sc_ccq_get_cqe_info()
3401 	info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;  in irdma_sc_ccq_get_cqe_info()
3402 	info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);  in irdma_sc_ccq_get_cqe_info()
3403 	if (info->error) {  in irdma_sc_ccq_get_cqe_info()
3404 		info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);  in irdma_sc_ccq_get_cqe_info()
3405 		error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);  in irdma_sc_ccq_get_cqe_info()
3406 		ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_ccq_get_cqe_info()
3411 	info->scratch = cqp->scratch_array[wqe_idx];  in irdma_sc_ccq_get_cqe_info()
3414 	info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);  in irdma_sc_ccq_get_cqe_info()
3415 	get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);  in irdma_sc_ccq_get_cqe_info()
3416 	info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);  in irdma_sc_ccq_get_cqe_info()
3417 	info->cqp = cqp;  in irdma_sc_ccq_get_cqe_info()
3420 	IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);  in irdma_sc_ccq_get_cqe_info()
3421 	if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring))  in irdma_sc_ccq_get_cqe_info()
3422 		ccq->cq_uk.polarity ^= 1;  in irdma_sc_ccq_get_cqe_info()
3425 	IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);  in irdma_sc_ccq_get_cqe_info()
3426 	set_64bit_val(ccq->cq_uk.shadow_area, 0,  in irdma_sc_ccq_get_cqe_info()
3427 		      IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));  in irdma_sc_ccq_get_cqe_info()
3431 	IRDMA_RING_MOVE_TAIL(cqp->sq_ring);  in irdma_sc_ccq_get_cqe_info()
3432 	ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;  in irdma_sc_ccq_get_cqe_info()
3438  * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
3451 	ccq = cqp->dev->ccq;  in irdma_sc_poll_for_cqp_op_done()
3453 		if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)  in irdma_sc_poll_for_cqp_op_done()
3457 			udelay(cqp->dev->hw_attrs.max_sleep_count);  in irdma_sc_poll_for_cqp_op_done()
3467 		ibdev_dbg(to_ibdev(cqp->dev),  in irdma_sc_poll_for_cqp_op_done()
3479  * irdma_sc_manage_hmc_pm_func_table - manage of function table
3505 	hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) |  in irdma_sc_manage_hmc_pm_func_table()
3508 	      FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) |  in irdma_sc_manage_hmc_pm_func_table()
3509 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_manage_hmc_pm_func_table()
3524  * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion
3536  * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values
3559 	set_64bit_val(wqe, 32, commit_fpm_mem->pa);  in irdma_sc_commit_fpm_val()
3563 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_commit_fpm_val()
3577 							    cqp->dev->hw_attrs.max_done_count);  in irdma_sc_commit_fpm_val()
3586  * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for
3598  * irdma_sc_query_fpm_val - cqp wqe query fpm values
3621 	set_64bit_val(wqe, 32, query_fpm_mem->pa);  in irdma_sc_query_fpm_val()
3624 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_query_fpm_val()
3637 							    cqp->dev->hw_attrs.max_done_count);  in irdma_sc_query_fpm_val()
3646  * irdma_sc_ceq_init - initialize ceq
3655 	if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||  in irdma_sc_ceq_init()
3656 	    info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)  in irdma_sc_ceq_init()
3659 	if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))  in irdma_sc_ceq_init()
3661 	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_ceq_init()
3663 	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)  in irdma_sc_ceq_init()
3666 	ceq->size = sizeof(*ceq);  in irdma_sc_ceq_init()
3667 	ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;  in irdma_sc_ceq_init()
3668 	ceq->ceq_id = info->ceq_id;  in irdma_sc_ceq_init()
3669 	ceq->dev = info->dev;  in irdma_sc_ceq_init()
3670 	ceq->elem_cnt = info->elem_cnt;  in irdma_sc_ceq_init()
3671 	ceq->ceq_elem_pa = info->ceqe_pa;  in irdma_sc_ceq_init()
3672 	ceq->virtual_map = info->virtual_map;  in irdma_sc_ceq_init()
3673 	ceq->itr_no_expire = info->itr_no_expire;  in irdma_sc_ceq_init()
3674 	ceq->reg_cq = info->reg_cq;  in irdma_sc_ceq_init()
3675 	ceq->reg_cq_size = 0;  in irdma_sc_ceq_init()
3676 	spin_lock_init(&ceq->req_cq_lock);  in irdma_sc_ceq_init()
3677 	ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);  in irdma_sc_ceq_init()
3678 	ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);  in irdma_sc_ceq_init()
3679 	ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);  in irdma_sc_ceq_init()
3680 	ceq->tph_en = info->tph_en;  in irdma_sc_ceq_init()
3681 	ceq->tph_val = info->tph_val;  in irdma_sc_ceq_init()
3682 	ceq->vsi = info->vsi;  in irdma_sc_ceq_init()
3683 	ceq->polarity = 1;  in irdma_sc_ceq_init()
3684 	IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);  in irdma_sc_ceq_init()
3685 	ceq->dev->ceq[info->ceq_id] = ceq;  in irdma_sc_ceq_init()
3691  * irdma_sc_ceq_create - create ceq wqe
3704 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_create()
3708 	set_64bit_val(wqe, 16, ceq->elem_cnt);  in irdma_sc_ceq_create()
3710 		      (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));  in irdma_sc_ceq_create()
3712 		      (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));  in irdma_sc_ceq_create()
3714 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |  in irdma_sc_ceq_create()
3715 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));  in irdma_sc_ceq_create()
3716 	hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |  in irdma_sc_ceq_create()
3718 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_create()
3719 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_create()
3720 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |  in irdma_sc_ceq_create()
3721 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_create()
3722 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_ceq_create()
3736  * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
3744 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_create_done()
3750  * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
3757 	if (ceq->reg_cq)  in irdma_sc_cceq_destroy_done()
3758 		irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_destroy_done()
3760 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_destroy_done()
3761 	cqp->process_cqp_sds = irdma_update_sds_noccq;  in irdma_sc_cceq_destroy_done()
3768  * irdma_sc_cceq_create - create cceq
3775 	struct irdma_sc_dev *dev = ceq->dev;  in irdma_sc_cceq_create()
3777 	dev->ccq->vsi = ceq->vsi;  in irdma_sc_cceq_create()
3778 	if (ceq->reg_cq) {  in irdma_sc_cceq_create()
3779 		ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_create()
3792  * irdma_sc_ceq_destroy - destroy ceq
3804 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_destroy()
3809 	set_64bit_val(wqe, 16, ceq->elem_cnt);  in irdma_sc_ceq_destroy()
3810 	set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);  in irdma_sc_ceq_destroy()
3811 	hdr = ceq->ceq_id |  in irdma_sc_ceq_destroy()
3813 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_destroy()
3814 	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_destroy()
3815 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_destroy()
3816 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_ceq_destroy()
3830  * irdma_sc_process_ceq - process ceq
3852 		if (polarity != ceq->polarity)  in irdma_sc_process_ceq()
3858 			IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3860 			if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3861 				ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3866 		if (ceq->reg_cq) {  in irdma_sc_process_ceq()
3867 			spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3869 			spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3872 		IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3873 		if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3874 			ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3883  * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq
3894 	u8 ceq_polarity = ceq->polarity;  in irdma_sc_cleanup_ceqes()
3901 	next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);  in irdma_sc_cleanup_ceqes()
3915 		next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);  in irdma_sc_cleanup_ceqes()
3922  * irdma_sc_aeq_init - initialize aeq
3931 	if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||  in irdma_sc_aeq_init()
3932 	    info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)  in irdma_sc_aeq_init()
3935 	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_aeq_init()
3937 	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)  in irdma_sc_aeq_init()
3940 	aeq->size = sizeof(*aeq);  in irdma_sc_aeq_init()
3941 	aeq->polarity = 1;  in irdma_sc_aeq_init()
3942 	aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;  in irdma_sc_aeq_init()
3943 	aeq->dev = info->dev;  in irdma_sc_aeq_init()
3944 	aeq->elem_cnt = info->elem_cnt;  in irdma_sc_aeq_init()
3945 	aeq->aeq_elem_pa = info->aeq_elem_pa;  in irdma_sc_aeq_init()
3946 	IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);  in irdma_sc_aeq_init()
3947 	aeq->virtual_map = info->virtual_map;  in irdma_sc_aeq_init()
3948 	aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);  in irdma_sc_aeq_init()
3949 	aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);  in irdma_sc_aeq_init()
3950 	aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);  in irdma_sc_aeq_init()
3951 	aeq->msix_idx = info->msix_idx;  in irdma_sc_aeq_init()
3952 	info->dev->aeq = aeq;  in irdma_sc_aeq_init()
3958  * irdma_sc_aeq_create - create aeq
3970 	cqp = aeq->dev->cqp;  in irdma_sc_aeq_create()
3974 	set_64bit_val(wqe, 16, aeq->elem_cnt);  in irdma_sc_aeq_create()
3976 		      (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));  in irdma_sc_aeq_create()
3978 		      (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));  in irdma_sc_aeq_create()
3981 	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |  in irdma_sc_aeq_create()
3982 	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |  in irdma_sc_aeq_create()
3983 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_aeq_create()
3997  * irdma_sc_aeq_destroy - destroy aeq during close
4010 	dev = aeq->dev;  in irdma_sc_aeq_destroy()
4011 	writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]);  in irdma_sc_aeq_destroy()
4013 	cqp = dev->cqp;  in irdma_sc_aeq_destroy()
4017 	set_64bit_val(wqe, 16, aeq->elem_cnt);  in irdma_sc_aeq_destroy()
4018 	set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);  in irdma_sc_aeq_destroy()
4020 	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |  in irdma_sc_aeq_destroy()
4021 	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |  in irdma_sc_aeq_destroy()
4022 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_aeq_destroy()
4035  * irdma_sc_get_next_aeqe - get next aeq entry
4053 	if (aeq->polarity != polarity)  in irdma_sc_get_next_aeqe()
4061 	info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |  in irdma_sc_get_next_aeqe()
4063 	info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);  in irdma_sc_get_next_aeqe()
4064 	info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);  in irdma_sc_get_next_aeqe()
4065 	info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);  in irdma_sc_get_next_aeqe()
4066 	info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);  in irdma_sc_get_next_aeqe()
4067 	info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp);  in irdma_sc_get_next_aeqe()
4069 	info->ae_src = ae_src;  in irdma_sc_get_next_aeqe()
4070 	switch (info->ae_id) {  in irdma_sc_get_next_aeqe()
4111 		info->qp = true;  in irdma_sc_get_next_aeqe()
4112 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4115 		info->cq = true;  in irdma_sc_get_next_aeqe()
4116 		info->compl_ctx = compl_ctx << 1;  in irdma_sc_get_next_aeqe()
4131 		info->qp = true;  in irdma_sc_get_next_aeqe()
4132 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4141 		info->qp = true;  in irdma_sc_get_next_aeqe()
4142 		info->rq = true;  in irdma_sc_get_next_aeqe()
4143 		info->wqe_idx = wqe_idx;  in irdma_sc_get_next_aeqe()
4144 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4150 		info->cq = true;  in irdma_sc_get_next_aeqe()
4151 		info->compl_ctx = compl_ctx << 1;  in irdma_sc_get_next_aeqe()
4155 		info->qp = true;  in irdma_sc_get_next_aeqe()
4156 		info->sq = true;  in irdma_sc_get_next_aeqe()
4157 		info->wqe_idx = wqe_idx;  in irdma_sc_get_next_aeqe()
4158 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4162 		info->qp = true;  in irdma_sc_get_next_aeqe()
4163 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4164 		info->in_rdrsp_wr = true;  in irdma_sc_get_next_aeqe()
4168 		info->qp = true;  in irdma_sc_get_next_aeqe()
4169 		info->compl_ctx = compl_ctx;  in irdma_sc_get_next_aeqe()
4170 		info->out_rdrsp = true;  in irdma_sc_get_next_aeqe()
4177 	IRDMA_RING_MOVE_TAIL(aeq->aeq_ring);  in irdma_sc_get_next_aeqe()
4178 	if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring))  in irdma_sc_get_next_aeqe()
4179 		aeq->polarity ^= 1;  in irdma_sc_get_next_aeqe()
4185  * irdma_sc_repost_aeq_entries - repost completed aeq entries
4191 	writel(count, dev->hw_regs[IRDMA_AEQALLOC]);  in irdma_sc_repost_aeq_entries()
4195  * irdma_sc_ccq_init - initialize control cq
4204 	if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||  in irdma_sc_ccq_init()
4205 	    info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)  in irdma_sc_ccq_init()
4208 	if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))  in irdma_sc_ccq_init()
4211 	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;  in irdma_sc_ccq_init()
4213 	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)  in irdma_sc_ccq_init()
4216 	cq->cq_pa = info->cq_pa;  in irdma_sc_ccq_init()
4217 	cq->cq_uk.cq_base = info->cq_base;  in irdma_sc_ccq_init()
4218 	cq->shadow_area_pa = info->shadow_area_pa;  in irdma_sc_ccq_init()
4219 	cq->cq_uk.shadow_area = info->shadow_area;  in irdma_sc_ccq_init()
4220 	cq->shadow_read_threshold = info->shadow_read_threshold;  in irdma_sc_ccq_init()
4221 	cq->dev = info->dev;  in irdma_sc_ccq_init()
4222 	cq->ceq_id = info->ceq_id;  in irdma_sc_ccq_init()
4223 	cq->cq_uk.cq_size = info->num_elem;  in irdma_sc_ccq_init()
4224 	cq->cq_type = IRDMA_CQ_TYPE_CQP;  in irdma_sc_ccq_init()
4225 	cq->ceqe_mask = info->ceqe_mask;  in irdma_sc_ccq_init()
4226 	IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);  in irdma_sc_ccq_init()
4227 	cq->cq_uk.cq_id = 0; /* control cq is id 0 always */  in irdma_sc_ccq_init()
4228 	cq->ceq_id_valid = info->ceq_id_valid;  in irdma_sc_ccq_init()
4229 	cq->tph_en = info->tph_en;  in irdma_sc_ccq_init()
4230 	cq->tph_val = info->tph_val;  in irdma_sc_ccq_init()
4231 	cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;  in irdma_sc_ccq_init()
4232 	cq->pbl_list = info->pbl_list;  in irdma_sc_ccq_init()
4233 	cq->virtual_map = info->virtual_map;  in irdma_sc_ccq_init()
4234 	cq->pbl_chunk_size = info->pbl_chunk_size;  in irdma_sc_ccq_init()
4235 	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;  in irdma_sc_ccq_init()
4236 	cq->cq_uk.polarity = true;  in irdma_sc_ccq_init()
4237 	cq->vsi = info->vsi;  in irdma_sc_ccq_init()
4238 	cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;  in irdma_sc_ccq_init()
4241 	cq->cq_uk.cqe_alloc_db = NULL;  in irdma_sc_ccq_init()
4243 	info->dev->ccq = cq;  in irdma_sc_ccq_init()
4248  * irdma_sc_ccq_create_done - poll cqp for ccq create
4255 	cqp = ccq->dev->cqp;  in irdma_sc_ccq_create_done()
4261  * irdma_sc_ccq_create - create control cq
4281 	ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;  in irdma_sc_ccq_create()
4287  * irdma_sc_ccq_destroy - destroy ccq during close
4301 	cqp = ccq->dev->cqp;  in irdma_sc_ccq_destroy()
4306 	set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);  in irdma_sc_ccq_destroy()
4308 	set_64bit_val(wqe, 40, ccq->shadow_area_pa);  in irdma_sc_ccq_destroy()
4310 	hdr = ccq->cq_uk.cq_id |  in irdma_sc_ccq_destroy()
4311 	      FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0),  in irdma_sc_ccq_destroy()
4314 	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) |  in irdma_sc_ccq_destroy()
4315 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) |  in irdma_sc_ccq_destroy()
4316 	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) |  in irdma_sc_ccq_destroy()
4317 	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) |  in irdma_sc_ccq_destroy()
4318 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_ccq_destroy()
4330 						    cqp->dev->hw_attrs.max_done_count);  in irdma_sc_ccq_destroy()
4333 	cqp->process_cqp_sds = irdma_update_sds_noccq;  in irdma_sc_ccq_destroy()
4339  * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
4352 	hmc_info = dev->hmc_info;  in irdma_sc_init_iw_hmc()
4353 	hmc_fpm_misc = &dev->hmc_fpm_misc;  in irdma_sc_init_iw_hmc()
4354 	query_fpm_mem.pa = dev->fpm_query_buf_pa;  in irdma_sc_init_iw_hmc()
4355 	query_fpm_mem.va = dev->fpm_query_buf;  in irdma_sc_init_iw_hmc()
4356 	hmc_info->hmc_fn_id = hmc_fn_id;  in irdma_sc_init_iw_hmc()
4359 	ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,  in irdma_sc_init_iw_hmc()
4375  * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp
4390 	hmc_info = dev->hmc_info;  in irdma_sc_cfg_iw_fpm()
4391 	obj_info = hmc_info->hmc_obj;  in irdma_sc_cfg_iw_fpm()
4392 	buf = dev->fpm_commit_buf;  in irdma_sc_cfg_iw_fpm()
4424 	commit_fpm_mem.pa = dev->fpm_commit_buf_pa;  in irdma_sc_cfg_iw_fpm()
4425 	commit_fpm_mem.va = dev->fpm_commit_buf;  in irdma_sc_cfg_iw_fpm()
4431 	ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,  in irdma_sc_cfg_iw_fpm()
4434 		irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,  in irdma_sc_cfg_iw_fpm()
4435 					      hmc_info->hmc_obj,  in irdma_sc_cfg_iw_fpm()
4436 					      &hmc_info->sd_table.sd_cnt);  in irdma_sc_cfg_iw_fpm()
4445  * cqp_sds_wqe_fill - fill cqp wqe doe sd
4458 	struct irdma_dma_mem *sdbuf = &cqp->sdbuf;  in cqp_sds_wqe_fill()
4466 	wqe_entries = (info->cnt > 3) ? 3 : info->cnt;  in cqp_sds_wqe_fill()
4467 	mem_entries = info->cnt - wqe_entries;  in cqp_sds_wqe_fill()
4471 		memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);  in cqp_sds_wqe_fill()
4473 		data = (u64)sdbuf->pa + offset;  in cqp_sds_wqe_fill()
4477 	data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id);  in cqp_sds_wqe_fill()
4483 			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) |  in cqp_sds_wqe_fill()
4486 		set_64bit_val(wqe, 56, info->entry[2].data);  in cqp_sds_wqe_fill()
4490 			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) |  in cqp_sds_wqe_fill()
4493 		set_64bit_val(wqe, 40, info->entry[1].data);  in cqp_sds_wqe_fill()
4497 			      FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd));  in cqp_sds_wqe_fill()
4499 		set_64bit_val(wqe, 8, info->entry[0].data);  in cqp_sds_wqe_fill()
4506 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |  in cqp_sds_wqe_fill()
4515 				     (char *)sdbuf->va + offset,  in cqp_sds_wqe_fill()
4525  * irdma_update_pe_sds - cqp wqe for sd
4534 	struct irdma_sc_cqp *cqp = dev->cqp;  in irdma_update_pe_sds()
4545  * irdma_update_sds_noccq - update sd before ccq created
4554 	struct irdma_sc_cqp *cqp = dev->cqp;  in irdma_update_sds_noccq()
4565 					cqp->dev->hw_attrs.max_done_count);  in irdma_update_sds_noccq()
4569  * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
4594 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);  in irdma_sc_static_hmc_pages_allocated()
4609 							cqp->dev->hw_attrs.max_done_count);  in irdma_sc_static_hmc_pages_allocated()
4620  * irdma_cqp_ring_full - check if cqp ring is full
4625 	return IRDMA_RING_FULL_ERR(cqp->sq_ring);  in irdma_cqp_ring_full()
4629  * irdma_est_sd - returns approximate number of SDs for HMC
4642 			size += round_up(hmc_info->hmc_obj[i].cnt *  in irdma_est_sd()
4643 					 hmc_info->hmc_obj[i].size, 512);  in irdma_est_sd()
4644 	size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *  in irdma_est_sd()
4645 			 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);  in irdma_est_sd()
4652 		sd = 0xFFFFFFFF - 1;  in irdma_est_sd()
4659  * irdma_sc_query_rdma_features_done - poll cqp for query features done
4671  * irdma_sc_query_rdma_features - query RDMA features and FW ver
4687 	temp = buf->pa;  in irdma_sc_query_rdma_features()
4691 			  cqp->polarity) |  in irdma_sc_query_rdma_features()
4692 	       FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) |  in irdma_sc_query_rdma_features()
4706  * irdma_get_rdma_features - get RDMA features
4718 	feat_buf.va = dma_alloc_coherent(dev->hw->device, feat_buf.size,  in irdma_get_rdma_features()
4723 	ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);  in irdma_get_rdma_features()
4725 		ret_code = irdma_sc_query_rdma_features_done(dev->cqp);  in irdma_get_rdma_features()
4737 		dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,  in irdma_get_rdma_features()
4742 		feat_buf.va = dma_alloc_coherent(dev->hw->device,  in irdma_get_rdma_features()
4748 		ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);  in irdma_get_rdma_features()
4750 			ret_code = irdma_sc_query_rdma_features_done(dev->cqp);  in irdma_get_rdma_features()
4775 		dev->feature_info[feat_type] = temp;  in irdma_get_rdma_features()
4778 	dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,  in irdma_get_rdma_features()
4789 	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {  in irdma_q1_cnt()
4790 		q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted);  in irdma_q1_cnt()
4792 		if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)  in irdma_q1_cnt()
4793 			q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512);  in irdma_q1_cnt()
4795 			q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted;  in irdma_q1_cnt()
4804 	hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes);  in cfg_fpm_value_gen_1()
4810 	struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;  in cfg_fpm_value_gen_2()
4812 	hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =  in cfg_fpm_value_gen_2()
4813 		4 * hmc_fpm_misc->xf_block_size * qpwanted;  in cfg_fpm_value_gen_2()
4815 	hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;  in cfg_fpm_value_gen_2()
4817 	if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)  in cfg_fpm_value_gen_2()
4818 		hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;  in cfg_fpm_value_gen_2()
4819 	if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)  in cfg_fpm_value_gen_2()
4820 		hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =  in cfg_fpm_value_gen_2()
4821 			hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /  in cfg_fpm_value_gen_2()
4822 			hmc_fpm_misc->rrf_block_size;  in cfg_fpm_value_gen_2()
4823 	if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)  in cfg_fpm_value_gen_2()
4824 		hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;  in cfg_fpm_value_gen_2()
4825 	if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)  in cfg_fpm_value_gen_2()
4826 		hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =  in cfg_fpm_value_gen_2()
4827 			hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /  in cfg_fpm_value_gen_2()
4828 			hmc_fpm_misc->ooiscf_block_size;  in cfg_fpm_value_gen_2()
4832  * irdma_cfg_fpm_val - configure HMC objects
4849 	hmc_info = dev->hmc_info;  in irdma_cfg_fpm_val()
4850 	hmc_fpm_misc = &dev->hmc_fpm_misc;  in irdma_cfg_fpm_val()
4852 	ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);  in irdma_cfg_fpm_val()
4861 		hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;  in irdma_cfg_fpm_val()
4865 		  sd_needed, hmc_info->first_sd_index);  in irdma_cfg_fpm_val()
4867 		  hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds);  in irdma_cfg_fpm_val()
4869 	qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);  in irdma_cfg_fpm_val()
4877 	mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;  in irdma_cfg_fpm_val()
4878 	pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;  in irdma_cfg_fpm_val()
4882 		  qp_count, hmc_fpm_misc->max_sds,  in irdma_cfg_fpm_val()
4883 		  hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,  in irdma_cfg_fpm_val()
4884 		  hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,  in irdma_cfg_fpm_val()
4885 		  hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,  in irdma_cfg_fpm_val()
4886 		  hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,  in irdma_cfg_fpm_val()
4887 		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,  in irdma_cfg_fpm_val()
4888 		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);  in irdma_cfg_fpm_val()
4889 	hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =  in irdma_cfg_fpm_val()
4890 		hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;  in irdma_cfg_fpm_val()
4891 	hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =  in irdma_cfg_fpm_val()
4892 		hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;  in irdma_cfg_fpm_val()
4893 	hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =  in irdma_cfg_fpm_val()
4894 		hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;  in irdma_cfg_fpm_val()
4896 	hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;  in irdma_cfg_fpm_val()
4898 	while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)  in irdma_cfg_fpm_val()
4903 		hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;  in irdma_cfg_fpm_val()
4904 		hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =  in irdma_cfg_fpm_val()
4905 			min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);  in irdma_cfg_fpm_val()
4906 		hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */  in irdma_cfg_fpm_val()
4907 		hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;  in irdma_cfg_fpm_val()
4909 		hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);  in irdma_cfg_fpm_val()
4913 		hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =  in irdma_cfg_fpm_val()
4914 			powerof2 * hmc_fpm_misc->ht_multiplier;  in irdma_cfg_fpm_val()
4915 		if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_cfg_fpm_val()
4920 		hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted);  in irdma_cfg_fpm_val()
4921 		hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =  in irdma_cfg_fpm_val()
4922 			hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;  in irdma_cfg_fpm_val()
4923 		hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =  in irdma_cfg_fpm_val()
4924 			hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;  in irdma_cfg_fpm_val()
4925 		hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =  in irdma_cfg_fpm_val()
4926 			(round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;  in irdma_cfg_fpm_val()
4928 		hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;  in irdma_cfg_fpm_val()
4931 			  "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",  in irdma_cfg_fpm_val()
4932 			  sd_needed, hmc_fpm_misc->max_sds, mrwanted,  in irdma_cfg_fpm_val()
4936 		if (sd_needed <= hmc_fpm_misc->max_sds)  in irdma_cfg_fpm_val()
4939 		sd_diff = sd_needed - hmc_fpm_misc->max_sds;  in irdma_cfg_fpm_val()
4947 		if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&  in irdma_cfg_fpm_val()
4949 			pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;  in irdma_cfg_fpm_val()
4952 			pblewanted -= 10 * FPM_MULTIPLIER;  in irdma_cfg_fpm_val()
4954 			pblewanted -= FPM_MULTIPLIER;  in irdma_cfg_fpm_val()
4956 			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)  in irdma_cfg_fpm_val()
4957 				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;  in irdma_cfg_fpm_val()
4958 			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)  in irdma_cfg_fpm_val()
4959 				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;  in irdma_cfg_fpm_val()
4962 			mrwanted -= FPM_MULTIPLIER;  in irdma_cfg_fpm_val()
4965 			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)  in irdma_cfg_fpm_val()
4966 				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;  in irdma_cfg_fpm_val()
4970 	if (sd_needed > hmc_fpm_misc->max_sds) {  in irdma_cfg_fpm_val()
4973 			  loop_count, sd_needed, hmc_info->sd_table.sd_cnt);  in irdma_cfg_fpm_val()
4977 	if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {  in irdma_cfg_fpm_val()
4978 		pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 *  in irdma_cfg_fpm_val()
4980 		hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;  in irdma_cfg_fpm_val()
4987 		  hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,  in irdma_cfg_fpm_val()
4988 		  hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,  in irdma_cfg_fpm_val()
4989 		  hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,  in irdma_cfg_fpm_val()
4990 		  hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,  in irdma_cfg_fpm_val()
4991 		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,  in irdma_cfg_fpm_val()
4992 		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,  in irdma_cfg_fpm_val()
4993 		  hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index);  in irdma_cfg_fpm_val()
4995 	ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);  in irdma_cfg_fpm_val()
4999 			  readl(dev->hw_regs[IRDMA_CQPERRCODES]));  in irdma_cfg_fpm_val()
5004 		   (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);  in irdma_cfg_fpm_val()
5012 	hmc_info->sd_table.sd_entry = virt_mem.va;  in irdma_cfg_fpm_val()
5018  * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
5029 	dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;  in irdma_exec_cqp_cmd()
5030 	switch (pcmdinfo->cqp_cmd) {  in irdma_exec_cqp_cmd()
5032 		status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,  in irdma_exec_cqp_cmd()
5033 					      pcmdinfo->in.u.ceq_destroy.scratch,  in irdma_exec_cqp_cmd()
5034 					      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5037 		status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,  in irdma_exec_cqp_cmd()
5038 					      pcmdinfo->in.u.aeq_destroy.scratch,  in irdma_exec_cqp_cmd()
5039 					      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5043 		status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,  in irdma_exec_cqp_cmd()
5044 					     pcmdinfo->in.u.ceq_create.scratch,  in irdma_exec_cqp_cmd()
5045 					     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5048 		status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,  in irdma_exec_cqp_cmd()
5049 					     pcmdinfo->in.u.aeq_create.scratch,  in irdma_exec_cqp_cmd()
5050 					     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5053 		status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,  in irdma_exec_cqp_cmd()
5054 						    &pcmdinfo->in.u.qp_upload_context.info,  in irdma_exec_cqp_cmd()
5055 						    pcmdinfo->in.u.qp_upload_context.scratch,  in irdma_exec_cqp_cmd()
5056 						    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5059 		status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,  in irdma_exec_cqp_cmd()
5060 					    pcmdinfo->in.u.cq_create.scratch,  in irdma_exec_cqp_cmd()
5061 					    pcmdinfo->in.u.cq_create.check_overflow,  in irdma_exec_cqp_cmd()
5062 					    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5065 		status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq,  in irdma_exec_cqp_cmd()
5066 					    &pcmdinfo->in.u.cq_modify.info,  in irdma_exec_cqp_cmd()
5067 					    pcmdinfo->in.u.cq_modify.scratch,  in irdma_exec_cqp_cmd()
5068 					    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5071 		status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,  in irdma_exec_cqp_cmd()
5072 					     pcmdinfo->in.u.cq_destroy.scratch,  in irdma_exec_cqp_cmd()
5073 					     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5076 		status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,  in irdma_exec_cqp_cmd()
5077 						&pcmdinfo->in.u.qp_flush_wqes.info,  in irdma_exec_cqp_cmd()
5078 						pcmdinfo->in.u.qp_flush_wqes.scratch,  in irdma_exec_cqp_cmd()
5079 						pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5082 		status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,  in irdma_exec_cqp_cmd()
5083 					 &pcmdinfo->in.u.gen_ae.info,  in irdma_exec_cqp_cmd()
5084 					 pcmdinfo->in.u.gen_ae.scratch,  in irdma_exec_cqp_cmd()
5085 					 pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5088 		status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,  in irdma_exec_cqp_cmd()
5089 						   &pcmdinfo->in.u.manage_push_page.info,  in irdma_exec_cqp_cmd()
5090 						   pcmdinfo->in.u.manage_push_page.scratch,  in irdma_exec_cqp_cmd()
5091 						   pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5094 		status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,  in irdma_exec_cqp_cmd()
5095 					     &pcmdinfo->in.u.update_pe_sds.info,  in irdma_exec_cqp_cmd()
5096 					     pcmdinfo->in.u.update_pe_sds.scratch);  in irdma_exec_cqp_cmd()
5101 			irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,  in irdma_exec_cqp_cmd()
5102 							  &pcmdinfo->in.u.manage_hmc_pm.info,  in irdma_exec_cqp_cmd()
5103 							  pcmdinfo->in.u.manage_hmc_pm.scratch,  in irdma_exec_cqp_cmd()
5107 		status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,  in irdma_exec_cqp_cmd()
5108 					     pcmdinfo->in.u.suspend_resume.qp,  in irdma_exec_cqp_cmd()
5109 					     pcmdinfo->in.u.suspend_resume.scratch);  in irdma_exec_cqp_cmd()
5112 		status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,  in irdma_exec_cqp_cmd()
5113 					    pcmdinfo->in.u.suspend_resume.qp,  in irdma_exec_cqp_cmd()
5114 					    pcmdinfo->in.u.suspend_resume.scratch);  in irdma_exec_cqp_cmd()
5117 		val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;  in irdma_exec_cqp_cmd()
5118 		val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;  in irdma_exec_cqp_cmd()
5119 		status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,  in irdma_exec_cqp_cmd()
5120 						pcmdinfo->in.u.query_fpm_val.scratch,  in irdma_exec_cqp_cmd()
5121 						pcmdinfo->in.u.query_fpm_val.hmc_fn_id,  in irdma_exec_cqp_cmd()
5125 		val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;  in irdma_exec_cqp_cmd()
5126 		val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;  in irdma_exec_cqp_cmd()
5127 		status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,  in irdma_exec_cqp_cmd()
5128 						 pcmdinfo->in.u.commit_fpm_val.scratch,  in irdma_exec_cqp_cmd()
5129 						 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,  in irdma_exec_cqp_cmd()
5138 		status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,  in irdma_exec_cqp_cmd()
5139 						    &pcmdinfo->in.u.stats_manage.info,  in irdma_exec_cqp_cmd()
5141 						    pcmdinfo->in.u.stats_manage.scratch);  in irdma_exec_cqp_cmd()
5144 		status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,  in irdma_exec_cqp_cmd()
5145 					       &pcmdinfo->in.u.stats_gather.info,  in irdma_exec_cqp_cmd()
5146 					       pcmdinfo->in.u.stats_gather.scratch);  in irdma_exec_cqp_cmd()
5149 		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,  in irdma_exec_cqp_cmd()
5150 						 &pcmdinfo->in.u.ws_node.info,  in irdma_exec_cqp_cmd()
5152 						 pcmdinfo->in.u.ws_node.scratch);  in irdma_exec_cqp_cmd()
5155 		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,  in irdma_exec_cqp_cmd()
5156 						 &pcmdinfo->in.u.ws_node.info,  in irdma_exec_cqp_cmd()
5158 						 pcmdinfo->in.u.ws_node.scratch);  in irdma_exec_cqp_cmd()
5161 		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,  in irdma_exec_cqp_cmd()
5162 						 &pcmdinfo->in.u.ws_node.info,  in irdma_exec_cqp_cmd()
5164 						 pcmdinfo->in.u.ws_node.scratch);  in irdma_exec_cqp_cmd()
5167 		status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,  in irdma_exec_cqp_cmd()
5168 					     &pcmdinfo->in.u.up_map.info,  in irdma_exec_cqp_cmd()
5169 					     pcmdinfo->in.u.up_map.scratch);  in irdma_exec_cqp_cmd()
5172 		status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,  in irdma_exec_cqp_cmd()
5173 						      &pcmdinfo->in.u.query_rdma.query_buff_mem,  in irdma_exec_cqp_cmd()
5174 						      pcmdinfo->in.u.query_rdma.scratch);  in irdma_exec_cqp_cmd()
5177 		status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,  in irdma_exec_cqp_cmd()
5178 						      pcmdinfo->in.u.del_arp_cache_entry.scratch,  in irdma_exec_cqp_cmd()
5179 						      pcmdinfo->in.u.del_arp_cache_entry.arp_index,  in irdma_exec_cqp_cmd()
5180 						      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5183 		status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,  in irdma_exec_cqp_cmd()
5184 						     &pcmdinfo->in.u.manage_apbvt_entry.info,  in irdma_exec_cqp_cmd()
5185 						     pcmdinfo->in.u.manage_apbvt_entry.scratch,  in irdma_exec_cqp_cmd()
5186 						     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5189 		status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,  in irdma_exec_cqp_cmd()
5190 							   &pcmdinfo->in.u.manage_qhash_table_entry.info,  in irdma_exec_cqp_cmd()
5191 							   pcmdinfo->in.u.manage_qhash_table_entry.scratch,  in irdma_exec_cqp_cmd()
5192 							   pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5195 		status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,  in irdma_exec_cqp_cmd()
5196 					    &pcmdinfo->in.u.qp_modify.info,  in irdma_exec_cqp_cmd()
5197 					    pcmdinfo->in.u.qp_modify.scratch,  in irdma_exec_cqp_cmd()
5198 					    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5201 		status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,  in irdma_exec_cqp_cmd()
5202 					    &pcmdinfo->in.u.qp_create.info,  in irdma_exec_cqp_cmd()
5203 					    pcmdinfo->in.u.qp_create.scratch,  in irdma_exec_cqp_cmd()
5204 					    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5207 		status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,  in irdma_exec_cqp_cmd()
5208 					     pcmdinfo->in.u.qp_destroy.scratch,  in irdma_exec_cqp_cmd()
5209 					     pcmdinfo->in.u.qp_destroy.remove_hash_idx,  in irdma_exec_cqp_cmd()
5210 					     pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,  in irdma_exec_cqp_cmd()
5211 					     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5214 		status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,  in irdma_exec_cqp_cmd()
5215 					     &pcmdinfo->in.u.alloc_stag.info,  in irdma_exec_cqp_cmd()
5216 					     pcmdinfo->in.u.alloc_stag.scratch,  in irdma_exec_cqp_cmd()
5217 					     pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5220 		status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,  in irdma_exec_cqp_cmd()
5221 						    &pcmdinfo->in.u.mr_reg_non_shared.info,  in irdma_exec_cqp_cmd()
5222 						    pcmdinfo->in.u.mr_reg_non_shared.scratch,  in irdma_exec_cqp_cmd()
5223 						    pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5226 		status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,  in irdma_exec_cqp_cmd()
5227 					       &pcmdinfo->in.u.dealloc_stag.info,  in irdma_exec_cqp_cmd()
5228 					       pcmdinfo->in.u.dealloc_stag.scratch,  in irdma_exec_cqp_cmd()
5229 					       pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5232 		status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,  in irdma_exec_cqp_cmd()
5233 					   &pcmdinfo->in.u.mw_alloc.info,  in irdma_exec_cqp_cmd()
5234 					   pcmdinfo->in.u.mw_alloc.scratch,  in irdma_exec_cqp_cmd()
5235 					   pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5238 		status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,  in irdma_exec_cqp_cmd()
5239 						      &pcmdinfo->in.u.add_arp_cache_entry.info,  in irdma_exec_cqp_cmd()
5240 						      pcmdinfo->in.u.add_arp_cache_entry.scratch,  in irdma_exec_cqp_cmd()
5241 						      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5244 		status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,  in irdma_exec_cqp_cmd()
5245 							pcmdinfo->in.u.alloc_local_mac_entry.scratch,  in irdma_exec_cqp_cmd()
5246 							pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5249 		status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,  in irdma_exec_cqp_cmd()
5250 						      &pcmdinfo->in.u.add_local_mac_entry.info,  in irdma_exec_cqp_cmd()
5251 						      pcmdinfo->in.u.add_local_mac_entry.scratch,  in irdma_exec_cqp_cmd()
5252 						      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5255 		status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,  in irdma_exec_cqp_cmd()
5256 						      pcmdinfo->in.u.del_local_mac_entry.scratch,  in irdma_exec_cqp_cmd()
5257 						      pcmdinfo->in.u.del_local_mac_entry.entry_idx,  in irdma_exec_cqp_cmd()
5258 						      pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,  in irdma_exec_cqp_cmd()
5259 						      pcmdinfo->post_sq);  in irdma_exec_cqp_cmd()
5262 		status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp,  in irdma_exec_cqp_cmd()
5263 					    &pcmdinfo->in.u.ah_create.info,  in irdma_exec_cqp_cmd()
5264 					    pcmdinfo->in.u.ah_create.scratch);  in irdma_exec_cqp_cmd()
5267 		status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,  in irdma_exec_cqp_cmd()
5268 					     &pcmdinfo->in.u.ah_destroy.info,  in irdma_exec_cqp_cmd()
5269 					     pcmdinfo->in.u.ah_destroy.scratch);  in irdma_exec_cqp_cmd()
5272 		status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp,  in irdma_exec_cqp_cmd()
5273 						   &pcmdinfo->in.u.mc_create.info,  in irdma_exec_cqp_cmd()
5274 						   pcmdinfo->in.u.mc_create.scratch);  in irdma_exec_cqp_cmd()
5277 		status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp,  in irdma_exec_cqp_cmd()
5278 						    &pcmdinfo->in.u.mc_destroy.info,  in irdma_exec_cqp_cmd()
5279 						    pcmdinfo->in.u.mc_destroy.scratch);  in irdma_exec_cqp_cmd()
5282 		status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp,  in irdma_exec_cqp_cmd()
5283 						   &pcmdinfo->in.u.mc_modify.info,  in irdma_exec_cqp_cmd()
5284 						   pcmdinfo->in.u.mc_modify.scratch);  in irdma_exec_cqp_cmd()
5295  * irdma_process_cqp_cmd - process all cqp commands
5305 	spin_lock_irqsave(&dev->cqp_lock, flags);  in irdma_process_cqp_cmd()
5306 	if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))  in irdma_process_cqp_cmd()
5309 		list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);  in irdma_process_cqp_cmd()
5310 	spin_unlock_irqrestore(&dev->cqp_lock, flags);  in irdma_process_cqp_cmd()
5315  * irdma_process_bh - called from tasklet for cqp list
5324 	spin_lock_irqsave(&dev->cqp_lock, flags);  in irdma_process_bh()
5325 	while (!list_empty(&dev->cqp_cmd_head) &&  in irdma_process_bh()
5326 	       !irdma_cqp_ring_full(dev->cqp)) {  in irdma_process_bh()
5332 	spin_unlock_irqrestore(&dev->cqp_lock, flags);  in irdma_process_bh()
5337  * irdma_cfg_aeq- Configure AEQ interrupt
5349 	writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]);  in irdma_cfg_aeq()
5353  * sc_vsi_update_stats - Update statistics
5361 	gather_stats = vsi->pestat->gather_info.gather_stats_va;  in sc_vsi_update_stats()
5362 	last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;  in sc_vsi_update_stats()
5363 	irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,  in sc_vsi_update_stats()
5368  * irdma_wait_pe_ready - Check if firmware is ready
5379 		statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);  in irdma_wait_pe_ready()
5380 		statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);  in irdma_wait_pe_ready()
5381 		statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);  in irdma_wait_pe_ready()
5386 	} while (retrycount++ < dev->hw_attrs.max_pe_ready_count);  in irdma_wait_pe_ready()
5387 	return -1;  in irdma_wait_pe_ready()
5392 	switch (dev->hw_attrs.uk_attrs.hw_rev) {  in irdma_sc_init_hw()
5403  * irdma_sc_dev_init - Initialize control part of device
5416 	INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */  in irdma_sc_dev_init()
5417 	mutex_init(&dev->ws_mutex);  in irdma_sc_dev_init()
5418 	dev->hmc_fn_id = info->hmc_fn_id;  in irdma_sc_dev_init()
5419 	dev->fpm_query_buf_pa = info->fpm_query_buf_pa;  in irdma_sc_dev_init()
5420 	dev->fpm_query_buf = info->fpm_query_buf;  in irdma_sc_dev_init()
5421 	dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;  in irdma_sc_dev_init()
5422 	dev->fpm_commit_buf = info->fpm_commit_buf;  in irdma_sc_dev_init()
5423 	dev->hw = info->hw;  in irdma_sc_dev_init()
5424 	dev->hw->hw_addr = info->bar0;  in irdma_sc_dev_init()
5426 	dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;  in irdma_sc_dev_init()
5427 	dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;  in irdma_sc_dev_init()
5428 	dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;  in irdma_sc_dev_init()
5429 	dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;  in irdma_sc_dev_init()
5430 	dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;  in irdma_sc_dev_init()
5431 	dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;  in irdma_sc_dev_init()
5432 	dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;  in irdma_sc_dev_init()
5433 	dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;  in irdma_sc_dev_init()
5434 	dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;  in irdma_sc_dev_init()
5435 	dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;  in irdma_sc_dev_init()
5436 	dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;  in irdma_sc_dev_init()
5437 	dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;  in irdma_sc_dev_init()
5438 	dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;  in irdma_sc_dev_init()
5439 	dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;  in irdma_sc_dev_init()
5440 	dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;  in irdma_sc_dev_init()
5441 	dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);  in irdma_sc_dev_init()
5443 	dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;  in irdma_sc_dev_init()
5444 	dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;  in irdma_sc_dev_init()
5445 	dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;  in irdma_sc_dev_init()
5446 	dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;  in irdma_sc_dev_init()
5448 	dev->hw_attrs.max_pe_ready_count = 14;  in irdma_sc_dev_init()
5449 	dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;  in irdma_sc_dev_init()
5450 	dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;  in irdma_sc_dev_init()
5451 	dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;  in irdma_sc_dev_init()
5453 	dev->hw_attrs.uk_attrs.hw_rev = ver;  in irdma_sc_dev_init()
5459 	val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);  in irdma_sc_dev_init()
5467 	dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];  in irdma_sc_dev_init()
5473  * irdma_update_stats - Update statistics
5482 	u64 *stats_val = hw_stats->stats_val_32;  in irdma_update_stats()
5485 		IRDMA_STATS_DELTA(gather_stats->rxvlanerr,  in irdma_update_stats()
5486 				  last_gather_stats->rxvlanerr,  in irdma_update_stats()
5489 		IRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,  in irdma_update_stats()
5490 				  last_gather_stats->ip4rxdiscard,  in irdma_update_stats()
5493 		IRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,  in irdma_update_stats()
5494 				  last_gather_stats->ip4rxtrunc,  in irdma_update_stats()
5497 		IRDMA_STATS_DELTA(gather_stats->ip4txnoroute,  in irdma_update_stats()
5498 				  last_gather_stats->ip4txnoroute,  in irdma_update_stats()
5501 		IRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,  in irdma_update_stats()
5502 				  last_gather_stats->ip6rxdiscard,  in irdma_update_stats()
5505 		IRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,  in irdma_update_stats()
5506 				  last_gather_stats->ip6rxtrunc,  in irdma_update_stats()
5509 		IRDMA_STATS_DELTA(gather_stats->ip6txnoroute,  in irdma_update_stats()
5510 				  last_gather_stats->ip6txnoroute,  in irdma_update_stats()
5513 		IRDMA_STATS_DELTA(gather_stats->tcprtxseg,  in irdma_update_stats()
5514 				  last_gather_stats->tcprtxseg,  in irdma_update_stats()
5517 		IRDMA_STATS_DELTA(gather_stats->tcprxopterr,  in irdma_update_stats()
5518 				  last_gather_stats->tcprxopterr,  in irdma_update_stats()
5521 		IRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,  in irdma_update_stats()
5522 				  last_gather_stats->tcprxprotoerr,  in irdma_update_stats()
5525 		IRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,  in irdma_update_stats()
5526 				  last_gather_stats->rxrpcnphandled,  in irdma_update_stats()
5529 		IRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,  in irdma_update_stats()
5530 				  last_gather_stats->rxrpcnpignored,  in irdma_update_stats()
5533 		IRDMA_STATS_DELTA(gather_stats->txnpcnpsent,  in irdma_update_stats()
5534 				  last_gather_stats->txnpcnpsent,  in irdma_update_stats()
5536 	stats_val = hw_stats->stats_val_64;  in irdma_update_stats()
5538 		IRDMA_STATS_DELTA(gather_stats->ip4rxocts,  in irdma_update_stats()
5539 				  last_gather_stats->ip4rxocts,  in irdma_update_stats()
5542 		IRDMA_STATS_DELTA(gather_stats->ip4rxpkts,  in irdma_update_stats()
5543 				  last_gather_stats->ip4rxpkts,  in irdma_update_stats()
5546 		IRDMA_STATS_DELTA(gather_stats->ip4txfrag,  in irdma_update_stats()
5547 				  last_gather_stats->ip4txfrag,  in irdma_update_stats()
5550 		IRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,  in irdma_update_stats()
5551 				  last_gather_stats->ip4rxmcpkts,  in irdma_update_stats()
5554 		IRDMA_STATS_DELTA(gather_stats->ip4txocts,  in irdma_update_stats()
5555 				  last_gather_stats->ip4txocts,  in irdma_update_stats()
5558 		IRDMA_STATS_DELTA(gather_stats->ip4txpkts,  in irdma_update_stats()
5559 				  last_gather_stats->ip4txpkts,  in irdma_update_stats()
5562 		IRDMA_STATS_DELTA(gather_stats->ip4txfrag,  in irdma_update_stats()
5563 				  last_gather_stats->ip4txfrag,  in irdma_update_stats()
5566 		IRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,  in irdma_update_stats()
5567 				  last_gather_stats->ip4txmcpkts,  in irdma_update_stats()
5570 		IRDMA_STATS_DELTA(gather_stats->ip6rxocts,  in irdma_update_stats()
5571 				  last_gather_stats->ip6rxocts,  in irdma_update_stats()
5574 		IRDMA_STATS_DELTA(gather_stats->ip6rxpkts,  in irdma_update_stats()
5575 				  last_gather_stats->ip6rxpkts,  in irdma_update_stats()
5578 		IRDMA_STATS_DELTA(gather_stats->ip6txfrags,  in irdma_update_stats()
5579 				  last_gather_stats->ip6txfrags,  in irdma_update_stats()
5582 		IRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,  in irdma_update_stats()
5583 				  last_gather_stats->ip6rxmcpkts,  in irdma_update_stats()
5586 		IRDMA_STATS_DELTA(gather_stats->ip6txocts,  in irdma_update_stats()
5587 				  last_gather_stats->ip6txocts,  in irdma_update_stats()
5590 		IRDMA_STATS_DELTA(gather_stats->ip6txpkts,  in irdma_update_stats()
5591 				  last_gather_stats->ip6txpkts,  in irdma_update_stats()
5594 		IRDMA_STATS_DELTA(gather_stats->ip6txfrags,  in irdma_update_stats()
5595 				  last_gather_stats->ip6txfrags,  in irdma_update_stats()
5598 		IRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,  in irdma_update_stats()
5599 				  last_gather_stats->ip6txmcpkts,  in irdma_update_stats()
5602 		IRDMA_STATS_DELTA(gather_stats->tcprxsegs,  in irdma_update_stats()
5603 				  last_gather_stats->tcprxsegs,  in irdma_update_stats()
5606 		IRDMA_STATS_DELTA(gather_stats->tcptxsegs,  in irdma_update_stats()
5607 				  last_gather_stats->tcptxsegs,  in irdma_update_stats()
5610 		IRDMA_STATS_DELTA(gather_stats->rdmarxrds,  in irdma_update_stats()
5611 				  last_gather_stats->rdmarxrds,  in irdma_update_stats()
5614 		IRDMA_STATS_DELTA(gather_stats->rdmarxsnds,  in irdma_update_stats()
5615 				  last_gather_stats->rdmarxsnds,  in irdma_update_stats()
5618 		IRDMA_STATS_DELTA(gather_stats->rdmarxwrs,  in irdma_update_stats()
5619 				  last_gather_stats->rdmarxwrs,  in irdma_update_stats()
5622 		IRDMA_STATS_DELTA(gather_stats->rdmatxrds,  in irdma_update_stats()
5623 				  last_gather_stats->rdmatxrds,  in irdma_update_stats()
5626 		IRDMA_STATS_DELTA(gather_stats->rdmatxsnds,  in irdma_update_stats()
5627 				  last_gather_stats->rdmatxsnds,  in irdma_update_stats()
5630 		IRDMA_STATS_DELTA(gather_stats->rdmatxwrs,  in irdma_update_stats()
5631 				  last_gather_stats->rdmatxwrs,  in irdma_update_stats()
5634 		IRDMA_STATS_DELTA(gather_stats->rdmavbn,  in irdma_update_stats()
5635 				  last_gather_stats->rdmavbn,  in irdma_update_stats()
5638 		IRDMA_STATS_DELTA(gather_stats->rdmavinv,  in irdma_update_stats()
5639 				  last_gather_stats->rdmavinv,  in irdma_update_stats()
5642 		IRDMA_STATS_DELTA(gather_stats->udprxpkts,  in irdma_update_stats()
5643 				  last_gather_stats->udprxpkts,  in irdma_update_stats()
5646 		IRDMA_STATS_DELTA(gather_stats->udptxpkts,  in irdma_update_stats()
5647 				  last_gather_stats->udptxpkts,  in irdma_update_stats()
5650 		IRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,  in irdma_update_stats()
5651 				  last_gather_stats->rxnpecnmrkpkts,  in irdma_update_stats()