61 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_query_device()  local
66 	ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);  in i40iw_query_device()
67 	props->fw_ver = i40iw_fw_major_ver(&iwdev->sc_dev) << 32 |  in i40iw_query_device()
68 			i40iw_fw_minor_ver(&iwdev->sc_dev);  in i40iw_query_device()
69 	props->device_cap_flags = iwdev->device_cap_flags;  in i40iw_query_device()
70 	props->vendor_id = iwdev->ldev->pcidev->vendor;  in i40iw_query_device()
71 	props->vendor_part_id = iwdev->ldev->pcidev->device;  in i40iw_query_device()
72 	props->hw_ver = (u32)iwdev->sc_dev.hw_rev;  in i40iw_query_device()
74 	props->max_qp = iwdev->max_qp - iwdev->used_qps;  in i40iw_query_device()
78 	props->max_cq = iwdev->max_cq - iwdev->used_cqs;  in i40iw_query_device()
79 	props->max_cqe = iwdev->max_cqe;  in i40iw_query_device()
80 	props->max_mr = iwdev->max_mr - iwdev->used_mrs;  in i40iw_query_device()
81 	props->max_pd = iwdev->max_pd - iwdev->used_pds;  in i40iw_query_device()
122 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_alloc_ucontext()  local
135 	uresp.max_qps = iwdev->max_qp;  in i40iw_alloc_ucontext()
136 	uresp.max_pds = iwdev->max_pd;  in i40iw_alloc_ucontext()
137 	uresp.wq_size = iwdev->max_qp_wr * 2;  in i40iw_alloc_ucontext()
140 	ucontext->iwdev = iwdev;  in i40iw_alloc_ucontext()
176 	dbaddr = I40IW_DB_ADDR_OFFSET + pci_resource_start(ucontext->iwdev->ldev->pcidev, 0);  in i40iw_mmap()
187 static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)  in i40iw_alloc_push_page()  argument
196 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_alloc_push_page()
208 	cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;  in i40iw_alloc_push_page()
211 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_alloc_push_page()
216 	i40iw_put_cqp_request(&iwdev->cqp, cqp_request);  in i40iw_alloc_push_page()
224 static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)  in i40iw_dealloc_push_page()  argument
233 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);  in i40iw_dealloc_push_page()
244 	cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;  in i40iw_dealloc_push_page()
247 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_dealloc_push_page()
262 	struct i40iw_device *iwdev = to_iwdev(pd->device);  in i40iw_alloc_pd()  local
263 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_alloc_pd()
269 	if (iwdev->closing)  in i40iw_alloc_pd()
272 	err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds,  in i40iw_alloc_pd()
273 				   iwdev->max_pd, &pd_id, &iwdev->next_pd);  in i40iw_alloc_pd()
299 	i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id);  in i40iw_alloc_pd()
311 	struct i40iw_device *iwdev = to_iwdev(ibpd->device);  in i40iw_dealloc_pd()  local
313 	i40iw_rem_pdusecount(iwpd, iwdev);  in i40iw_dealloc_pd()
347 	struct i40iw_device *iwdev = iwqp->iwdev;  in i40iw_free_qp_resources()  local
350 	i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);  in i40iw_free_qp_resources()
351 	i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);  in i40iw_free_qp_resources()
353 		i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);  in i40iw_free_qp_resources()
355 		i40iw_free_pble(iwdev->pble_rsrc, &iwpbl->pble_alloc);  in i40iw_free_qp_resources()
356 	i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->q2_ctx_mem);  in i40iw_free_qp_resources()
357 	i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem);  in i40iw_free_qp_resources()
383 	struct i40iw_device *iwdev = iwqp->iwdev;  in i40iw_destroy_qp()  local
404 	i40iw_cqp_qp_destroy_cmd(&iwdev->sc_dev, &iwqp->sc_qp);  in i40iw_destroy_qp()
405 	i40iw_rem_pdusecount(iwqp->iwpd, iwdev);  in i40iw_destroy_qp()
407 	i40iw_rem_devusecount(iwdev);  in i40iw_destroy_qp()
418 static int i40iw_setup_virt_qp(struct i40iw_device *iwdev,  in i40iw_setup_virt_qp()  argument
444 static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev,  in i40iw_setup_kmode_qp()  argument
476 	status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256);  in i40iw_setup_kmode_qp()
509 	struct i40iw_device *iwdev = to_iwdev(ibpd->device);  in i40iw_create_qp()  local
510 	struct i40iw_cqp *iwcqp = &iwdev->cqp;  in i40iw_create_qp()
522 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_create_qp()
532 	if (iwdev->closing)  in i40iw_create_qp()
551 	init_info.vsi = &iwdev->vsi;  in i40iw_create_qp()
566 	iwqp->iwdev = iwdev;  in i40iw_create_qp()
584 	err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp,  in i40iw_create_qp()
585 					&qp_num, &iwdev->next_qp);  in i40iw_create_qp()
609 	if (iwdev->push_mode)  in i40iw_create_qp()
610 		i40iw_alloc_push_page(iwdev, qp);  in i40iw_create_qp()
638 		err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info);  in i40iw_create_qp()
640 		err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info);  in i40iw_create_qp()
697 	ret = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_create_qp()
707 	iwdev->qp_table[qp_num] = iwqp;  in i40iw_create_qp()
709 	i40iw_add_devusecount(iwdev);  in i40iw_create_qp()
775 void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,  in i40iw_hw_modify_qp()  argument
783 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);  in i40iw_hw_modify_qp()
794 	if (!i40iw_handle_cqp_op(iwdev, cqp_request))  in i40iw_hw_modify_qp()
807 		i40iw_gen_ae(iwdev, &iwqp->sc_qp, &ae_info, false);  in i40iw_hw_modify_qp()
826 	struct i40iw_device *iwdev = iwqp->iwdev;  in i40iw_modify_qp()  local
842 		if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {  in i40iw_modify_qp()
910 			    iwdev->iw_status &&  in i40iw_modify_qp()
945 		struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_modify_qp()
963 		i40iw_hw_modify_qp(iwdev, iwqp, &info, true);  in i40iw_modify_qp()
1003 static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq)  in cq_free_resources()  argument
1008 		i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwcq->kmem);  in cq_free_resources()
1009 	i40iw_free_resource(iwdev, iwdev->allocated_cqs, cq->cq_uk.cq_id);  in cq_free_resources()
1017 void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)  in i40iw_cq_wq_destroy()  argument
1023 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_cq_wq_destroy()
1033 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_cq_wq_destroy()
1046 	struct i40iw_device *iwdev;  in i40iw_destroy_cq()  local
1050 	iwdev = to_iwdev(ib_cq->device);  in i40iw_destroy_cq()
1052 	i40iw_cq_wq_destroy(iwdev, cq);  in i40iw_destroy_cq()
1053 	cq_free_resources(iwdev, iwcq);  in i40iw_destroy_cq()
1054 	i40iw_rem_devusecount(iwdev);  in i40iw_destroy_cq()
1069 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_create_cq()  local
1074 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_create_cq()
1084 	if (iwdev->closing)  in i40iw_create_cq()
1087 	if (entries > iwdev->max_cqe)  in i40iw_create_cq()
1090 	err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs,  in i40iw_create_cq()
1091 					iwdev->max_cq, &cq_num,  in i40iw_create_cq()
1092 					&iwdev->next_cq);  in i40iw_create_cq()
1105 	if (attr->comp_vector < iwdev->ceqs_count)  in i40iw_create_cq()
1169 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_create_cq()
1180 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_create_cq()
1200 	i40iw_add_devusecount(iwdev);  in i40iw_create_cq()
1204 	i40iw_cq_wq_destroy(iwdev, cq);  in i40iw_create_cq()
1206 	cq_free_resources(iwdev, iwcq);  in i40iw_create_cq()
1230 static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)  in i40iw_free_stag()  argument
1234 	stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT;  in i40iw_free_stag()
1235 	i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx);  in i40iw_free_stag()
1236 	i40iw_rem_devusecount(iwdev);  in i40iw_free_stag()
1243 static u32 i40iw_create_stag(struct i40iw_device *iwdev)  in i40iw_create_stag()  argument
1256 	driver_key = random & ~iwdev->mr_stagmask;  in i40iw_create_stag()
1257 	next_stag_index = (random & iwdev->mr_stagmask) >> 8;  in i40iw_create_stag()
1258 	next_stag_index %= iwdev->max_mr;  in i40iw_create_stag()
1260 	ret = i40iw_alloc_resource(iwdev,  in i40iw_create_stag()
1261 				   iwdev->allocated_mrs, iwdev->max_mr,  in i40iw_create_stag()
1267 		i40iw_add_devusecount(iwdev);  in i40iw_create_stag()
1376 static int i40iw_setup_pbles(struct i40iw_device *iwdev,  in i40iw_setup_pbles()  argument
1388 		mutex_lock(&iwdev->pbl_mutex);  in i40iw_setup_pbles()
1389 		status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_setup_pbles()
1390 		mutex_unlock(&iwdev->pbl_mutex);  in i40iw_setup_pbles()
1417 static int i40iw_handle_q_mem(struct i40iw_device *iwdev,  in i40iw_handle_q_mem()  argument
1436 	err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_handle_q_mem()
1441 		i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_handle_q_mem()
1482 		i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_handle_q_mem()
1494 static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)  in i40iw_hw_alloc_stag()  argument
1503 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_hw_alloc_stag()
1517 	cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev;  in i40iw_hw_alloc_stag()
1520 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_hw_alloc_stag()
1538 	struct i40iw_device *iwdev = to_iwdev(pd->device);  in i40iw_alloc_mr()  local
1550 	stag = i40iw_create_stag(iwdev);  in i40iw_alloc_mr()
1566 	mutex_lock(&iwdev->pbl_mutex);  in i40iw_alloc_mr()
1567 	status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_alloc_mr()
1568 	mutex_unlock(&iwdev->pbl_mutex);  in i40iw_alloc_mr()
1574 	err_code = i40iw_hw_alloc_stag(iwdev, iwmr);  in i40iw_alloc_mr()
1581 	i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_alloc_mr()
1583 	i40iw_free_stag(iwdev, stag);  in i40iw_alloc_mr()
1656 static int i40iw_hwreg_mr(struct i40iw_device *iwdev,  in i40iw_hwreg_mr()  argument
1669 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_hwreg_mr()
1699 	cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->sc_dev;  in i40iw_hwreg_mr()
1702 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_hwreg_mr()
1727 	struct i40iw_device *iwdev = to_iwdev(pd->device);  in i40iw_reg_user_mr()  local
1745 	if (iwdev->closing)  in i40iw_reg_user_mr()
1786 		err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);  in i40iw_reg_user_mr()
1796 		err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);  in i40iw_reg_user_mr()
1809 		err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_reg_user_mr()
1816 				i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_reg_user_mr()
1822 		stag = i40iw_create_stag(iwdev);  in i40iw_reg_user_mr()
1832 		err = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_user_mr()
1834 			i40iw_free_stag(iwdev, stag);  in i40iw_reg_user_mr()
1850 		i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_reg_user_mr()
1871 	struct i40iw_device *iwdev = to_iwdev(pd->device);  in i40iw_reg_phys_mr()  local
1888 	stag = i40iw_create_stag(iwdev);  in i40iw_reg_phys_mr()
1900 	status = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_phys_mr()
1902 		i40iw_free_stag(iwdev, stag);  in i40iw_reg_phys_mr()
1968 	struct i40iw_device *iwdev = to_iwdev(ib_mr->device);  in i40iw_dereg_mr()  local
1991 			i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_dereg_mr()
1996 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);  in i40iw_dereg_mr()
2013 	cqp_info->in.u.dealloc_stag.dev = &iwdev->sc_dev;  in i40iw_dereg_mr()
2015 	status = i40iw_handle_cqp_op(iwdev, cqp_request);  in i40iw_dereg_mr()
2018 	i40iw_rem_pdusecount(iwpd, iwdev);  in i40iw_dereg_mr()
2019 	i40iw_free_stag(iwdev, iwmr->stag);  in i40iw_dereg_mr()
2021 		i40iw_free_pble(iwdev->pble_rsrc, palloc);  in i40iw_dereg_mr()
2034 	u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev;  in hw_rev_show()
2215 			struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;  in i40iw_post_send()
2513 	struct i40iw_device *iwdev = to_iwdev(dev);  in i40iw_get_dev_fw_str()  local
2516 		 i40iw_fw_major_ver(&iwdev->sc_dev),  in i40iw_get_dev_fw_str()
2517 		 i40iw_fw_minor_ver(&iwdev->sc_dev));  in i40iw_get_dev_fw_str()
2528 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_alloc_hw_stats()  local
2529 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_alloc_hw_stats()
2559 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_get_hw_stats()  local
2560 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;  in i40iw_get_hw_stats()
2561 	struct i40iw_vsi_pestat *devstat = iwdev->vsi.pestat;  in i40iw_get_hw_stats()
2588 	struct i40iw_device *iwdev = to_iwdev(ibdev);  in i40iw_query_gid()  local
2591 	ether_addr_copy(gid->raw, iwdev->netdev->dev_addr);  in i40iw_query_gid()
2647 static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev)  in i40iw_init_rdma_device()  argument
2650 	struct net_device *netdev = iwdev->netdev;  in i40iw_init_rdma_device()
2651 	struct pci_dev *pcidev = iwdev->hw.pcidev;  in i40iw_init_rdma_device()
2658 	iwdev->iwibdev = iwibdev;  in i40iw_init_rdma_device()
2659 	iwibdev->iwdev = iwdev;  in i40iw_init_rdma_device()
2686 	iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;  in i40iw_init_rdma_device()
2699 void i40iw_port_ibevent(struct i40iw_device *iwdev)  in i40iw_port_ibevent()  argument
2701 	struct i40iw_ib_device *iwibdev = iwdev->iwibdev;  in i40iw_port_ibevent()
2706 	event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;  in i40iw_port_ibevent()
2717 	wait_event_timeout(iwibdev->iwdev->close_wq,  in i40iw_destroy_rdma_device()
2718 			   !atomic64_read(&iwibdev->iwdev->use_count),  in i40iw_destroy_rdma_device()
2727 int i40iw_register_rdma_device(struct i40iw_device *iwdev)  in i40iw_register_rdma_device()  argument
2732 	iwdev->iwibdev = i40iw_init_rdma_device(iwdev);  in i40iw_register_rdma_device()
2733 	if (!iwdev->iwibdev)  in i40iw_register_rdma_device()
2735 	iwibdev = iwdev->iwibdev;  in i40iw_register_rdma_device()
2737 	ret = ib_device_set_netdev(&iwibdev->ibdev, iwdev->netdev, 1);  in i40iw_register_rdma_device()
2741 	dma_set_max_seg_size(&iwdev->hw.pcidev->dev, UINT_MAX);  in i40iw_register_rdma_device()
2742 	ret = ib_register_device(&iwibdev->ibdev, "i40iw%d", &iwdev->hw.pcidev->dev);  in i40iw_register_rdma_device()
2748 	ib_dealloc_device(&iwdev->iwibdev->ibdev);  in i40iw_register_rdma_device()