71 int cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,  in cxio_hal_cq_op()  argument
82 	ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);  in cxio_hal_cq_op()
113 				pr_err("%s: stalled rnic\n", rdev_p->dev_name);  in cxio_hal_cq_op()
124 static int cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)  in cxio_hal_clear_cq_ctx()  argument
133 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_hal_clear_cq_ctx()
136 static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)  in cxio_hal_clear_qp_ctx()  argument
153 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_hal_clear_qp_ctx()
156 int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)  in cxio_create_cq()  argument
162 	cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);  in cxio_create_cq()
170 	cq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev), size,  in cxio_create_cq()
182 	if (rdev_p->t3cdev_p->type != T3A)  in cxio_create_cq()
186 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_create_cq()
189 static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in get_qpid()  argument
203 		qpid = cxio_hal_get_qpid(rdev_p->rscp);  in get_qpid()
206 		for (i = qpid+1; i & rdev_p->qpmask; i++) {  in get_qpid()
220 static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid,  in put_qpid()  argument
235 void cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in cxio_release_ucontext()  argument
244 		if (!(entry->qpid & rdev_p->qpmask))  in cxio_release_ucontext()
245 			cxio_hal_put_qpid(rdev_p->rscp, entry->qpid);  in cxio_release_ucontext()
251 void cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)  in cxio_init_ucontext()  argument
257 int cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,  in cxio_create_qp()  argument
263 	wq->qpid = get_qpid(rdev_p, uctx);  in cxio_create_qp()
271 	wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);  in cxio_create_qp()
279 	wq->queue = dma_alloc_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_create_qp()
286 	wq->doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;  in cxio_create_qp()
288 		wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +  in cxio_create_qp()
289 					(wq->qpid << rdev_p->qpshift);  in cxio_create_qp()
290 	wq->rdev = rdev_p;  in cxio_create_qp()
297 	cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);  in cxio_create_qp()
301 	put_qpid(rdev_p, wq->qpid, uctx);  in cxio_create_qp()
305 void cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)  in cxio_destroy_cq()  argument
307 	cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);  in cxio_destroy_cq()
309 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_destroy_cq()
313 	cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);  in cxio_destroy_cq()
316 int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,  in cxio_destroy_qp()  argument
319 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_destroy_qp()
324 	cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));  in cxio_destroy_qp()
326 	put_qpid(rdev_p, wq->qpid, uctx);  in cxio_destroy_qp()
478 static int cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)  in cxio_hal_init_ctrl_cq()  argument
489 	return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));  in cxio_hal_init_ctrl_cq()
492 static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)  in cxio_hal_init_ctrl_qp()  argument
505 	err = cxio_hal_init_ctrl_cq(rdev_p);  in cxio_hal_init_ctrl_qp()
510 	rdev_p->ctrl_qp.workq = dma_alloc_coherent(  in cxio_hal_init_ctrl_qp()
511 					&(rdev_p->rnic_info.pdev->dev),  in cxio_hal_init_ctrl_qp()
514 					&(rdev_p->ctrl_qp.dma_addr),  in cxio_hal_init_ctrl_qp()
516 	if (!rdev_p->ctrl_qp.workq) {  in cxio_hal_init_ctrl_qp()
521 	dma_unmap_addr_set(&rdev_p->ctrl_qp, mapping,  in cxio_hal_init_ctrl_qp()
522 			   rdev_p->ctrl_qp.dma_addr);  in cxio_hal_init_ctrl_qp()
523 	rdev_p->ctrl_qp.doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr;  in cxio_hal_init_ctrl_qp()
525 	mutex_init(&rdev_p->ctrl_qp.lock);  in cxio_hal_init_ctrl_qp()
526 	init_waitqueue_head(&rdev_p->ctrl_qp.waitq);  in cxio_hal_init_ctrl_qp()
529 	base_addr = rdev_p->ctrl_qp.dma_addr;  in cxio_hal_init_ctrl_qp()
550 		 &rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq,  in cxio_hal_init_ctrl_qp()
553 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_hal_init_ctrl_qp()
559 static int cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)  in cxio_hal_destroy_ctrl_qp()  argument
561 	dma_free_coherent(&(rdev_p->rnic_info.pdev->dev),  in cxio_hal_destroy_ctrl_qp()
563 			  * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,  in cxio_hal_destroy_ctrl_qp()
564 			  dma_unmap_addr(&rdev_p->ctrl_qp, mapping));  in cxio_hal_destroy_ctrl_qp()
565 	return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);  in cxio_hal_destroy_ctrl_qp()
572 static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,  in cxio_hal_ctrl_qp_write_mem()  argument
584 		 __func__, rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,  in cxio_hal_ctrl_qp_write_mem()
588 		if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
592 				 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);  in cxio_hal_ctrl_qp_write_mem()
593 			if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in cxio_hal_ctrl_qp_write_mem()
594 					     !Q_FULL(rdev_p->ctrl_qp.rptr,  in cxio_hal_ctrl_qp_write_mem()
595 						     rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
604 		wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %  in cxio_hal_ctrl_qp_write_mem()
646 		wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %  in cxio_hal_ctrl_qp_write_mem()
650 		((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;  in cxio_hal_ctrl_qp_write_mem()
657 			       Q_GENBIT(rdev_p->ctrl_qp.wptr,  in cxio_hal_ctrl_qp_write_mem()
661 			ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);  in cxio_hal_ctrl_qp_write_mem()
663 		rdev_p->ctrl_qp.wptr++;  in cxio_hal_ctrl_qp_write_mem()
672 static int __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,  in __cxio_tpt_op()  argument
683 	if (cxio_fatal_error(rdev_p))  in __cxio_tpt_op()
690 		stag_idx = cxio_hal_get_stag(rdev_p->rscp);  in __cxio_tpt_op()
698 	mutex_lock(&rdev_p->ctrl_qp.lock);  in __cxio_tpt_op()
713 		tpt.rsvd_pbl_addr = cpu_to_be32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3));  in __cxio_tpt_op()
720 	err = cxio_hal_ctrl_qp_write_mem(rdev_p,  in __cxio_tpt_op()
722 				       (rdev_p->rnic_info.tpt_base >> 5),  in __cxio_tpt_op()
727 		cxio_hal_put_stag(rdev_p->rscp, stag_idx);  in __cxio_tpt_op()
729 	wptr = rdev_p->ctrl_qp.wptr;  in __cxio_tpt_op()
730 	mutex_unlock(&rdev_p->ctrl_qp.lock);  in __cxio_tpt_op()
732 		if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in __cxio_tpt_op()
733 					     SEQ32_GE(rdev_p->ctrl_qp.rptr,  in __cxio_tpt_op()
739 int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,  in cxio_write_pbl()  argument
746 		 __func__, pbl_addr, rdev_p->rnic_info.pbl_base,  in cxio_write_pbl()
749 	mutex_lock(&rdev_p->ctrl_qp.lock);  in cxio_write_pbl()
750 	err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3,  in cxio_write_pbl()
752 	wptr = rdev_p->ctrl_qp.wptr;  in cxio_write_pbl()
753 	mutex_unlock(&rdev_p->ctrl_qp.lock);  in cxio_write_pbl()
757 	if (wait_event_interruptible(rdev_p->ctrl_qp.waitq,  in cxio_write_pbl()
758 				     SEQ32_GE(rdev_p->ctrl_qp.rptr,  in cxio_write_pbl()
765 int cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,  in cxio_register_phys_mem()  argument
770 	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,  in cxio_register_phys_mem()
774 int cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,  in cxio_reregister_phys_mem()  argument
778 	return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,  in cxio_reregister_phys_mem()
782 int cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,  in cxio_dereg_mem()  argument
785 	return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,  in cxio_dereg_mem()
789 int cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)  in cxio_allocate_window()  argument
792 	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,  in cxio_allocate_window()
796 int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)  in cxio_deallocate_window()  argument
798 	return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,  in cxio_deallocate_window()
802 int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)  in cxio_allocate_stag()  argument
805 	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,  in cxio_allocate_stag()
809 int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)  in cxio_rdma_init()  argument
815 	pr_debug("%s rdev_p %p\n", __func__, rdev_p);  in cxio_rdma_init()
825 	wqe->rq_addr = cpu_to_be32(attr->rq_addr - rdev_p->rnic_info.rqt_base);  in cxio_rdma_init()
840 	return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb);  in cxio_rdma_init()
856 	struct cxio_rdev *rdev_p = NULL;  in cxio_hal_ev_handler()  local
868 	rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;  in cxio_hal_ev_handler()
869 	if (!rdev_p) {  in cxio_hal_ev_handler()
875 		rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;  in cxio_hal_ev_handler()
876 		wake_up_interruptible(&rdev_p->ctrl_qp.waitq);  in cxio_hal_ev_handler()
881 		(*cxio_ev_cb) (rdev_p, skb);  in cxio_hal_ev_handler()
889 int cxio_rdev_open(struct cxio_rdev *rdev_p)  in cxio_rdev_open()  argument
893 	if (strlen(rdev_p->dev_name)) {  in cxio_rdev_open()
894 		if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {  in cxio_rdev_open()
897 		netdev_p = dev_get_by_name(&init_net, rdev_p->dev_name);  in cxio_rdev_open()
902 	} else if (rdev_p->t3cdev_p) {  in cxio_rdev_open()
903 		if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p)) {  in cxio_rdev_open()
906 		netdev_p = rdev_p->t3cdev_p->lldev;  in cxio_rdev_open()
907 		strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,  in cxio_rdev_open()
914 	list_add_tail(&rdev_p->entry, &rdev_list);  in cxio_rdev_open()
916 	pr_debug("%s opening rnic dev %s\n", __func__, rdev_p->dev_name);  in cxio_rdev_open()
917 	memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));  in cxio_rdev_open()
918 	if (!rdev_p->t3cdev_p)  in cxio_rdev_open()
919 		rdev_p->t3cdev_p = dev2t3cdev(netdev_p);  in cxio_rdev_open()
920 	rdev_p->t3cdev_p->ulp = (void *) rdev_p;  in cxio_rdev_open()
922 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_EMBEDDED_INFO,  in cxio_rdev_open()
923 					 &(rdev_p->fw_info));  in cxio_rdev_open()
926 		       __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
929 	if (G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers) != CXIO_FW_MAJ) {  in cxio_rdev_open()
932 		       G_FW_VERSION_MAJOR(rdev_p->fw_info.fw_vers));  in cxio_rdev_open()
937 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,  in cxio_rdev_open()
938 					 &(rdev_p->rnic_info));  in cxio_rdev_open()
941 		       __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
944 	err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,  in cxio_rdev_open()
945 				    &(rdev_p->port_info));  in cxio_rdev_open()
948 		       __func__, rdev_p->t3cdev_p, err);  in cxio_rdev_open()
956 	cxio_init_ucontext(rdev_p, &rdev_p->uctx);  in cxio_rdev_open()
957 	rdev_p->qpshift = PAGE_SHIFT -  in cxio_rdev_open()
959 			            ilog2(rdev_p->rnic_info.udbell_len >>  in cxio_rdev_open()
961 	rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;  in cxio_rdev_open()
962 	rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;  in cxio_rdev_open()
964 		 __func__, rdev_p->dev_name, rdev_p->rnic_info.tpt_base,  in cxio_rdev_open()
965 		 rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p),  in cxio_rdev_open()
966 		 rdev_p->rnic_info.pbl_base,  in cxio_rdev_open()
967 		 rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,  in cxio_rdev_open()
968 		 rdev_p->rnic_info.rqt_top);  in cxio_rdev_open()
970 		 rdev_p->rnic_info.udbell_len,  in cxio_rdev_open()
971 		 rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,  in cxio_rdev_open()
972 		 rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);  in cxio_rdev_open()
974 	err = cxio_hal_init_ctrl_qp(rdev_p);  in cxio_rdev_open()
979 	err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,  in cxio_rdev_open()
987 	err = cxio_hal_pblpool_create(rdev_p);  in cxio_rdev_open()
993 	err = cxio_hal_rqtpool_create(rdev_p);  in cxio_rdev_open()
1001 	cxio_hal_pblpool_destroy(rdev_p);  in cxio_rdev_open()
1003 	cxio_hal_destroy_resource(rdev_p->rscp);  in cxio_rdev_open()
1005 	cxio_hal_destroy_ctrl_qp(rdev_p);  in cxio_rdev_open()
1007 	rdev_p->t3cdev_p->ulp = NULL;  in cxio_rdev_open()
1008 	list_del(&rdev_p->entry);  in cxio_rdev_open()
1012 void cxio_rdev_close(struct cxio_rdev *rdev_p)  in cxio_rdev_close()  argument
1014 	if (rdev_p) {  in cxio_rdev_close()
1015 		cxio_hal_pblpool_destroy(rdev_p);  in cxio_rdev_close()
1016 		cxio_hal_rqtpool_destroy(rdev_p);  in cxio_rdev_close()
1017 		list_del(&rdev_p->entry);  in cxio_rdev_close()
1018 		cxio_hal_destroy_ctrl_qp(rdev_p);  in cxio_rdev_close()
1019 		cxio_hal_destroy_resource(rdev_p->rscp);  in cxio_rdev_close()
1020 		rdev_p->t3cdev_p->ulp = NULL;  in cxio_rdev_close()