897 		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq);  in free_qp_work()
1360 static void defer_srq_wr(struct t4_srq *srq, union t4_recv_wr *wqe,  in defer_srq_wr()  argument
1363 	struct t4_srq_pending_wr *pwr = &srq->pending_wrs[srq->pending_pidx];  in defer_srq_wr()
1366 		 __func__, srq->cidx, srq->pidx, srq->wq_pidx,  in defer_srq_wr()
1367 		 srq->in_use, srq->ooo_count,  in defer_srq_wr()
1368 		 (unsigned long long)wr_id, srq->pending_cidx,  in defer_srq_wr()
1369 		 srq->pending_pidx, srq->pending_in_use);  in defer_srq_wr()
1373 	t4_srq_produce_pending_wr(srq);  in defer_srq_wr()
1380 	struct c4iw_srq *srq;  in c4iw_post_srq_recv()  local
1387 	srq = to_c4iw_srq(ibsrq);  in c4iw_post_srq_recv()
1388 	spin_lock_irqsave(&srq->lock, flag);  in c4iw_post_srq_recv()
1389 	num_wrs = t4_srq_avail(&srq->wq);  in c4iw_post_srq_recv()
1391 		spin_unlock_irqrestore(&srq->lock, flag);  in c4iw_post_srq_recv()
1412 		wqe->recv.wrid = srq->wq.pidx;  in c4iw_post_srq_recv()
1418 		if (srq->wq.ooo_count ||  in c4iw_post_srq_recv()
1419 		    srq->wq.pending_in_use ||  in c4iw_post_srq_recv()
1420 		    srq->wq.sw_rq[srq->wq.pidx].valid) {  in c4iw_post_srq_recv()
1421 			defer_srq_wr(&srq->wq, wqe, wr->wr_id, len16);  in c4iw_post_srq_recv()
1423 			srq->wq.sw_rq[srq->wq.pidx].wr_id = wr->wr_id;  in c4iw_post_srq_recv()
1424 			srq->wq.sw_rq[srq->wq.pidx].valid = 1;  in c4iw_post_srq_recv()
1425 			c4iw_copy_wr_to_srq(&srq->wq, wqe, len16);  in c4iw_post_srq_recv()
1427 				 __func__, srq->wq.cidx,  in c4iw_post_srq_recv()
1428 				 srq->wq.pidx, srq->wq.wq_pidx,  in c4iw_post_srq_recv()
1429 				 srq->wq.in_use,  in c4iw_post_srq_recv()
1431 			t4_srq_produce(&srq->wq, len16);  in c4iw_post_srq_recv()
1438 		t4_ring_srq_db(&srq->wq, idx, len16, wqe);  in c4iw_post_srq_recv()
1439 	spin_unlock_irqrestore(&srq->lock, flag);  in c4iw_post_srq_recv()
1641 	if (!qhp->srq) {  in __flush_qp()
1823 	if (qhp->srq) {  in rdma_init()
1825 						  qhp->srq->idx);  in rdma_init()
2151 	if (!attrs->srq) {  in c4iw_create_qp()
2182 	if (!attrs->srq) {  in c4iw_create_qp()
2191 		if (!attrs->srq)  in c4iw_create_qp()
2198 			qhp->wr_waitp, !attrs->srq);  in c4iw_create_qp()
2213 	if (!attrs->srq) {  in c4iw_create_qp()
2241 		if (!attrs->srq) {  in c4iw_create_qp()
2253 		if (!attrs->srq) {  in c4iw_create_qp()
2277 		if (!attrs->srq) {  in c4iw_create_qp()
2289 		if (!attrs->srq) {  in c4iw_create_qp()
2295 		if (!attrs->srq) {  in c4iw_create_qp()
2307 		if (!attrs->srq) {  in c4iw_create_qp()
2317 		if (!attrs->srq) {  in c4iw_create_qp()
2336 	if (!attrs->srq) {  in c4iw_create_qp()
2347 	if (attrs->srq)  in c4iw_create_qp()
2348 		qhp->srq = to_c4iw_srq(attrs->srq);  in c4iw_create_qp()
2358 	if (!attrs->srq)  in c4iw_create_qp()
2363 	if (!attrs->srq)  in c4iw_create_qp()
2371 		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !attrs->srq);  in c4iw_create_qp()
2437 void c4iw_dispatch_srq_limit_reached_event(struct c4iw_srq *srq)  in c4iw_dispatch_srq_limit_reached_event()  argument
2441 	event.device = &srq->rhp->ibdev;  in c4iw_dispatch_srq_limit_reached_event()
2442 	event.element.srq = &srq->ibsrq;  in c4iw_dispatch_srq_limit_reached_event()
2451 	struct c4iw_srq *srq = to_c4iw_srq(ib_srq);  in c4iw_modify_srq()  local
2458 		c4iw_dispatch_srq_limit_reached_event(srq);  in c4iw_modify_srq()
2469 		srq->armed = true;  in c4iw_modify_srq()
2470 		srq->srq_limit = attr->srq_limit;  in c4iw_modify_srq()
2493 static void free_srq_queue(struct c4iw_srq *srq, struct c4iw_dev_ucontext *uctx,  in free_srq_queue()  argument
2496 	struct c4iw_rdev *rdev = &srq->rhp->rdev;  in free_srq_queue()
2497 	struct sk_buff *skb = srq->destroy_skb;  in free_srq_queue()
2498 	struct t4_srq *wq = &srq->wq;  in free_srq_queue()
2514 	res->u.srq.restype = FW_RI_RES_TYPE_SRQ;  in free_srq_queue()
2515 	res->u.srq.op = FW_RI_RES_OP_RESET;  in free_srq_queue()
2516 	res->u.srq.srqid = cpu_to_be32(srq->idx);  in free_srq_queue()
2517 	res->u.srq.eqid = cpu_to_be32(wq->qid);  in free_srq_queue()
2530 static int alloc_srq_queue(struct c4iw_srq *srq, struct c4iw_dev_ucontext *uctx,  in alloc_srq_queue()  argument
2533 	struct c4iw_rdev *rdev = &srq->rhp->rdev;  in alloc_srq_queue()
2535 	struct t4_srq *wq = &srq->wq;  in alloc_srq_queue()
2552 		wq->pending_wrs = kcalloc(srq->wq.size,  in alloc_srq_queue()
2553 					  sizeof(*srq->wq.pending_wrs),  in alloc_srq_queue()
2606 	res->u.srq.restype = FW_RI_RES_TYPE_SRQ;  in alloc_srq_queue()
2607 	res->u.srq.op = FW_RI_RES_OP_WRITE;  in alloc_srq_queue()
2614 	res->u.srq.eqid = cpu_to_be32(wq->qid);  in alloc_srq_queue()
2615 	res->u.srq.fetchszm_to_iqid =  in alloc_srq_queue()
2621 	res->u.srq.dcaen_to_eqsize =  in alloc_srq_queue()
2629 	res->u.srq.eqaddr = cpu_to_be64(wq->dma_addr);  in alloc_srq_queue()
2630 	res->u.srq.srqid = cpu_to_be32(srq->idx);  in alloc_srq_queue()
2631 	res->u.srq.pdid = cpu_to_be32(srq->pdid);  in alloc_srq_queue()
2632 	res->u.srq.hwsrqsize = cpu_to_be32(wq->rqt_size);  in alloc_srq_queue()
2633 	res->u.srq.hwsrqaddr = cpu_to_be32(wq->rqt_hwaddr -  in alloc_srq_queue()
2644 			__func__, srq->idx, wq->qid, srq->pdid, wq->queue,  in alloc_srq_queue()
2667 void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16)  in c4iw_copy_wr_to_srq()  argument
2672 	dst = (u64 *)((u8 *)srq->queue + srq->wq_pidx * T4_EQ_ENTRY_SIZE);  in c4iw_copy_wr_to_srq()
2675 		if (dst >= (u64 *)&srq->queue[srq->size])  in c4iw_copy_wr_to_srq()
2676 			dst = (u64 *)srq->queue;  in c4iw_copy_wr_to_srq()
2678 		if (dst >= (u64 *)&srq->queue[srq->size])  in c4iw_copy_wr_to_srq()
2679 			dst = (u64 *)srq->queue;  in c4iw_copy_wr_to_srq()
2688 	struct c4iw_srq *srq;  in c4iw_create_srq()  local
2702 	if (!rhp->rdev.lldi.vr->srq.size)  in c4iw_create_srq()
2717 	srq = kzalloc(sizeof(*srq), GFP_KERNEL);  in c4iw_create_srq()
2718 	if (!srq)  in c4iw_create_srq()
2721 	srq->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);  in c4iw_create_srq()
2722 	if (!srq->wr_waitp) {  in c4iw_create_srq()
2727 	srq->idx = c4iw_alloc_srq_idx(&rhp->rdev);  in c4iw_create_srq()
2728 	if (srq->idx < 0) {  in c4iw_create_srq()
2734 	srq->destroy_skb = alloc_skb(wr_len, GFP_KERNEL);  in c4iw_create_srq()
2735 	if (!srq->destroy_skb) {  in c4iw_create_srq()
2740 	srq->rhp = rhp;  in c4iw_create_srq()
2741 	srq->pdid = php->pdid;  in c4iw_create_srq()
2743 	srq->wq.size = rqsize;  in c4iw_create_srq()
2744 	srq->wq.memsize =  in c4iw_create_srq()
2746 		sizeof(*srq->wq.queue);  in c4iw_create_srq()
2748 		srq->wq.memsize = roundup(srq->wq.memsize, PAGE_SIZE);  in c4iw_create_srq()
2750 	ret = alloc_srq_queue(srq, ucontext ? &ucontext->uctx :  in c4iw_create_srq()
2751 			&rhp->rdev.uctx, srq->wr_waitp);  in c4iw_create_srq()
2757 		srq->flags = T4_SRQ_LIMIT_SUPPORT;  in c4iw_create_srq()
2759 	ret = insert_handle(rhp, &rhp->qpidr, srq, srq->wq.qid);  in c4iw_create_srq()
2775 		uresp.flags = srq->flags;  in c4iw_create_srq()
2777 		uresp.srqid = srq->wq.qid;  in c4iw_create_srq()
2778 		uresp.srq_size = srq->wq.size;  in c4iw_create_srq()
2779 		uresp.srq_memsize = srq->wq.memsize;  in c4iw_create_srq()
2780 		uresp.rqt_abs_idx = srq->wq.rqt_abs_idx;  in c4iw_create_srq()
2791 		srq_key_mm->addr = virt_to_phys(srq->wq.queue);  in c4iw_create_srq()
2792 		srq_key_mm->len = PAGE_ALIGN(srq->wq.memsize);  in c4iw_create_srq()
2795 		srq_db_key_mm->addr = (u64)(unsigned long)srq->wq.bar2_pa;  in c4iw_create_srq()
2801 		 __func__, srq->wq.qid, srq->idx, srq->wq.size,  in c4iw_create_srq()
2802 			(unsigned long)srq->wq.memsize, attrs->attr.max_wr);  in c4iw_create_srq()
2804 	spin_lock_init(&srq->lock);  in c4iw_create_srq()
2805 	return &srq->ibsrq;  in c4iw_create_srq()
2811 	remove_handle(rhp, &rhp->qpidr, srq->wq.qid);  in c4iw_create_srq()
2813 	free_srq_queue(srq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx,  in c4iw_create_srq()
2814 		       srq->wr_waitp);  in c4iw_create_srq()
2816 	if (srq->destroy_skb)  in c4iw_create_srq()
2817 		kfree_skb(srq->destroy_skb);  in c4iw_create_srq()
2819 	c4iw_free_srq_idx(&rhp->rdev, srq->idx);  in c4iw_create_srq()
2821 	c4iw_put_wr_wait(srq->wr_waitp);  in c4iw_create_srq()
2823 	kfree(srq);  in c4iw_create_srq()
2830 	struct c4iw_srq *srq;  in c4iw_destroy_srq()  local
2833 	srq = to_c4iw_srq(ibsrq);  in c4iw_destroy_srq()
2834 	rhp = srq->rhp;  in c4iw_destroy_srq()
2836 	pr_debug("%s id %d\n", __func__, srq->wq.qid);  in c4iw_destroy_srq()
2838 	remove_handle(rhp, &rhp->qpidr, srq->wq.qid);  in c4iw_destroy_srq()
2841 	free_srq_queue(srq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx,  in c4iw_destroy_srq()
2842 		       srq->wr_waitp);  in c4iw_destroy_srq()
2843 	c4iw_free_srq_idx(&rhp->rdev, srq->idx);  in c4iw_destroy_srq()
2844 	c4iw_put_wr_wait(srq->wr_waitp);  in c4iw_destroy_srq()
2845 	kfree(srq);  in c4iw_destroy_srq()