37 	struct ib_cq *cq = dim->priv;  in ib_cq_rdma_dim_work()  local
44 	cq->device->ops.modify_cq(cq, comps, usec);  in ib_cq_rdma_dim_work()
47 static void rdma_dim_init(struct ib_cq *cq)  in rdma_dim_init()  argument
51 	if (!cq->device->ops.modify_cq || !cq->device->use_cq_dim ||  in rdma_dim_init()
52 	    cq->poll_ctx == IB_POLL_DIRECT)  in rdma_dim_init()
62 	dim->priv = cq;  in rdma_dim_init()
63 	cq->dim = dim;  in rdma_dim_init()
68 static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,  in __ib_process_cq()  argument
78 	while ((n = ib_poll_cq(cq, min_t(u32, batch,  in __ib_process_cq()
84 				wc->wr_cqe->done(cq, wc);  in __ib_process_cq()
112 int ib_process_cq_direct(struct ib_cq *cq, int budget)  in ib_process_cq_direct()  argument
116 	return __ib_process_cq(cq, budget, wcs, IB_POLL_BATCH_DIRECT);  in ib_process_cq_direct()
120 static void ib_cq_completion_direct(struct ib_cq *cq, void *private)  in ib_cq_completion_direct()  argument
122 	WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);  in ib_cq_completion_direct()
127 	struct ib_cq *cq = container_of(iop, struct ib_cq, iop);  in ib_poll_handler()  local
128 	struct dim *dim = cq->dim;  in ib_poll_handler()
131 	completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);  in ib_poll_handler()
133 		irq_poll_complete(&cq->iop);  in ib_poll_handler()
134 		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)  in ib_poll_handler()
135 			irq_poll_sched(&cq->iop);  in ib_poll_handler()
144 static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)  in ib_cq_completion_softirq()  argument
146 	irq_poll_sched(&cq->iop);  in ib_cq_completion_softirq()
151 	struct ib_cq *cq = container_of(work, struct ib_cq, work);  in ib_cq_poll_work()  local
154 	completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, cq->wc,  in ib_cq_poll_work()
157 	    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)  in ib_cq_poll_work()
158 		queue_work(cq->comp_wq, &cq->work);  in ib_cq_poll_work()
159 	else if (cq->dim)  in ib_cq_poll_work()
160 		rdma_dim(cq->dim, completed);  in ib_cq_poll_work()
163 static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)  in ib_cq_completion_workqueue()  argument
165 	queue_work(cq->comp_wq, &cq->work);  in ib_cq_completion_workqueue()
192 	struct ib_cq *cq;  in __ib_alloc_cq_user()  local
195 	cq = rdma_zalloc_drv_obj(dev, ib_cq);  in __ib_alloc_cq_user()
196 	if (!cq)  in __ib_alloc_cq_user()
199 	cq->device = dev;  in __ib_alloc_cq_user()
200 	cq->cq_context = private;  in __ib_alloc_cq_user()
201 	cq->poll_ctx = poll_ctx;  in __ib_alloc_cq_user()
202 	atomic_set(&cq->usecnt, 0);  in __ib_alloc_cq_user()
204 	cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);  in __ib_alloc_cq_user()
205 	if (!cq->wc)  in __ib_alloc_cq_user()
208 	cq->res.type = RDMA_RESTRACK_CQ;  in __ib_alloc_cq_user()
209 	rdma_restrack_set_task(&cq->res, caller);  in __ib_alloc_cq_user()
211 	ret = dev->ops.create_cq(cq, &cq_attr, NULL);  in __ib_alloc_cq_user()
215 	rdma_restrack_kadd(&cq->res);  in __ib_alloc_cq_user()
217 	rdma_dim_init(cq);  in __ib_alloc_cq_user()
219 	switch (cq->poll_ctx) {  in __ib_alloc_cq_user()
221 		cq->comp_handler = ib_cq_completion_direct;  in __ib_alloc_cq_user()
224 		cq->comp_handler = ib_cq_completion_softirq;  in __ib_alloc_cq_user()
226 		irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);  in __ib_alloc_cq_user()
227 		ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);  in __ib_alloc_cq_user()
231 		cq->comp_handler = ib_cq_completion_workqueue;  in __ib_alloc_cq_user()
232 		INIT_WORK(&cq->work, ib_cq_poll_work);  in __ib_alloc_cq_user()
233 		ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);  in __ib_alloc_cq_user()
234 		cq->comp_wq = (cq->poll_ctx == IB_POLL_WORKQUEUE) ?  in __ib_alloc_cq_user()
242 	return cq;  in __ib_alloc_cq_user()
245 	rdma_restrack_del(&cq->res);  in __ib_alloc_cq_user()
246 	cq->device->ops.destroy_cq(cq, udata);  in __ib_alloc_cq_user()
248 	kfree(cq->wc);  in __ib_alloc_cq_user()
250 	kfree(cq);  in __ib_alloc_cq_user()
288 void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)  in ib_free_cq_user()  argument
290 	if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))  in ib_free_cq_user()
293 	switch (cq->poll_ctx) {  in ib_free_cq_user()
297 		irq_poll_disable(&cq->iop);  in ib_free_cq_user()
301 		cancel_work_sync(&cq->work);  in ib_free_cq_user()
307 	rdma_restrack_del(&cq->res);  in ib_free_cq_user()
308 	cq->device->ops.destroy_cq(cq, udata);  in ib_free_cq_user()
309 	if (cq->dim)  in ib_free_cq_user()
310 		cancel_work_sync(&cq->dim->work);  in ib_free_cq_user()
311 	kfree(cq->dim);  in ib_free_cq_user()
312 	kfree(cq->wc);  in ib_free_cq_user()
313 	kfree(cq);  in ib_free_cq_user()