137 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init_sq()  local
138 	struct efa_com_admin_sq *sq = &aq->sq;  in efa_com_admin_init_sq()
139 	u16 size = aq->depth * sizeof(*sq->entries);  in efa_com_admin_init_sq()
145 		dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);  in efa_com_admin_init_sq()
163 	aq_caps = aq->depth & EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in efa_com_admin_init_sq()
175 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init_cq()  local
176 	struct efa_com_admin_cq *cq = &aq->cq;  in efa_com_admin_init_cq()
177 	u16 size = aq->depth * sizeof(*cq->entries);  in efa_com_admin_init_cq()
183 		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);  in efa_com_admin_init_cq()
198 	acq_caps = aq->depth & EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in efa_com_admin_init_cq()
202 	acq_caps |= (aq->msix_vector_idx <<  in efa_com_admin_init_cq()
259 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq)  in efa_com_alloc_ctx_id()  argument
263 	spin_lock(&aq->comp_ctx_lock);  in efa_com_alloc_ctx_id()
264 	ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next];  in efa_com_alloc_ctx_id()
265 	aq->comp_ctx_pool_next++;  in efa_com_alloc_ctx_id()
266 	spin_unlock(&aq->comp_ctx_lock);  in efa_com_alloc_ctx_id()
271 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,  in efa_com_dealloc_ctx_id()  argument
274 	spin_lock(&aq->comp_ctx_lock);  in efa_com_dealloc_ctx_id()
275 	aq->comp_ctx_pool_next--;  in efa_com_dealloc_ctx_id()
276 	aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id;  in efa_com_dealloc_ctx_id()
277 	spin_unlock(&aq->comp_ctx_lock);  in efa_com_dealloc_ctx_id()
280 static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,  in efa_com_put_comp_ctx()  argument
285 	u16 ctx_id = cmd_id & (aq->depth - 1);  in efa_com_put_comp_ctx()
287 	ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id);  in efa_com_put_comp_ctx()
289 	efa_com_dealloc_ctx_id(aq, ctx_id);  in efa_com_put_comp_ctx()
292 static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,  in efa_com_get_comp_ctx()  argument
295 	u16 ctx_id = cmd_id & (aq->depth - 1);  in efa_com_get_comp_ctx()
297 	if (aq->comp_ctx[ctx_id].occupied && capture) {  in efa_com_get_comp_ctx()
299 			aq->efa_dev,  in efa_com_get_comp_ctx()
306 		aq->comp_ctx[ctx_id].occupied = 1;  in efa_com_get_comp_ctx()
307 		ibdev_dbg(aq->efa_dev,  in efa_com_get_comp_ctx()
311 	return &aq->comp_ctx[ctx_id];  in efa_com_get_comp_ctx()
314 static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,  in __efa_com_submit_admin_cmd()  argument
326 	queue_size_mask = aq->depth - 1;  in __efa_com_submit_admin_cmd()
327 	pi = aq->sq.pc & queue_size_mask;  in __efa_com_submit_admin_cmd()
329 	ctx_id = efa_com_alloc_ctx_id(aq);  in __efa_com_submit_admin_cmd()
333 	cmd_id |= aq->sq.pc & ~queue_size_mask;  in __efa_com_submit_admin_cmd()
337 	cmd->aq_common_descriptor.flags |= aq->sq.phase &  in __efa_com_submit_admin_cmd()
340 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);  in __efa_com_submit_admin_cmd()
342 		efa_com_dealloc_ctx_id(aq, ctx_id);  in __efa_com_submit_admin_cmd()
353 	memcpy(&aq->sq.entries[pi], cmd, cmd_size_in_bytes);  in __efa_com_submit_admin_cmd()
355 	aq->sq.pc++;  in __efa_com_submit_admin_cmd()
356 	atomic64_inc(&aq->stats.submitted_cmd);  in __efa_com_submit_admin_cmd()
358 	if ((aq->sq.pc & queue_size_mask) == 0)  in __efa_com_submit_admin_cmd()
359 		aq->sq.phase = !aq->sq.phase;  in __efa_com_submit_admin_cmd()
362 	writel(aq->sq.pc, aq->sq.db_addr);  in __efa_com_submit_admin_cmd()
367 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)  in efa_com_init_comp_ctxt()  argument
369 	size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool);  in efa_com_init_comp_ctxt()
370 	size_t size = aq->depth * sizeof(struct efa_comp_ctx);  in efa_com_init_comp_ctxt()
374 	aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL);  in efa_com_init_comp_ctxt()
375 	aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL);  in efa_com_init_comp_ctxt()
376 	if (!aq->comp_ctx || !aq->comp_ctx_pool) {  in efa_com_init_comp_ctxt()
377 		devm_kfree(aq->dmadev, aq->comp_ctx_pool);  in efa_com_init_comp_ctxt()
378 		devm_kfree(aq->dmadev, aq->comp_ctx);  in efa_com_init_comp_ctxt()
382 	for (i = 0; i < aq->depth; i++) {  in efa_com_init_comp_ctxt()
383 		comp_ctx = efa_com_get_comp_ctx(aq, i, false);  in efa_com_init_comp_ctxt()
387 		aq->comp_ctx_pool[i] = i;  in efa_com_init_comp_ctxt()
390 	spin_lock_init(&aq->comp_ctx_lock);  in efa_com_init_comp_ctxt()
392 	aq->comp_ctx_pool_next = 0;  in efa_com_init_comp_ctxt()
397 static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,  in efa_com_submit_admin_cmd()  argument
405 	spin_lock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
406 	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {  in efa_com_submit_admin_cmd()
407 		ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n");  in efa_com_submit_admin_cmd()
408 		spin_unlock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
412 	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,  in efa_com_submit_admin_cmd()
414 	spin_unlock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
416 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_submit_admin_cmd()
421 static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,  in efa_com_handle_single_admin_completion()  argument
430 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);  in efa_com_handle_single_admin_completion()
432 		ibdev_err(aq->efa_dev,  in efa_com_handle_single_admin_completion()
434 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_handle_single_admin_completion()
443 	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))  in efa_com_handle_single_admin_completion()
447 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)  in efa_com_handle_admin_completion()  argument
455 	queue_size_mask = aq->depth - 1;  in efa_com_handle_admin_completion()
457 	ci = aq->cq.cc & queue_size_mask;  in efa_com_handle_admin_completion()
458 	phase = aq->cq.phase;  in efa_com_handle_admin_completion()
460 	cqe = &aq->cq.entries[ci];  in efa_com_handle_admin_completion()
470 		efa_com_handle_single_admin_completion(aq, cqe);  in efa_com_handle_admin_completion()
474 		if (ci == aq->depth) {  in efa_com_handle_admin_completion()
479 		cqe = &aq->cq.entries[ci];  in efa_com_handle_admin_completion()
482 	aq->cq.cc += comp_num;  in efa_com_handle_admin_completion()
483 	aq->cq.phase = phase;  in efa_com_handle_admin_completion()
484 	aq->sq.cc += comp_num;  in efa_com_handle_admin_completion()
485 	atomic64_add(comp_num, &aq->stats.completed_cmd);  in efa_com_handle_admin_completion()
508 						     struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq_polling()  argument
514 	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);  in efa_com_wait_and_process_admin_cq_polling()
517 		spin_lock_irqsave(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_polling()
518 		efa_com_handle_admin_completion(aq);  in efa_com_wait_and_process_admin_cq_polling()
519 		spin_unlock_irqrestore(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_polling()
526 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_polling()
529 			atomic64_inc(&aq->stats.no_completion);  in efa_com_wait_and_process_admin_cq_polling()
531 			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_wait_and_process_admin_cq_polling()
536 		msleep(aq->poll_interval);  in efa_com_wait_and_process_admin_cq_polling()
541 	efa_com_put_comp_ctx(aq, comp_ctx);  in efa_com_wait_and_process_admin_cq_polling()
546 							struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq_interrupts()  argument
552 				    usecs_to_jiffies(aq->completion_timeout));  in efa_com_wait_and_process_admin_cq_interrupts()
561 		spin_lock_irqsave(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_interrupts()
562 		efa_com_handle_admin_completion(aq);  in efa_com_wait_and_process_admin_cq_interrupts()
563 		spin_unlock_irqrestore(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_interrupts()
565 		atomic64_inc(&aq->stats.no_completion);  in efa_com_wait_and_process_admin_cq_interrupts()
569 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_interrupts()
573 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);  in efa_com_wait_and_process_admin_cq_interrupts()
576 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_interrupts()
580 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);  in efa_com_wait_and_process_admin_cq_interrupts()
582 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_wait_and_process_admin_cq_interrupts()
589 	efa_com_put_comp_ctx(aq, comp_ctx);  in efa_com_wait_and_process_admin_cq_interrupts()
602 					     struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq()  argument
604 	if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))  in efa_com_wait_and_process_admin_cq()
605 		return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq);  in efa_com_wait_and_process_admin_cq()
607 	return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq);  in efa_com_wait_and_process_admin_cq()
623 int efa_com_cmd_exec(struct efa_com_admin_queue *aq,  in efa_com_cmd_exec()  argument
635 	down(&aq->avail_cmds);  in efa_com_cmd_exec()
637 	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",  in efa_com_cmd_exec()
640 	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);  in efa_com_cmd_exec()
643 			aq->efa_dev,  in efa_com_cmd_exec()
648 		up(&aq->avail_cmds);  in efa_com_cmd_exec()
652 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);  in efa_com_cmd_exec()
655 			aq->efa_dev,  in efa_com_cmd_exec()
661 	up(&aq->avail_cmds);  in efa_com_cmd_exec()
672 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_destroy()  local
674 	struct efa_com_admin_cq *cq = &aq->cq;  in efa_com_admin_destroy()
675 	struct efa_com_admin_sq *sq = &aq->sq;  in efa_com_admin_destroy()
678 	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_admin_destroy()
680 	devm_kfree(edev->dmadev, aq->comp_ctx_pool);  in efa_com_admin_destroy()
681 	devm_kfree(edev->dmadev, aq->comp_ctx);  in efa_com_admin_destroy()
683 	size = aq->depth * sizeof(*sq->entries);  in efa_com_admin_destroy()
686 	size = aq->depth * sizeof(*cq->entries);  in efa_com_admin_destroy()
709 		set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);  in efa_com_set_admin_polling_mode()
711 		clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);  in efa_com_set_admin_polling_mode()
716 	atomic64_t *s = (atomic64_t *)&edev->aq.stats;  in efa_com_stats_init()
719 	for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++)  in efa_com_stats_init()
736 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init()  local
749 	aq->depth = EFA_ADMIN_QUEUE_DEPTH;  in efa_com_admin_init()
751 	aq->dmadev = edev->dmadev;  in efa_com_admin_init()
752 	aq->efa_dev = edev->efa_dev;  in efa_com_admin_init()
753 	set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);  in efa_com_admin_init()
755 	sema_init(&aq->avail_cmds, aq->depth);  in efa_com_admin_init()
759 	err = efa_com_init_comp_ctxt(aq);  in efa_com_admin_init()
782 		aq->completion_timeout = timeout * 100000;  in efa_com_admin_init()
784 		aq->completion_timeout = ADMIN_CMD_TIMEOUT_US;  in efa_com_admin_init()
786 	aq->poll_interval = EFA_POLL_INTERVAL_MS;  in efa_com_admin_init()
788 	set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_admin_init()
793 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries),  in efa_com_admin_init()
794 			  aq->cq.entries, aq->cq.dma_addr);  in efa_com_admin_init()
796 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries),  in efa_com_admin_init()
797 			  aq->sq.entries, aq->sq.dma_addr);  in efa_com_admin_init()
799 	devm_kfree(edev->dmadev, aq->comp_ctx);  in efa_com_admin_init()
817 	spin_lock_irqsave(&edev->aq.cq.lock, flags);  in efa_com_admin_q_comp_intr_handler()
818 	efa_com_handle_admin_completion(&edev->aq);  in efa_com_admin_q_comp_intr_handler()
819 	spin_unlock_irqrestore(&edev->aq.cq.lock, flags);  in efa_com_admin_q_comp_intr_handler()
1093 		edev->aq.completion_timeout = timeout * 100000;  in efa_com_dev_reset()
1095 		edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in efa_com_dev_reset()