84 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
86 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
87 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
89 	sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_sq()
106 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
108 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
109 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
111 	cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_cq()
175 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
178 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
180 		       command_id, admin_queue->q_depth);  in get_comp_ctxt()
184 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
189 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
195 		atomic_inc(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
196 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
199 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
202 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
213 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
215 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
218 	cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
219 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
221 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
225 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
227 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
233 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
244 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
246 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
249 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
250 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
252 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
253 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
255 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
260 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
262 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
266 	admin_queue->comp_ctx =  in ena_com_init_comp_ctxt()
267 		devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);  in ena_com_init_comp_ctxt()
268 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
273 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
274 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
282 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
291 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
292 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
293 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
296 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
301 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
302 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
436 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
445 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
448 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
458 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
462 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
469 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
470 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
472 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
481 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
485 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
490 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
493 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
494 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
495 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
496 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
531 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
538 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
541 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
542 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
543 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
551 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
552 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
553 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
554 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
561 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
566 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
567 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
568 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
578 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
590 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
597 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
611 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
748 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
755 					    admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
763 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
764 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
765 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
766 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
771 			       admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
773 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
774 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
783 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
784 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
792 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
864 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
866 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
868 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
871 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
877 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
897 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
989 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
999 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1021 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1189 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1231 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1312 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1321 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1334 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1336 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1347 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1372 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1420 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1424 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1427 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1428 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1440 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1444 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1445 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1446 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1449 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1451 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1457 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1467 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1481 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1486 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1489 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1490 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1491 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1508 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1527 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1534 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1620 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1623 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1626 	devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_free_ena_admin_queue_comp_ctx()
1628 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1633 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1634 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1635 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1639 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1641 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1647 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1668 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1674 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1737 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1753 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1755 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1756 	admin_queue->polling = false;  in ena_com_admin_init()
1757 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1759 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1761 	spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1763 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1767 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1771 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1775 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1778 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1779 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1784 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1785 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1791 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1797 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1808 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1809 	admin_queue->running_state = true;  in ena_com_admin_init()
1991 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2126 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2128 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2139 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2142 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2148 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2192 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2203 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2210 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2242 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2287 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2423 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2456 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2583 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2622 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2765 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2776 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2799 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()