223 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
234 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
236 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
239 	cnt = atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
240 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
242 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
246 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
248 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
254 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
265 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
267 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
270 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
271 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
273 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
274 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
276 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
302 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
311 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
312 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
313 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
316 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
321 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
322 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
423 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
432 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
435 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
445 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
449 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
456 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
457 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
459 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
468 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
472 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
477 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
480 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
481 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
482 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
483 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
512 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
517 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
520 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
521 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
522 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
530 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
531 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
532 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
533 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
544 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
545 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
546 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
556 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
561 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
568 					    admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
576 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
577 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
578 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
579 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
588 		admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
595 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
667 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
669 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
671 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
674 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
680 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
700 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
787 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
797 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
819 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
964 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1006 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1122 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1131 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1144 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1146 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1157 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1182 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1230 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1234 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1237 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1238 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1250 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1253 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1254 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1255 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1257 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1259 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1265 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1275 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1289 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1294 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1297 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1298 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1299 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1316 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1335 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1342 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1433 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1434 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1435 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1439 	if (admin_queue->comp_ctx)  in ena_com_admin_destroy()
1440 		devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_admin_destroy()
1441 	admin_queue->comp_ctx = NULL;  in ena_com_admin_destroy()
1442 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1448 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1469 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1529 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1545 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1547 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1548 	admin_queue->polling = false;  in ena_com_admin_init()
1549 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1551 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1554 		spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1556 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1560 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1564 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1568 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1571 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1572 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1577 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1578 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1584 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1590 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1601 	admin_queue->running_state = true;  in ena_com_admin_init()
1756 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
1890 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
1892 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
1903 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
1906 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
1912 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
1941 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
1952 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
1959 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
1991 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2036 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2155 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2188 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2315 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2354 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2491 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2502 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2525 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()