87 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
89 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
90 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
91 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
93 	sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_sq()
110 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
112 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
113 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
114 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
116 	cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,  in ena_com_admin_init_cq()
181 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
184 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
185 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
187 			   command_id, admin_queue->q_depth);  in get_comp_ctxt()
191 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
192 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
197 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
198 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
204 		atomic_inc(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
205 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
208 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
211 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
222 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
224 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
227 	cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
228 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
229 		netdev_dbg(admin_queue->ena_dev->net_device,  in __ena_com_submit_admin_cmd()
231 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
235 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
237 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
243 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
254 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
256 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
259 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
260 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
262 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
263 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
265 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
270 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
272 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
273 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
277 	admin_queue->comp_ctx =  in ena_com_init_comp_ctxt()
278 		devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);  in ena_com_init_comp_ctxt()
279 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
284 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
285 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
293 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
302 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
303 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
304 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
307 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
312 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
313 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
449 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
458 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
460 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_handle_single_admin_completion()
462 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
472 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
476 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
483 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
484 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
486 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
495 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
499 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
504 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
507 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
508 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
509 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
510 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
513 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
517 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_comp_status_to_errno()
548 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
555 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
558 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
559 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
560 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
566 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
569 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
570 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
571 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
572 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
579 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
583 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
585 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
586 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
587 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
595 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
597 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
609 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
616 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
630 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
777 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
784 					    admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
792 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
793 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
794 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
795 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
798 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
801 				   admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
803 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
804 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
806 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
814 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
815 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
821 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
823 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
897 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
899 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
901 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
904 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
910 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
930 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
1024 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
1035 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1057 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1227 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1270 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1354 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1363 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1368 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1371 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1377 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1379 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1380 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1383 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1392 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1417 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1468 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1472 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1475 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1476 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1488 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1492 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1493 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1494 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1497 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1499 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1505 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1515 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1530 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1535 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1538 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1539 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1540 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1557 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1577 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1584 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1674 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1677 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1680 	devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_free_ena_admin_queue_comp_ctx()
1682 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1687 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1688 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1689 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1693 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1695 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1701 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1722 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1728 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1791 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1808 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1810 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1811 	admin_queue->polling = false;  in ena_com_admin_init()
1812 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1814 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1816 	spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1818 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1822 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1826 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1830 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1833 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1834 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1839 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1840 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1846 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1852 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1863 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1864 	admin_queue->running_state = true;  in ena_com_admin_init()
2050 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2189 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2191 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2202 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2205 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2211 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2263 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2275 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2282 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2316 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2362 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2491 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2524 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2655 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2695 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2839 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2850 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2873 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()