390 		request->qpair->num_retries++;  in nvme_cmd_request_complete()
396 		request->qpair->num_failures++;  in nvme_cmd_request_complete()
408 		nvme_cmd_qpair_submit_request(request->qpair, request);  in nvme_cmd_request_complete()
422 static void nvme_cmd_qpair_process_completion(struct nvme_cmd_qpair *qpair)  in nvme_cmd_qpair_process_completion()  argument
428 	if (qpair->num_intr_handler_calls == 0 && qpair->phase == 0) {  in nvme_cmd_qpair_process_completion()
432 	qpair->num_intr_handler_calls++;  in nvme_cmd_qpair_process_completion()
437 		status = sys_le16_to_cpu(qpair->cpl[qpair->cq_head].status);  in nvme_cmd_qpair_process_completion()
438 		if (NVME_STATUS_GET_P(status) != qpair->phase) {  in nvme_cmd_qpair_process_completion()
442 		cpl = qpair->cpl[qpair->cq_head];  in nvme_cmd_qpair_process_completion()
458 			qpair->sq_head = cpl.sqhd;  in nvme_cmd_qpair_process_completion()
463 		qpair->cq_head++;  in nvme_cmd_qpair_process_completion()
464 		if (qpair->cq_head == qpair->num_entries) {  in nvme_cmd_qpair_process_completion()
465 			qpair->cq_head = 0;  in nvme_cmd_qpair_process_completion()
466 			qpair->phase = !qpair->phase;  in nvme_cmd_qpair_process_completion()
471 		mm_reg_t regs = DEVICE_MMIO_GET(qpair->ctrlr->dev);  in nvme_cmd_qpair_process_completion()
473 		sys_write32(qpair->cq_head, regs + qpair->cq_hdbl_off);  in nvme_cmd_qpair_process_completion()
479 	const struct nvme_cmd_qpair *qpair = arg;  in nvme_cmd_qpair_msi_handler()  local
481 	nvme_cmd_qpair_process_completion((struct nvme_cmd_qpair *)qpair);  in nvme_cmd_qpair_msi_handler()
484 int nvme_cmd_qpair_setup(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_setup()  argument
491 	qpair->ctrlr = ctrlr;  in nvme_cmd_qpair_setup()
492 	qpair->id = id;  in nvme_cmd_qpair_setup()
493 	qpair->vector = qpair->id;  in nvme_cmd_qpair_setup()
495 	qpair->num_cmds = 0;  in nvme_cmd_qpair_setup()
496 	qpair->num_intr_handler_calls = 0;  in nvme_cmd_qpair_setup()
497 	qpair->num_retries = 0;  in nvme_cmd_qpair_setup()
498 	qpair->num_failures = 0;  in nvme_cmd_qpair_setup()
499 	qpair->num_ignored = 0;  in nvme_cmd_qpair_setup()
501 	qpair->cmd_bus_addr = (uintptr_t)qpair->cmd;  in nvme_cmd_qpair_setup()
502 	qpair->cpl_bus_addr = (uintptr_t)qpair->cpl;  in nvme_cmd_qpair_setup()
504 	qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell) +  in nvme_cmd_qpair_setup()
505 		(qpair->id << (ctrlr->dstrd + 1));  in nvme_cmd_qpair_setup()
506 	qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell) +  in nvme_cmd_qpair_setup()
507 		(qpair->id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);  in nvme_cmd_qpair_setup()
510 				     &ctrlr->vectors[qpair->vector],  in nvme_cmd_qpair_setup()
511 				     nvme_cmd_qpair_msi_handler, qpair, 0)) {  in nvme_cmd_qpair_setup()
512 		LOG_ERR("Failed to connect MSI-X vector %u", qpair->id);  in nvme_cmd_qpair_setup()
518 		qpair->id, qpair->num_entries, qpair->cmd_bus_addr,  in nvme_cmd_qpair_setup()
519 		qpair->cpl_bus_addr, qpair->sq_tdbl_off, qpair->cq_hdbl_off);  in nvme_cmd_qpair_setup()
524 void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair)  in nvme_cmd_qpair_reset()  argument
526 	qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;  in nvme_cmd_qpair_reset()
535 	qpair->phase = 1;  in nvme_cmd_qpair_reset()
537 	memset(qpair->cmd, 0,  in nvme_cmd_qpair_reset()
538 	       qpair->num_entries * sizeof(struct nvme_command));  in nvme_cmd_qpair_reset()
539 	memset(qpair->cpl, 0,  in nvme_cmd_qpair_reset()
540 	       qpair->num_entries * sizeof(struct nvme_completion));  in nvme_cmd_qpair_reset()
543 static int nvme_cmd_qpair_fill_prp_list(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_fill_prp_list()  argument
599 static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_fill_dptr()  argument
608 		if (request->payload_size > qpair->ctrlr->max_xfer_size) {  in nvme_cmd_qpair_fill_dptr()
629 		return nvme_cmd_qpair_fill_prp_list(qpair, request, n_prp);  in nvme_cmd_qpair_fill_dptr()
637 int nvme_cmd_qpair_submit_request(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_submit_request()  argument
640 	mm_reg_t regs = DEVICE_MMIO_GET(qpair->ctrlr->dev);  in nvme_cmd_qpair_submit_request()
643 	request->qpair = qpair;  in nvme_cmd_qpair_submit_request()
648 	ret = nvme_cmd_qpair_fill_dptr(qpair, request);  in nvme_cmd_qpair_submit_request()
656 	memcpy(&qpair->cmd[qpair->sq_tail],  in nvme_cmd_qpair_submit_request()
659 	qpair->sq_tail++;  in nvme_cmd_qpair_submit_request()
660 	if (qpair->sq_tail == qpair->num_entries) {  in nvme_cmd_qpair_submit_request()
661 		qpair->sq_tail = 0;  in nvme_cmd_qpair_submit_request()
664 	sys_write32(qpair->sq_tail, regs + qpair->sq_tdbl_off);  in nvme_cmd_qpair_submit_request()
665 	qpair->num_cmds++;  in nvme_cmd_qpair_submit_request()
668 		request, qpair->num_cmds, request->cmd.cdw0.cid,  in nvme_cmd_qpair_submit_request()
669 		qpair->sq_tail - 1);  in nvme_cmd_qpair_submit_request()