387 		request->qpair->num_retries++;  in nvme_cmd_request_complete()
393 		request->qpair->num_failures++;  in nvme_cmd_request_complete()
405 		nvme_cmd_qpair_submit_request(request->qpair, request);  in nvme_cmd_request_complete()
419 static void nvme_cmd_qpair_process_completion(struct nvme_cmd_qpair *qpair)  in nvme_cmd_qpair_process_completion()  argument
425 	if (qpair->num_intr_handler_calls == 0 && qpair->phase == 0) {  in nvme_cmd_qpair_process_completion()
429 	qpair->num_intr_handler_calls++;  in nvme_cmd_qpair_process_completion()
434 		status = sys_le16_to_cpu(qpair->cpl[qpair->cq_head].status);  in nvme_cmd_qpair_process_completion()
435 		if (NVME_STATUS_GET_P(status) != qpair->phase) {  in nvme_cmd_qpair_process_completion()
439 		cpl = qpair->cpl[qpair->cq_head];  in nvme_cmd_qpair_process_completion()
455 			qpair->sq_head = cpl.sqhd;  in nvme_cmd_qpair_process_completion()
460 		qpair->cq_head++;  in nvme_cmd_qpair_process_completion()
461 		if (qpair->cq_head == qpair->num_entries) {  in nvme_cmd_qpair_process_completion()
462 			qpair->cq_head = 0;  in nvme_cmd_qpair_process_completion()
463 			qpair->phase = !qpair->phase;  in nvme_cmd_qpair_process_completion()
468 		mm_reg_t regs = DEVICE_MMIO_GET(qpair->ctrlr->dev);  in nvme_cmd_qpair_process_completion()
470 		sys_write32(qpair->cq_head, regs + qpair->cq_hdbl_off);  in nvme_cmd_qpair_process_completion()
476 	const struct nvme_cmd_qpair *qpair = arg;  in nvme_cmd_qpair_msi_handler()  local
478 	nvme_cmd_qpair_process_completion((struct nvme_cmd_qpair *)qpair);  in nvme_cmd_qpair_msi_handler()
481 int nvme_cmd_qpair_setup(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_setup()  argument
488 	qpair->ctrlr = ctrlr;  in nvme_cmd_qpair_setup()
489 	qpair->id = id;  in nvme_cmd_qpair_setup()
490 	qpair->vector = qpair->id;  in nvme_cmd_qpair_setup()
492 	qpair->num_cmds = 0;  in nvme_cmd_qpair_setup()
493 	qpair->num_intr_handler_calls = 0;  in nvme_cmd_qpair_setup()
494 	qpair->num_retries = 0;  in nvme_cmd_qpair_setup()
495 	qpair->num_failures = 0;  in nvme_cmd_qpair_setup()
496 	qpair->num_ignored = 0;  in nvme_cmd_qpair_setup()
498 	qpair->cmd_bus_addr = (uintptr_t)qpair->cmd;  in nvme_cmd_qpair_setup()
499 	qpair->cpl_bus_addr = (uintptr_t)qpair->cpl;  in nvme_cmd_qpair_setup()
501 	qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell) +  in nvme_cmd_qpair_setup()
502 		(qpair->id << (ctrlr->dstrd + 1));  in nvme_cmd_qpair_setup()
503 	qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell) +  in nvme_cmd_qpair_setup()
504 		(qpair->id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);  in nvme_cmd_qpair_setup()
507 				     &ctrlr->vectors[qpair->vector],  in nvme_cmd_qpair_setup()
508 				     nvme_cmd_qpair_msi_handler, qpair, 0)) {  in nvme_cmd_qpair_setup()
509 		LOG_ERR("Failed to connect MSI-X vector %u", qpair->id);  in nvme_cmd_qpair_setup()
515 		qpair->id, qpair->num_entries, qpair->cmd_bus_addr,  in nvme_cmd_qpair_setup()
516 		qpair->cpl_bus_addr, qpair->sq_tdbl_off, qpair->cq_hdbl_off);  in nvme_cmd_qpair_setup()
521 void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair)  in nvme_cmd_qpair_reset()  argument
523 	qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;  in nvme_cmd_qpair_reset()
532 	qpair->phase = 1;  in nvme_cmd_qpair_reset()
534 	memset(qpair->cmd, 0,  in nvme_cmd_qpair_reset()
535 	       qpair->num_entries * sizeof(struct nvme_command));  in nvme_cmd_qpair_reset()
536 	memset(qpair->cpl, 0,  in nvme_cmd_qpair_reset()
537 	       qpair->num_entries * sizeof(struct nvme_completion));  in nvme_cmd_qpair_reset()
540 static int nvme_cmd_qpair_fill_prp_list(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_fill_prp_list()  argument
596 static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_fill_dptr()  argument
605 		if (request->payload_size > qpair->ctrlr->max_xfer_size) {  in nvme_cmd_qpair_fill_dptr()
626 		return nvme_cmd_qpair_fill_prp_list(qpair, request, n_prp);  in nvme_cmd_qpair_fill_dptr()
634 int nvme_cmd_qpair_submit_request(struct nvme_cmd_qpair *qpair,  in nvme_cmd_qpair_submit_request()  argument
637 	mm_reg_t regs = DEVICE_MMIO_GET(qpair->ctrlr->dev);  in nvme_cmd_qpair_submit_request()
640 	request->qpair = qpair;  in nvme_cmd_qpair_submit_request()
645 	ret = nvme_cmd_qpair_fill_dptr(qpair, request);  in nvme_cmd_qpair_submit_request()
653 	memcpy(&qpair->cmd[qpair->sq_tail],  in nvme_cmd_qpair_submit_request()
656 	qpair->sq_tail++;  in nvme_cmd_qpair_submit_request()
657 	if (qpair->sq_tail == qpair->num_entries) {  in nvme_cmd_qpair_submit_request()
658 		qpair->sq_tail = 0;  in nvme_cmd_qpair_submit_request()
661 	sys_write32(qpair->sq_tail, regs + qpair->sq_tdbl_off);  in nvme_cmd_qpair_submit_request()
662 	qpair->num_cmds++;  in nvme_cmd_qpair_submit_request()
665 		request, qpair->num_cmds, request->cmd.cdw0.cid,  in nvme_cmd_qpair_submit_request()
666 		qpair->sq_tail - 1);  in nvme_cmd_qpair_submit_request()