46 	{ NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
238 void nvme_cmd_request_free(struct nvme_request *request)  in nvme_cmd_request_free()  argument
240 	if (sys_dnode_is_linked(&request->node)) {  in nvme_cmd_request_free()
241 		sys_dlist_remove(&request->node);  in nvme_cmd_request_free()
244 	if (request->prp_list != NULL) {  in nvme_cmd_request_free()
245 		nvme_prp_list_free(request->prp_list);  in nvme_cmd_request_free()
248 	memset(request, 0, sizeof(struct nvme_request));  in nvme_cmd_request_free()
249 	sys_dlist_append(&free_request, &request->node);  in nvme_cmd_request_free()
258 		LOG_ERR("Could not allocate request");  in nvme_cmd_request_alloc()
267 static void nvme_cmd_register_request(struct nvme_request *request)  in nvme_cmd_register_request()  argument
269 	sys_dlist_append(&pending_request, &request->node);  in nvme_cmd_register_request()
271 	request->req_start = k_uptime_get_32();  in nvme_cmd_register_request()
282 	struct nvme_request *request, *next;  in request_timeout()  local
287 					  request, next, node) {  in request_timeout()
288 		if ((int32_t)(request->req_start +  in request_timeout()
293 		LOG_WRN("Request %p CID %u timed-out",  in request_timeout()
294 			request, request->cmd.cdw0.cid);  in request_timeout()
302 		if (request->cb_fn) {  in request_timeout()
303 			request->cb_fn(request->cb_arg, NULL);  in request_timeout()
306 		nvme_cmd_request_free(request);  in request_timeout()
309 	if (request) {  in request_timeout()
311 				  K_SECONDS(request->req_start +  in request_timeout()
378 static void nvme_cmd_request_complete(struct nvme_request *request,  in nvme_cmd_request_complete()  argument
386 		request->retries < CONFIG_NVME_RETRY_COUNT;  in nvme_cmd_request_complete()
390 		request->qpair->num_retries++;  in nvme_cmd_request_complete()
394 	    (!retriable || (request->retries >= CONFIG_NVME_RETRY_COUNT))) {  in nvme_cmd_request_complete()
396 		request->qpair->num_failures++;  in nvme_cmd_request_complete()
399 	if (cpl->cid != request->cmd.cdw0.cid) {  in nvme_cmd_request_complete()
406 		sys_dlist_remove(&request->node);  in nvme_cmd_request_complete()
408 		nvme_cmd_qpair_submit_request(request->qpair, request);  in nvme_cmd_request_complete()
409 		request->retries++;  in nvme_cmd_request_complete()
411 		LOG_DBG("Request %p CMD complete on %p/%p",  in nvme_cmd_request_complete()
412 			request, request->cb_fn, request->cb_arg);  in nvme_cmd_request_complete()
414 		if (request->cb_fn) {  in nvme_cmd_request_complete()
415 			request->cb_fn(request->cb_arg, cpl);  in nvme_cmd_request_complete()
418 		nvme_cmd_request_free(request);  in nvme_cmd_request_complete()
424 	struct nvme_request *request;  in nvme_cmd_qpair_process_completion()  local
450 			request = &request_pool[cpl.cid];  in nvme_cmd_qpair_process_completion()
452 			request = NULL;  in nvme_cmd_qpair_process_completion()
456 		if (request != NULL) {  in nvme_cmd_qpair_process_completion()
457 			nvme_cmd_request_complete(request, &cpl);  in nvme_cmd_qpair_process_completion()
544 					struct nvme_request *request,  in nvme_cmd_qpair_fill_prp_list()  argument
556 	p_addr = (uintptr_t)request->payload;  in nvme_cmd_qpair_fill_prp_list()
557 	request->cmd.dptr.prp1 =  in nvme_cmd_qpair_fill_prp_list()
559 	request->cmd.dptr.prp2 =  in nvme_cmd_qpair_fill_prp_list()
568 	request->prp_list = prp_list;  in nvme_cmd_qpair_fill_prp_list()
600 				    struct nvme_request *request)  in nvme_cmd_qpair_fill_dptr()  argument
602 	switch (request->type) {  in nvme_cmd_qpair_fill_dptr()
608 		if (request->payload_size > qpair->ctrlr->max_xfer_size) {  in nvme_cmd_qpair_fill_dptr()
609 			LOG_ERR("VADDR request's payload too big");  in nvme_cmd_qpair_fill_dptr()
613 		n_prp = compute_n_prp((uintptr_t)request->payload,  in nvme_cmd_qpair_fill_dptr()
614 				      request->payload_size);  in nvme_cmd_qpair_fill_dptr()
616 			request->cmd.dptr.prp1 =  in nvme_cmd_qpair_fill_dptr()
617 				(uint64_t)sys_cpu_to_le64(request->payload);  in nvme_cmd_qpair_fill_dptr()
619 				request->cmd.dptr.prp2 = (uint64_t)sys_cpu_to_le64(  in nvme_cmd_qpair_fill_dptr()
621 						(uintptr_t)request->payload));  in nvme_cmd_qpair_fill_dptr()
623 				request->cmd.dptr.prp2 = 0;  in nvme_cmd_qpair_fill_dptr()
629 		return nvme_cmd_qpair_fill_prp_list(qpair, request, n_prp);  in nvme_cmd_qpair_fill_dptr()
638 				  struct nvme_request *request)  in nvme_cmd_qpair_submit_request()  argument
643 	request->qpair = qpair;  in nvme_cmd_qpair_submit_request()
645 	request->cmd.cdw0.cid = sys_cpu_to_le16((uint16_t)(request -  in nvme_cmd_qpair_submit_request()
648 	ret = nvme_cmd_qpair_fill_dptr(qpair, request);  in nvme_cmd_qpair_submit_request()
650 		nvme_cmd_request_free(request);  in nvme_cmd_qpair_submit_request()
654 	nvme_cmd_register_request(request);  in nvme_cmd_qpair_submit_request()
657 	       &request->cmd, sizeof(request->cmd));  in nvme_cmd_qpair_submit_request()
667 	LOG_DBG("Request %p %llu submitted: CID %u - sq_tail %u",  in nvme_cmd_qpair_submit_request()
668 		request, qpair->num_cmds, request->cmd.cdw0.cid,  in nvme_cmd_qpair_submit_request()