505 	if (req->sq->size) {  in __nvmet_req_complete()
507 			old_sqhd = req->sq->sqhd;  in __nvmet_req_complete()
508 			new_sqhd = (old_sqhd + 1) % req->sq->size;  in __nvmet_req_complete()
509 		} while (cmpxchg(&req->sq->sqhd, old_sqhd, new_sqhd) !=  in __nvmet_req_complete()
512 	sqhd = req->sq->sqhd & 0x0000FFFF;  in __nvmet_req_complete()
514 	req->rsp->sq_id = cpu_to_le16(req->sq->qid);  in __nvmet_req_complete()
525 	percpu_ref_put(&req->sq->ref);  in nvmet_req_complete()
538 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,  in nvmet_sq_setup()  argument
541 	sq->sqhd = 0;  in nvmet_sq_setup()
542 	sq->qid = qid;  in nvmet_sq_setup()
543 	sq->size = size;  in nvmet_sq_setup()
545 	ctrl->sqs[qid] = sq;  in nvmet_sq_setup()
550 	struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);  in nvmet_confirm_sq()  local
552 	complete(&sq->confirm_done);  in nvmet_confirm_sq()
555 void nvmet_sq_destroy(struct nvmet_sq *sq)  in nvmet_sq_destroy()  argument
561 	if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq)  in nvmet_sq_destroy()
562 		nvmet_async_events_free(sq->ctrl);  in nvmet_sq_destroy()
563 	percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq);  in nvmet_sq_destroy()
564 	wait_for_completion(&sq->confirm_done);  in nvmet_sq_destroy()
565 	wait_for_completion(&sq->free_done);  in nvmet_sq_destroy()
566 	percpu_ref_exit(&sq->ref);  in nvmet_sq_destroy()
568 	if (sq->ctrl) {  in nvmet_sq_destroy()
569 		nvmet_ctrl_put(sq->ctrl);  in nvmet_sq_destroy()
570 		sq->ctrl = NULL; /* allows reusing the queue later */  in nvmet_sq_destroy()
577 	struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);  in nvmet_sq_free()  local
579 	complete(&sq->free_done);  in nvmet_sq_free()
582 int nvmet_sq_init(struct nvmet_sq *sq)  in nvmet_sq_init()  argument
586 	ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL);  in nvmet_sq_init()
591 	init_completion(&sq->free_done);  in nvmet_sq_init()
592 	init_completion(&sq->confirm_done);  in nvmet_sq_init()
636 	req->ns = nvmet_find_namespace(req->sq->ctrl, cmd->rw.nsid);  in nvmet_parse_io_cmd()
653 		struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops)  in nvmet_req_init()  argument
659 	req->sq = sq;  in nvmet_req_init()
683 	if (unlikely(!req->sq->ctrl))  in nvmet_req_init()
686 	else if (likely(req->sq->qid != 0))  in nvmet_req_init()
690 	else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC)  in nvmet_req_init()
698 	if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {  in nvmet_req_init()
713 	percpu_ref_put(&req->sq->ref);  in nvmet_req_uninit()
870 	if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) {  in nvmet_check_ctrl_status()
872 		       cmd->common.opcode, req->sq->qid);  in nvmet_check_ctrl_status()
876 	if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) {  in nvmet_check_ctrl_status()
878 		       cmd->common.opcode, req->sq->qid);  in nvmet_check_ctrl_status()