167 		u32 meta_seed, void **metap, struct io_uring_cmd *ioucmd,  in nvme_map_user_request()  argument
177 	if (ioucmd && (ioucmd->flags & IORING_URING_CMD_FIXED)) {  in nvme_map_user_request()
184 				rq_data_dir(req), &iter, ioucmd);  in nvme_map_user_request()
465 		struct io_uring_cmd *ioucmd)  in nvme_uring_cmd_pdu()  argument
467 	return (struct nvme_uring_cmd_pdu *)&ioucmd->pdu;  in nvme_uring_cmd_pdu()
470 static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd,  in nvme_uring_task_meta_cb()  argument
473 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_task_meta_cb()
492 	io_uring_cmd_done(ioucmd, status, result, issue_flags);  in nvme_uring_task_meta_cb()
495 static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,  in nvme_uring_task_cb()  argument
498 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_task_cb()
503 	io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result, issue_flags);  in nvme_uring_task_cb()
509 	struct io_uring_cmd *ioucmd = req->end_io_data;  in nvme_uring_cmd_end_io()  local
510 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_end_io()
524 		WRITE_ONCE(ioucmd->cookie, NULL);  in nvme_uring_cmd_end_io()
525 		nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED);  in nvme_uring_cmd_end_io()
527 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);  in nvme_uring_cmd_end_io()
536 	struct io_uring_cmd *ioucmd = req->end_io_data;  in nvme_uring_cmd_end_io_meta()  local
537 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_end_io_meta()
547 		WRITE_ONCE(ioucmd->cookie, NULL);  in nvme_uring_cmd_end_io_meta()
548 		nvme_uring_task_meta_cb(ioucmd, IO_URING_F_UNLOCKED);  in nvme_uring_cmd_end_io_meta()
550 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_meta_cb);  in nvme_uring_cmd_end_io_meta()
557 		struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)  in nvme_uring_cmd_io()  argument
559 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_io()
560 	const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);  in nvme_uring_cmd_io()
591 	if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE))  in nvme_uring_cmd_io()
615 			d.metadata_len, 0, &meta, ioucmd, vec);  in nvme_uring_cmd_io()
621 		ioucmd->flags |= IORING_URING_CMD_POLLED;  in nvme_uring_cmd_io()
622 		WRITE_ONCE(ioucmd->cookie, req);  in nvme_uring_cmd_io()
628 	req->end_io_data = ioucmd;  in nvme_uring_cmd_io()
748 static int nvme_ns_uring_cmd(struct nvme_ns *ns, struct io_uring_cmd *ioucmd,  in nvme_ns_uring_cmd()  argument
754 	BUILD_BUG_ON(sizeof(struct nvme_uring_cmd_pdu) > sizeof(ioucmd->pdu));  in nvme_ns_uring_cmd()
760 	switch (ioucmd->cmd_op) {  in nvme_ns_uring_cmd()
762 		ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, false);  in nvme_ns_uring_cmd()
765 		ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, true);  in nvme_ns_uring_cmd()
774 int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)  in nvme_ns_chr_uring_cmd()  argument
776 	struct nvme_ns *ns = container_of(file_inode(ioucmd->file)->i_cdev,  in nvme_ns_chr_uring_cmd()
779 	return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);  in nvme_ns_chr_uring_cmd()
782 int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,  in nvme_ns_chr_uring_cmd_iopoll()  argument
789 	if (!(ioucmd->flags & IORING_URING_CMD_POLLED))  in nvme_ns_chr_uring_cmd_iopoll()
792 	req = READ_ONCE(ioucmd->cookie);  in nvme_ns_chr_uring_cmd_iopoll()
873 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,  in nvme_ns_head_chr_uring_cmd()  argument
876 	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;  in nvme_ns_head_chr_uring_cmd()
883 		ret = nvme_ns_uring_cmd(ns, ioucmd, issue_flags);  in nvme_ns_head_chr_uring_cmd()
889 int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)  in nvme_dev_uring_cmd()  argument
891 	struct nvme_ctrl *ctrl = ioucmd->file->private_data;  in nvme_dev_uring_cmd()
902 	switch (ioucmd->cmd_op) {  in nvme_dev_uring_cmd()
904 		ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, false);  in nvme_dev_uring_cmd()
907 		ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, true);  in nvme_dev_uring_cmd()