78 static u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts)  in blk_to_nvme_status()  argument
93 		req->error_loc = offsetof(struct nvme_rw_command, length);  in blk_to_nvme_status()
97 		req->error_loc = offsetof(struct nvme_rw_command, slba);  in blk_to_nvme_status()
100 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
101 		switch (req->cmd->common.opcode) {  in blk_to_nvme_status()
112 		req->error_loc = offsetof(struct nvme_rw_command, nsid);  in blk_to_nvme_status()
118 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
121 	switch (req->cmd->common.opcode) {  in blk_to_nvme_status()
124 		req->error_slba = le64_to_cpu(req->cmd->rw.slba);  in blk_to_nvme_status()
127 		req->error_slba =  in blk_to_nvme_status()
128 			le64_to_cpu(req->cmd->write_zeroes.slba);  in blk_to_nvme_status()
131 		req->error_slba = 0;  in blk_to_nvme_status()
138 	struct nvmet_req *req = bio->bi_private;  in nvmet_bio_done()  local
140 	nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status));  in nvmet_bio_done()
141 	if (bio != &req->b.inline_bio)  in nvmet_bio_done()
145 static void nvmet_bdev_execute_rw(struct nvmet_req *req)  in nvmet_bdev_execute_rw()  argument
147 	int sg_cnt = req->sg_cnt;  in nvmet_bdev_execute_rw()
153 	if (!req->sg_cnt) {  in nvmet_bdev_execute_rw()
154 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_rw()
158 	if (req->cmd->rw.opcode == nvme_cmd_write) {  in nvmet_bdev_execute_rw()
161 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))  in nvmet_bdev_execute_rw()
167 	if (is_pci_p2pdma_page(sg_page(req->sg)))  in nvmet_bdev_execute_rw()
170 	sector = le64_to_cpu(req->cmd->rw.slba);  in nvmet_bdev_execute_rw()
171 	sector <<= (req->ns->blksize_shift - 9);  in nvmet_bdev_execute_rw()
173 	if (req->data_len <= NVMET_MAX_INLINE_DATA_LEN) {  in nvmet_bdev_execute_rw()
174 		bio = &req->b.inline_bio;  in nvmet_bdev_execute_rw()
175 		bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));  in nvmet_bdev_execute_rw()
179 	bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_rw()
181 	bio->bi_private = req;  in nvmet_bdev_execute_rw()
185 	for_each_sg(req->sg, sg, req->sg_cnt, i) {  in nvmet_bdev_execute_rw()
191 			bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_rw()
206 static void nvmet_bdev_execute_flush(struct nvmet_req *req)  in nvmet_bdev_execute_flush()  argument
208 	struct bio *bio = &req->b.inline_bio;  in nvmet_bdev_execute_flush()
210 	bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec));  in nvmet_bdev_execute_flush()
211 	bio_set_dev(bio, req->ns->bdev);  in nvmet_bdev_execute_flush()
212 	bio->bi_private = req;  in nvmet_bdev_execute_flush()
219 u16 nvmet_bdev_flush(struct nvmet_req *req)  in nvmet_bdev_flush()  argument
221 	if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL, NULL))  in nvmet_bdev_flush()
226 static u16 nvmet_bdev_discard_range(struct nvmet_req *req,  in nvmet_bdev_discard_range()  argument
229 	struct nvmet_ns *ns = req->ns;  in nvmet_bdev_discard_range()
237 		req->error_slba = le64_to_cpu(range->slba);  in nvmet_bdev_discard_range()
238 		return errno_to_nvme_status(req, ret);  in nvmet_bdev_discard_range()
243 static void nvmet_bdev_execute_discard(struct nvmet_req *req)  in nvmet_bdev_execute_discard()  argument
250 	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {  in nvmet_bdev_execute_discard()
251 		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,  in nvmet_bdev_execute_discard()
256 		status = nvmet_bdev_discard_range(req, &range, &bio);  in nvmet_bdev_execute_discard()
262 		bio->bi_private = req;  in nvmet_bdev_execute_discard()
271 		nvmet_req_complete(req, status);  in nvmet_bdev_execute_discard()
275 static void nvmet_bdev_execute_dsm(struct nvmet_req *req)  in nvmet_bdev_execute_dsm()  argument
277 	switch (le32_to_cpu(req->cmd->dsm.attributes)) {  in nvmet_bdev_execute_dsm()
279 		nvmet_bdev_execute_discard(req);  in nvmet_bdev_execute_dsm()
285 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_dsm()
290 static void nvmet_bdev_execute_write_zeroes(struct nvmet_req *req)  in nvmet_bdev_execute_write_zeroes()  argument
292 	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;  in nvmet_bdev_execute_write_zeroes()
299 		(req->ns->blksize_shift - 9);  in nvmet_bdev_execute_write_zeroes()
301 		(req->ns->blksize_shift - 9));  in nvmet_bdev_execute_write_zeroes()
303 	ret = __blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,  in nvmet_bdev_execute_write_zeroes()
306 		bio->bi_private = req;  in nvmet_bdev_execute_write_zeroes()
310 		nvmet_req_complete(req, errno_to_nvme_status(req, ret));  in nvmet_bdev_execute_write_zeroes()
314 u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)  in nvmet_bdev_parse_io_cmd()  argument
316 	struct nvme_command *cmd = req->cmd;  in nvmet_bdev_parse_io_cmd()
321 		req->execute = nvmet_bdev_execute_rw;  in nvmet_bdev_parse_io_cmd()
322 		req->data_len = nvmet_rw_len(req);  in nvmet_bdev_parse_io_cmd()
325 		req->execute = nvmet_bdev_execute_flush;  in nvmet_bdev_parse_io_cmd()
326 		req->data_len = 0;  in nvmet_bdev_parse_io_cmd()
329 		req->execute = nvmet_bdev_execute_dsm;  in nvmet_bdev_parse_io_cmd()
330 		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *  in nvmet_bdev_parse_io_cmd()
334 		req->execute = nvmet_bdev_execute_write_zeroes;  in nvmet_bdev_parse_io_cmd()
335 		req->data_len = 0;  in nvmet_bdev_parse_io_cmd()
339 		       req->sq->qid);  in nvmet_bdev_parse_io_cmd()
340 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in nvmet_bdev_parse_io_cmd()