122 u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts)  in blk_to_nvme_status()  argument
137 		req->error_loc = offsetof(struct nvme_rw_command, length);  in blk_to_nvme_status()
141 		req->error_loc = offsetof(struct nvme_rw_command, slba);  in blk_to_nvme_status()
144 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
145 		switch (req->cmd->common.opcode) {  in blk_to_nvme_status()
156 		req->error_loc = offsetof(struct nvme_rw_command, nsid);  in blk_to_nvme_status()
161 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
164 	switch (req->cmd->common.opcode) {  in blk_to_nvme_status()
167 		req->error_slba = le64_to_cpu(req->cmd->rw.slba);  in blk_to_nvme_status()
170 		req->error_slba =  in blk_to_nvme_status()
171 			le64_to_cpu(req->cmd->write_zeroes.slba);  in blk_to_nvme_status()
174 		req->error_slba = 0;  in blk_to_nvme_status()
181 	struct nvmet_req *req = bio->bi_private;  in nvmet_bio_done()  local
183 	nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status));  in nvmet_bio_done()
184 	nvmet_req_bio_put(req, bio);  in nvmet_bio_done()
188 static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,  in nvmet_bdev_alloc_bip()  argument
196 	bi = bdev_get_integrity(req->ns->bdev);  in nvmet_bdev_alloc_bip()
203 					bio_max_segs(req->metadata_sg_cnt));  in nvmet_bdev_alloc_bip()
234 static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,  in nvmet_bdev_alloc_bip()  argument
241 static void nvmet_bdev_execute_rw(struct nvmet_req *req)  in nvmet_bdev_execute_rw()  argument
243 	unsigned int sg_cnt = req->sg_cnt;  in nvmet_bdev_execute_rw()
252 	unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;  in nvmet_bdev_execute_rw()
254 	if (!nvmet_check_transfer_len(req, total_len))  in nvmet_bdev_execute_rw()
257 	if (!req->sg_cnt) {  in nvmet_bdev_execute_rw()
258 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_rw()
262 	if (req->cmd->rw.opcode == nvme_cmd_write) {  in nvmet_bdev_execute_rw()
264 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))  in nvmet_bdev_execute_rw()
272 	if (is_pci_p2pdma_page(sg_page(req->sg)))  in nvmet_bdev_execute_rw()
275 	sector = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba);  in nvmet_bdev_execute_rw()
277 	if (nvmet_use_inline_bvec(req)) {  in nvmet_bdev_execute_rw()
278 		bio = &req->b.inline_bio;  in nvmet_bdev_execute_rw()
279 		bio_init(bio, req->ns->bdev, req->inline_bvec,  in nvmet_bdev_execute_rw()
280 			 ARRAY_SIZE(req->inline_bvec), opf);  in nvmet_bdev_execute_rw()
282 		bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt), opf,  in nvmet_bdev_execute_rw()
286 	bio->bi_private = req;  in nvmet_bdev_execute_rw()
290 	if (req->metadata_len)  in nvmet_bdev_execute_rw()
291 		sg_miter_start(&prot_miter, req->metadata_sg,  in nvmet_bdev_execute_rw()
292 			       req->metadata_sg_cnt, iter_flags);  in nvmet_bdev_execute_rw()
294 	for_each_sg(req->sg, sg, req->sg_cnt, i) {  in nvmet_bdev_execute_rw()
299 			if (req->metadata_len) {  in nvmet_bdev_execute_rw()
300 				rc = nvmet_bdev_alloc_bip(req, bio,  in nvmet_bdev_execute_rw()
308 			bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt),  in nvmet_bdev_execute_rw()
320 	if (req->metadata_len) {  in nvmet_bdev_execute_rw()
321 		rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter);  in nvmet_bdev_execute_rw()
332 static void nvmet_bdev_execute_flush(struct nvmet_req *req)  in nvmet_bdev_execute_flush()  argument
334 	struct bio *bio = &req->b.inline_bio;  in nvmet_bdev_execute_flush()
336 	if (!bdev_write_cache(req->ns->bdev)) {  in nvmet_bdev_execute_flush()
337 		nvmet_req_complete(req, NVME_SC_SUCCESS);  in nvmet_bdev_execute_flush()
341 	if (!nvmet_check_transfer_len(req, 0))  in nvmet_bdev_execute_flush()
344 	bio_init(bio, req->ns->bdev, req->inline_bvec,  in nvmet_bdev_execute_flush()
345 		 ARRAY_SIZE(req->inline_bvec), REQ_OP_WRITE | REQ_PREFLUSH);  in nvmet_bdev_execute_flush()
346 	bio->bi_private = req;  in nvmet_bdev_execute_flush()
352 u16 nvmet_bdev_flush(struct nvmet_req *req)  in nvmet_bdev_flush()  argument
354 	if (!bdev_write_cache(req->ns->bdev))  in nvmet_bdev_flush()
357 	if (blkdev_issue_flush(req->ns->bdev))  in nvmet_bdev_flush()
362 static u16 nvmet_bdev_discard_range(struct nvmet_req *req,  in nvmet_bdev_discard_range()  argument
365 	struct nvmet_ns *ns = req->ns;  in nvmet_bdev_discard_range()
373 		req->error_slba = le64_to_cpu(range->slba);  in nvmet_bdev_discard_range()
374 		return errno_to_nvme_status(req, ret);  in nvmet_bdev_discard_range()
379 static void nvmet_bdev_execute_discard(struct nvmet_req *req)  in nvmet_bdev_execute_discard()  argument
386 	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {  in nvmet_bdev_execute_discard()
387 		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,  in nvmet_bdev_execute_discard()
392 		status = nvmet_bdev_discard_range(req, &range, &bio);  in nvmet_bdev_execute_discard()
398 		bio->bi_private = req;  in nvmet_bdev_execute_discard()
405 		nvmet_req_complete(req, status);  in nvmet_bdev_execute_discard()
409 static void nvmet_bdev_execute_dsm(struct nvmet_req *req)  in nvmet_bdev_execute_dsm()  argument
411 	if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req)))  in nvmet_bdev_execute_dsm()
414 	switch (le32_to_cpu(req->cmd->dsm.attributes)) {  in nvmet_bdev_execute_dsm()
416 		nvmet_bdev_execute_discard(req);  in nvmet_bdev_execute_dsm()
422 		nvmet_req_complete(req, 0);  in nvmet_bdev_execute_dsm()
427 static void nvmet_bdev_execute_write_zeroes(struct nvmet_req *req)  in nvmet_bdev_execute_write_zeroes()  argument
429 	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;  in nvmet_bdev_execute_write_zeroes()
435 	if (!nvmet_check_transfer_len(req, 0))  in nvmet_bdev_execute_write_zeroes()
438 	sector = nvmet_lba_to_sect(req->ns, write_zeroes->slba);  in nvmet_bdev_execute_write_zeroes()
440 		(req->ns->blksize_shift - 9));  in nvmet_bdev_execute_write_zeroes()
442 	ret = __blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,  in nvmet_bdev_execute_write_zeroes()
445 		bio->bi_private = req;  in nvmet_bdev_execute_write_zeroes()
449 		nvmet_req_complete(req, errno_to_nvme_status(req, ret));  in nvmet_bdev_execute_write_zeroes()
453 u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)  in nvmet_bdev_parse_io_cmd()  argument
455 	switch (req->cmd->common.opcode) {  in nvmet_bdev_parse_io_cmd()
458 		req->execute = nvmet_bdev_execute_rw;  in nvmet_bdev_parse_io_cmd()
459 		if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns))  in nvmet_bdev_parse_io_cmd()
460 			req->metadata_len = nvmet_rw_metadata_len(req);  in nvmet_bdev_parse_io_cmd()
463 		req->execute = nvmet_bdev_execute_flush;  in nvmet_bdev_parse_io_cmd()
466 		req->execute = nvmet_bdev_execute_dsm;  in nvmet_bdev_parse_io_cmd()
469 		req->execute = nvmet_bdev_execute_write_zeroes;  in nvmet_bdev_parse_io_cmd()
472 		return nvmet_report_invalid_opcode(req);  in nvmet_bdev_parse_io_cmd()