115 static void end_clone_bio(struct bio *clone)  in end_clone_bio()  argument
118 		container_of(clone, struct dm_rq_clone_bio_info, clone);  in end_clone_bio()
121 	blk_status_t error = clone->bi_status;  in end_clone_bio()
122 	bool is_last = !clone->bi_next;  in end_clone_bio()
124 	bio_put(clone);  in end_clone_bio()
214 static void dm_end_request(struct request *clone, blk_status_t error)  in dm_end_request()  argument
216 	int rw = rq_data_dir(clone);  in dm_end_request()
217 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()
221 	blk_rq_unprep_clone(clone);  in dm_end_request()
222 	tio->ti->type->release_clone_rq(clone);  in dm_end_request()
271 	if (tio->clone) {  in dm_requeue_original_request()
272 		blk_rq_unprep_clone(tio->clone);  in dm_requeue_original_request()
273 		tio->ti->type->release_clone_rq(tio->clone);  in dm_requeue_original_request()
284 static void dm_done(struct request *clone, blk_status_t error, bool mapped)  in dm_done()  argument
287 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()
294 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
298 		if (req_op(clone) == REQ_OP_WRITE_SAME &&  in dm_done()
299 		    !clone->q->limits.max_write_same_sectors)  in dm_done()
301 		if (req_op(clone) == REQ_OP_WRITE_ZEROES &&  in dm_done()
302 		    !clone->q->limits.max_write_zeroes_sectors)  in dm_done()
309 		dm_end_request(clone, error);  in dm_done()
335 	struct request *clone = tio->clone;  in dm_softirq_done()  local
338 	if (!clone) {  in dm_softirq_done()
354 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
387 static void end_clone_request(struct request *clone, blk_status_t error)  in end_clone_request()  argument
389 	struct dm_rq_target_io *tio = clone->end_io_data;  in end_clone_request()
402 static blk_status_t dm_dispatch_clone_request(struct request *clone, struct request *rq)  in dm_dispatch_clone_request()  argument
406 	if (blk_queue_io_stat(clone->q))  in dm_dispatch_clone_request()
407 		clone->rq_flags |= RQF_IO_STAT;  in dm_dispatch_clone_request()
409 	clone->start_time_ns = ktime_get_ns();  in dm_dispatch_clone_request()
410 	r = blk_insert_cloned_request(clone->q, clone);  in dm_dispatch_clone_request()
422 		container_of(bio, struct dm_rq_clone_bio_info, clone);  in dm_rq_bio_constructor()
431 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
436 	r = blk_rq_prep_clone(clone, rq, &tio->md->bs, gfp_mask,  in setup_clone()
441 	clone->end_io = end_clone_request;  in setup_clone()
442 	clone->end_io_data = tio;  in setup_clone()
444 	tio->clone = clone;  in setup_clone()
456 	tio->clone = NULL;  in init_tio()
483 	struct request *clone = NULL;  in map_request()  local
486 	r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
493 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
495 			ti->type->release_clone_rq(clone);  in map_request()
500 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),  in map_request()
502 		ret = dm_dispatch_clone_request(clone, rq);  in map_request()
504 			blk_rq_unprep_clone(clone);  in map_request()
505 			tio->ti->type->release_clone_rq(clone);  in map_request()
506 			tio->clone = NULL;  in map_request()