76 static void copy_to_high_bio_irq(struct bio *to, struct bio *from)  in copy_to_high_bio_irq()
102 static void bounce_end_io(struct bio *bio)  in bounce_end_io()  argument
104 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io()
112 	bio_for_each_segment_all(bvec, bio, iter_all) {  in bounce_end_io()
121 	bio_orig->bi_status = bio->bi_status;  in bounce_end_io()
123 	bio_put(bio);  in bounce_end_io()
126 static void bounce_end_io_write(struct bio *bio)  in bounce_end_io_write()  argument
128 	bounce_end_io(bio);  in bounce_end_io_write()
131 static void bounce_end_io_read(struct bio *bio)  in bounce_end_io_read()  argument
133 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io_read()
135 	if (!bio->bi_status)  in bounce_end_io_read()
136 		copy_to_high_bio_irq(bio_orig, bio);  in bounce_end_io_read()
138 	bounce_end_io(bio);  in bounce_end_io_read()
141 static struct bio *bounce_clone_bio(struct bio *bio_src)  in bounce_clone_bio()
145 	struct bio *bio;  in bounce_clone_bio()  local
167 	bio = bio_alloc_bioset(bio_src->bi_bdev, bio_segments(bio_src),  in bounce_clone_bio()
170 		bio_set_flag(bio, BIO_REMAPPED);  in bounce_clone_bio()
171 	bio->bi_ioprio		= bio_src->bi_ioprio;  in bounce_clone_bio()
172 	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;  in bounce_clone_bio()
173 	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;  in bounce_clone_bio()
175 	switch (bio_op(bio)) {  in bounce_clone_bio()
182 			bio->bi_io_vec[bio->bi_vcnt++] = bv;  in bounce_clone_bio()
186 	if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
190 	    bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
193 	bio_clone_blkg_association(bio, bio_src);  in bounce_clone_bio()
195 	return bio;  in bounce_clone_bio()
198 	bio_put(bio);  in bounce_clone_bio()
202 struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q)  in __blk_queue_bounce()
204 	struct bio *bio;  in __blk_queue_bounce()  local
229 		bio = bio_split(bio_orig, sectors, GFP_NOIO, &bounce_bio_split);  in __blk_queue_bounce()
230 		bio_chain(bio, bio_orig);  in __blk_queue_bounce()
232 		bio_orig = bio;  in __blk_queue_bounce()
234 	bio = bounce_clone_bio(bio_orig);  in __blk_queue_bounce()
241 	for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {  in __blk_queue_bounce()
259 	bio->bi_flags |= (1 << BIO_BOUNCED);  in __blk_queue_bounce()
262 		bio->bi_end_io = bounce_end_io_read;  in __blk_queue_bounce()
264 		bio->bi_end_io = bounce_end_io_write;  in __blk_queue_bounce()
266 	bio->bi_private = bio_orig;  in __blk_queue_bounce()
267 	return bio;  in __blk_queue_bounce()