75 static void copy_to_high_bio_irq(struct bio *to, struct bio *from)  in copy_to_high_bio_irq()
101 static void bounce_end_io(struct bio *bio)  in bounce_end_io()  argument
103 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io()
111 	bio_for_each_segment_all(bvec, bio, iter_all) {  in bounce_end_io()
120 	bio_orig->bi_status = bio->bi_status;  in bounce_end_io()
122 	bio_put(bio);  in bounce_end_io()
125 static void bounce_end_io_write(struct bio *bio)  in bounce_end_io_write()  argument
127 	bounce_end_io(bio);  in bounce_end_io_write()
130 static void bounce_end_io_read(struct bio *bio)  in bounce_end_io_read()  argument
132 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io_read()
134 	if (!bio->bi_status)  in bounce_end_io_read()
135 		copy_to_high_bio_irq(bio_orig, bio);  in bounce_end_io_read()
137 	bounce_end_io(bio);  in bounce_end_io_read()
140 static struct bio *bounce_clone_bio(struct bio *bio_src)  in bounce_clone_bio()
144 	struct bio *bio;  in bounce_clone_bio()  local
167 	bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src),  in bounce_clone_bio()
169 	bio->bi_bdev		= bio_src->bi_bdev;  in bounce_clone_bio()
171 		bio_set_flag(bio, BIO_REMAPPED);  in bounce_clone_bio()
172 	bio->bi_opf		= bio_src->bi_opf;  in bounce_clone_bio()
173 	bio->bi_ioprio		= bio_src->bi_ioprio;  in bounce_clone_bio()
174 	bio->bi_write_hint	= bio_src->bi_write_hint;  in bounce_clone_bio()
175 	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;  in bounce_clone_bio()
176 	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;  in bounce_clone_bio()
178 	switch (bio_op(bio)) {  in bounce_clone_bio()
184 		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];  in bounce_clone_bio()
188 			bio->bi_io_vec[bio->bi_vcnt++] = bv;  in bounce_clone_bio()
192 	if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
196 	    bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
199 	bio_clone_blkg_association(bio, bio_src);  in bounce_clone_bio()
200 	blkcg_bio_issue_init(bio);  in bounce_clone_bio()
202 	return bio;  in bounce_clone_bio()
205 	bio_put(bio);  in bounce_clone_bio()
209 void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)  in __blk_queue_bounce()
211 	struct bio *bio;  in __blk_queue_bounce()  local
229 		bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);  in __blk_queue_bounce()
230 		bio_chain(bio, *bio_orig);  in __blk_queue_bounce()
232 		*bio_orig = bio;  in __blk_queue_bounce()
234 	bio = bounce_clone_bio(*bio_orig);  in __blk_queue_bounce()
241 	for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {  in __blk_queue_bounce()
259 	bio->bi_flags |= (1 << BIO_BOUNCED);  in __blk_queue_bounce()
262 		bio->bi_end_io = bounce_end_io_read;  in __blk_queue_bounce()
264 		bio->bi_end_io = bounce_end_io_write;  in __blk_queue_bounce()
266 	bio->bi_private = *bio_orig;  in __blk_queue_bounce()
267 	*bio_orig = bio;  in __blk_queue_bounce()