46 static void blkdev_bio_end_io_simple(struct bio *bio)  in blkdev_bio_end_io_simple()  argument
48 	struct task_struct *waiter = bio->bi_private;  in blkdev_bio_end_io_simple()
50 	WRITE_ONCE(bio->bi_private, NULL);  in blkdev_bio_end_io_simple()
62 	struct bio bio;  in __blkdev_direct_IO_simple()  local
79 	bio_init(&bio, vecs, nr_pages);  in __blkdev_direct_IO_simple()
80 	bio_set_dev(&bio, bdev);  in __blkdev_direct_IO_simple()
81 	bio.bi_iter.bi_sector = pos >> 9;  in __blkdev_direct_IO_simple()
82 	bio.bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO_simple()
83 	bio.bi_private = current;  in __blkdev_direct_IO_simple()
84 	bio.bi_end_io = blkdev_bio_end_io_simple;  in __blkdev_direct_IO_simple()
85 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
87 	ret = bio_iov_iter_get_pages(&bio, iter);  in __blkdev_direct_IO_simple()
90 	ret = bio.bi_iter.bi_size;  in __blkdev_direct_IO_simple()
93 		bio.bi_opf = REQ_OP_READ;  in __blkdev_direct_IO_simple()
97 		bio.bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO_simple()
101 		bio.bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO_simple()
103 		bio_set_polled(&bio, iocb);  in __blkdev_direct_IO_simple()
105 	qc = submit_bio(&bio);  in __blkdev_direct_IO_simple()
108 		if (!READ_ONCE(bio.bi_private))  in __blkdev_direct_IO_simple()
116 	bio_release_pages(&bio, should_dirty);  in __blkdev_direct_IO_simple()
117 	if (unlikely(bio.bi_status))  in __blkdev_direct_IO_simple()
118 		ret = blk_status_to_errno(bio.bi_status);  in __blkdev_direct_IO_simple()
124 	bio_uninit(&bio);  in __blkdev_direct_IO_simple()
139 	struct bio		bio;  member
152 static void blkdev_bio_end_io(struct bio *bio)  in blkdev_bio_end_io()  argument
154 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()
157 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
158 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
165 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
169 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
174 				bio_put(&dio->bio);  in blkdev_bio_end_io()
184 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io()
186 		bio_release_pages(bio, false);  in blkdev_bio_end_io()
187 		bio_put(bio);  in blkdev_bio_end_io()
199 	struct bio *bio;  in __blkdev_direct_IO()  local
210 	bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool);  in __blkdev_direct_IO()
212 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
216 		bio_get(bio);  in __blkdev_direct_IO()
233 		bio_set_dev(bio, bdev);  in __blkdev_direct_IO()
234 		bio->bi_iter.bi_sector = pos >> 9;  in __blkdev_direct_IO()
235 		bio->bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO()
236 		bio->bi_private = dio;  in __blkdev_direct_IO()
237 		bio->bi_end_io = blkdev_bio_end_io;  in __blkdev_direct_IO()
238 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
240 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO()
242 			bio->bi_status = BLK_STS_IOERR;  in __blkdev_direct_IO()
243 			bio_endio(bio);  in __blkdev_direct_IO()
248 			bio->bi_opf = REQ_OP_READ;  in __blkdev_direct_IO()
250 				bio_set_pages_dirty(bio);  in __blkdev_direct_IO()
252 			bio->bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO()
253 			task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO()
256 			bio->bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO()
258 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
259 		pos += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
266 				bio_set_polled(bio, iocb);  in __blkdev_direct_IO()
270 			qc = submit_bio(bio);  in __blkdev_direct_IO()
284 				bio_get(bio);  in __blkdev_direct_IO()
291 		submit_bio(bio);  in __blkdev_direct_IO()
292 		bio = bio_alloc(GFP_KERNEL, nr_pages);  in __blkdev_direct_IO()
313 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
317 	bio_put(&dio->bio);  in __blkdev_direct_IO()
636 				offsetof(struct blkdev_dio, bio),  in blkdev_init()