61 	struct bio bio;  in __blkdev_direct_IO_simple()  local
77 		bio_init(&bio, bdev, vecs, nr_pages, REQ_OP_READ);  in __blkdev_direct_IO_simple()
81 		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));  in __blkdev_direct_IO_simple()
83 	bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO_simple()
84 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
86 	ret = bio_iov_iter_get_pages(&bio, iter);  in __blkdev_direct_IO_simple()
89 	ret = bio.bi_iter.bi_size;  in __blkdev_direct_IO_simple()
95 		bio.bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO_simple()
97 	submit_bio_wait(&bio);  in __blkdev_direct_IO_simple()
99 	bio_release_pages(&bio, should_dirty);  in __blkdev_direct_IO_simple()
100 	if (unlikely(bio.bi_status))  in __blkdev_direct_IO_simple()
101 		ret = blk_status_to_errno(bio.bi_status);  in __blkdev_direct_IO_simple()
107 	bio_uninit(&bio);  in __blkdev_direct_IO_simple()
125 	struct bio		bio ____cacheline_aligned_in_smp;
130 static void blkdev_bio_end_io(struct bio *bio)  in blkdev_bio_end_io()  argument
132 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()
135 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
136 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
145 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
149 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
153 			bio_put(&dio->bio);  in blkdev_bio_end_io()
163 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io()
165 		bio_release_pages(bio, false);  in blkdev_bio_end_io()
166 		bio_put(bio);  in blkdev_bio_end_io()
176 	struct bio *bio;  in __blkdev_direct_IO()  local
187 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,  in __blkdev_direct_IO()
189 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
195 	bio_get(bio);  in __blkdev_direct_IO()
213 		bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO()
214 		bio->bi_private = dio;  in __blkdev_direct_IO()
215 		bio->bi_end_io = blkdev_bio_end_io;  in __blkdev_direct_IO()
216 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
218 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO()
220 			bio->bi_status = BLK_STS_IOERR;  in __blkdev_direct_IO()
221 			bio_endio(bio);  in __blkdev_direct_IO()
227 				bio_set_pages_dirty(bio);  in __blkdev_direct_IO()
229 			task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO()
232 			bio->bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO()
234 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
235 		pos += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
239 			submit_bio(bio);  in __blkdev_direct_IO()
243 		submit_bio(bio);  in __blkdev_direct_IO()
244 		bio = bio_alloc(bdev, nr_pages, opf, GFP_KERNEL);  in __blkdev_direct_IO()
261 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
265 	bio_put(&dio->bio);  in __blkdev_direct_IO()
269 static void blkdev_bio_end_io_async(struct bio *bio)  in blkdev_bio_end_io_async()  argument
271 	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);  in blkdev_bio_end_io_async()
277 	if (likely(!bio->bi_status)) {  in blkdev_bio_end_io_async()
281 		ret = blk_status_to_errno(bio->bi_status);  in blkdev_bio_end_io_async()
287 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io_async()
289 		bio_release_pages(bio, false);  in blkdev_bio_end_io_async()
290 		bio_put(bio);  in blkdev_bio_end_io_async()
302 	struct bio *bio;  in __blkdev_direct_IO_async()  local
311 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,  in __blkdev_direct_IO_async()
313 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO_async()
316 	bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO_async()
317 	bio->bi_end_io = blkdev_bio_end_io_async;  in __blkdev_direct_IO_async()
318 	bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_async()
327 		bio_iov_bvec_set(bio, iter);  in __blkdev_direct_IO_async()
329 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO_async()
331 			bio_put(bio);  in __blkdev_direct_IO_async()
335 	dio->size = bio->bi_iter.bi_size;  in __blkdev_direct_IO_async()
340 			bio_set_pages_dirty(bio);  in __blkdev_direct_IO_async()
343 		task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO_async()
347 		bio->bi_opf |= REQ_POLLED | REQ_NOWAIT;  in __blkdev_direct_IO_async()
348 		submit_bio(bio);  in __blkdev_direct_IO_async()
349 		WRITE_ONCE(iocb->private, bio);  in __blkdev_direct_IO_async()
352 			bio->bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO_async()
353 		submit_bio(bio);  in __blkdev_direct_IO_async()
694 				offsetof(struct blkdev_dio, bio),  in blkdev_init()