34 static unsigned int dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
39 	if (iocb->ki_flags & IOCB_DSYNC)  in dio_bio_write_op()
54 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,  in __blkdev_direct_IO_simple()  argument
57 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO_simple()
60 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
82 	bio.bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO_simple()
85 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
97 		bio.bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO_simple()
100 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_simple()
102 	if (iocb->ki_flags & IOCB_HIPRI)  in __blkdev_direct_IO_simple()
103 		bio_set_polled(&bio, iocb);  in __blkdev_direct_IO_simple()
110 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO_simple()
131 		struct kiocb		*iocb;  member
162 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
167 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
172 			dio->iocb->ki_complete(iocb, ret, 0);  in blkdev_bio_end_io()
191 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO()  argument
194 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO()
200 	bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;  in __blkdev_direct_IO()
202 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
210 	bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool);  in __blkdev_direct_IO()
213 	dio->is_sync = is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
218 		dio->iocb = iocb;  in __blkdev_direct_IO()
235 		bio->bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO()
238 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
252 			bio->bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO()
255 		if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO()
265 			if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO()
266 				bio_set_polled(bio, iocb);  in __blkdev_direct_IO()
273 				WRITE_ONCE(iocb->ki_cookie, qc);  in __blkdev_direct_IO()
306 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO()
321 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
329 	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_VECS)  in blkdev_direct_IO()
330 		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);  in blkdev_direct_IO()
332 	return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages));  in blkdev_direct_IO()
488 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
490 	struct file *file = iocb->ki_filp;  in blkdev_write_iter()
506 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
509 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
512 	size -= iocb->ki_pos;  in blkdev_write_iter()
519 	ret = __generic_file_write_iter(iocb, from);  in blkdev_write_iter()
521 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
527 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
529 	struct file *file = iocb->ki_filp;  in blkdev_read_iter()
532 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
545 	ret = generic_file_read_iter(iocb, to);  in blkdev_read_iter()