199 static unsigned int dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
204 	if (iocb->ki_flags & IOCB_DSYNC)  in dio_bio_write_op()
220 __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO_simple()  argument
223 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO_simple()
226 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
248 	bio.bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO_simple()
251 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
263 		bio.bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO_simple()
266 	if (iocb->ki_flags & IOCB_HIPRI)  in __blkdev_direct_IO_simple()
267 		bio_set_polled(&bio, iocb);  in __blkdev_direct_IO_simple()
274 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO_simple()
295 		struct kiocb		*iocb;  member
326 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
331 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
336 			dio->iocb->ki_complete(iocb, ret, 0);  in blkdev_bio_end_io()
356 __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)  in __blkdev_direct_IO()  argument
358 	struct file *file = iocb->ki_filp;  in __blkdev_direct_IO()
364 	bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;  in __blkdev_direct_IO()
366 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
377 	dio->is_sync = is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
382 		dio->iocb = iocb;  in __blkdev_direct_IO()
399 		bio->bi_write_hint = iocb->ki_hint;  in __blkdev_direct_IO()
402 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
416 			bio->bi_opf = dio_bio_write_op(iocb);  in __blkdev_direct_IO()
427 			if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO()
428 				bio_set_polled(bio, iocb);  in __blkdev_direct_IO()
435 				WRITE_ONCE(iocb->ki_cookie, qc);  in __blkdev_direct_IO()
468 		if (!(iocb->ki_flags & IOCB_HIPRI) ||  in __blkdev_direct_IO()
484 blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
491 	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)  in blkdev_direct_IO()
492 		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);  in blkdev_direct_IO()
494 	return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));  in blkdev_direct_IO()
1879 ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
1881 	struct file *file = iocb->ki_filp;  in blkdev_write_iter()
1896 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
1899 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
1902 	iov_iter_truncate(from, size - iocb->ki_pos);  in blkdev_write_iter()
1905 	ret = __generic_file_write_iter(iocb, from);  in blkdev_write_iter()
1907 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
1913 ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
1915 	struct file *file = iocb->ki_filp;  in blkdev_read_iter()
1918 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
1925 	return generic_file_read_iter(iocb, to);  in blkdev_read_iter()