35 static blk_opf_t dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
40 	if (iocb_is_dsync(iocb))  in dio_bio_write_op()
54 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,  in __blkdev_direct_IO_simple()  argument
57 	struct block_device *bdev = iocb->ki_filp->private_data;  in __blkdev_direct_IO_simple()
59 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
81 		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));  in __blkdev_direct_IO_simple()
84 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
94 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_simple()
119 		struct kiocb		*iocb;  member
140 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
143 			WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io()
147 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
152 			dio->iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io()
170 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO()  argument
173 	struct block_device *bdev = iocb->ki_filp->private_data;  in __blkdev_direct_IO()
178 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO()
179 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
185 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO()
197 	is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
203 		dio->iocb = iocb;  in __blkdev_direct_IO()
216 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
231 		if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO()
272 	struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io_async()  local
275 	WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io_async()
279 		iocb->ki_pos += ret;  in blkdev_bio_end_io_async()
284 	iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io_async()
294 static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,  in __blkdev_direct_IO_async()  argument
298 	struct block_device *bdev = iocb->ki_filp->private_data;  in __blkdev_direct_IO_async()
300 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO_async()
303 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_async()
309 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO_async()
315 	dio->iocb = iocb;  in __blkdev_direct_IO_async()
318 	bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_async()
346 	if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO_async()
349 		WRITE_ONCE(iocb->private, bio);  in __blkdev_direct_IO_async()
351 		if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_async()
358 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
367 		if (is_sync_kiocb(iocb))  in blkdev_direct_IO()
368 			return __blkdev_direct_IO_simple(iocb, iter, nr_pages);  in blkdev_direct_IO()
369 		return __blkdev_direct_IO_async(iocb, iter, nr_pages);  in blkdev_direct_IO()
371 	return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages));  in blkdev_direct_IO()
510 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
512 	struct block_device *bdev = iocb->ki_filp->private_data;  in blkdev_write_iter()
528 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
531 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
534 	size -= iocb->ki_pos;  in blkdev_write_iter()
541 	ret = __generic_file_write_iter(iocb, from);  in blkdev_write_iter()
543 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
549 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
551 	struct block_device *bdev = iocb->ki_filp->private_data;  in blkdev_read_iter()
553 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
570 	if (iocb->ki_flags & IOCB_DIRECT) {  in blkdev_read_iter()
571 		struct address_space *mapping = iocb->ki_filp->f_mapping;  in blkdev_read_iter()
573 		if (iocb->ki_flags & IOCB_NOWAIT) {  in blkdev_read_iter()
586 		file_accessed(iocb->ki_filp);  in blkdev_read_iter()
588 		ret = blkdev_direct_IO(iocb, to);  in blkdev_read_iter()
590 			iocb->ki_pos += ret;  in blkdev_read_iter()
598 	ret = filemap_read(iocb, to, ret);  in blkdev_read_iter()