55 static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)  in ext4_should_use_dio()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_should_use_dio()
66 	return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);  in ext4_should_use_dio()
69 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
72 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
74 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
81 	if (!ext4_should_use_dio(iocb, to)) {  in ext4_dio_read_iter()
90 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
91 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
94 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);  in ext4_dio_read_iter()
97 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
102 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
104 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
107 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
120 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
122 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
125 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
130 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
132 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
142 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
144 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
145 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
147 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
243 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
246 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
252 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
263 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
265 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
271 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
275 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
279 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
285 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
289 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
291 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
295 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
299 	ret = generic_perform_write(iocb, from);  in ext4_buffered_write_iter()
305 	return generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
396 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
399 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
400 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
451 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
455 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
463 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
467 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
488 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
506 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
528 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
532 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
533 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
548 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
564 	if (!ext4_should_use_dio(iocb, from)) {  in ext4_dio_write_iter()
569 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
572 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend,  in ext4_dio_write_iter()
584 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
605 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
623 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
624 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
637 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
640 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
650 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
657 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
659 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
666 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
670 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
690 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
697 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
703 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
705 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
712 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
714 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
715 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
717 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()