55 static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)  in ext4_should_use_dio()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_should_use_dio()
66 	return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);  in ext4_should_use_dio()
69 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
72 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
74 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
81 	if (!ext4_should_use_dio(iocb, to)) {  in ext4_dio_read_iter()
90 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
91 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
94 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);  in ext4_dio_read_iter()
97 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
102 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
104 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
107 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
120 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
122 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
125 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
130 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
132 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
142 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
144 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
145 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
147 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
228 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
231 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
237 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
248 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
250 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
256 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
260 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
264 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
270 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
274 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
276 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
280 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
285 	ret = generic_perform_write(iocb, from);  in ext4_buffered_write_iter()
291 		iocb->ki_pos += ret;  in ext4_buffered_write_iter()
292 		ret = generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
386 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
389 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
390 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
437 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
440 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
447 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
451 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
463 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
486 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
490 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
491 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
513 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
529 	if (!ext4_should_use_dio(iocb, from)) {  in ext4_dio_write_iter()
534 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
537 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend);  in ext4_dio_write_iter()
542 	if ((iocb->ki_flags & IOCB_NOWAIT) && (unaligned_io || extend)) {  in ext4_dio_write_iter()
553 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
587 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
606 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
607 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
620 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
623 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
633 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
640 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
642 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
649 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
653 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
673 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
680 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
686 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
688 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
695 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
697 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
698 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
700 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()