52 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
55 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
57 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
73 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
74 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
77 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL,  in ext4_dio_read_iter()
78 			   is_sync_kiocb(iocb));  in ext4_dio_read_iter()
81 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
86 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
88 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
91 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
104 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
106 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
109 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
114 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
116 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
126 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
128 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
129 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
131 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
212 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
215 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
221 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
232 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
234 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
240 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
244 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
248 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
254 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
258 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
260 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
265 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
270 	ret = generic_perform_write(iocb->ki_filp, from, iocb->ki_pos);  in ext4_buffered_write_iter()
277 		iocb->ki_pos += ret;  in ext4_buffered_write_iter()
278 		ret = generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
372 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
375 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_end_io()
376 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
406 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
409 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
416 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
420 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
432 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
455 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
459 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
460 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
482 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
503 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
506 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend);  in ext4_dio_write_iter()
511 	if ((iocb->ki_flags & IOCB_NOWAIT) && (unaligned_io || extend)) {  in ext4_dio_write_iter()
516 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
552 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
553 			   is_sync_kiocb(iocb) || unaligned_io || extend);  in ext4_dio_write_iter()
570 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
571 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
584 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
587 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
597 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
604 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
606 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
613 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
617 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
637 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
644 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
650 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
652 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
659 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
661 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
662 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
664 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()