61 static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap,  in iomap_dio_submit_bio()  argument
64 	atomic_inc(&dio->ref);  in iomap_dio_submit_bio()
66 	if (dio->iocb->ki_flags & IOCB_HIPRI)  in iomap_dio_submit_bio()
67 		bio_set_polled(bio, dio->iocb);  in iomap_dio_submit_bio()
69 	dio->submit.last_queue = bdev_get_queue(iomap->bdev);  in iomap_dio_submit_bio()
70 	dio->submit.cookie = submit_bio(bio);  in iomap_dio_submit_bio()
73 static ssize_t iomap_dio_complete(struct iomap_dio *dio)  in iomap_dio_complete()  argument
75 	const struct iomap_dio_ops *dops = dio->dops;  in iomap_dio_complete()
76 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete()
79 	ssize_t ret = dio->error;  in iomap_dio_complete()
82 		ret = dops->end_io(iocb, dio->size, ret, dio->flags);  in iomap_dio_complete()
85 		ret = dio->size;  in iomap_dio_complete()
87 		if (offset + ret > dio->i_size &&  in iomap_dio_complete()
88 		    !(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_complete()
89 			ret = dio->i_size - offset;  in iomap_dio_complete()
105 	if (!dio->error &&  in iomap_dio_complete()
106 	    (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {  in iomap_dio_complete()
110 				(offset + dio->size - 1) >> PAGE_SHIFT);  in iomap_dio_complete()
119 	if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))  in iomap_dio_complete()
123 	kfree(dio);  in iomap_dio_complete()
130 	struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);  in iomap_dio_complete_work()  local
131 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete_work()
133 	iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);  in iomap_dio_complete_work()
141 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)  in iomap_dio_set_error()  argument
143 	cmpxchg(&dio->error, 0, ret);  in iomap_dio_set_error()
148 	struct iomap_dio *dio = bio->bi_private;  in iomap_dio_bio_end_io()  local
149 	bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);  in iomap_dio_bio_end_io()
152 		iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));  in iomap_dio_bio_end_io()
154 	if (atomic_dec_and_test(&dio->ref)) {  in iomap_dio_bio_end_io()
155 		if (dio->wait_for_completion) {  in iomap_dio_bio_end_io()
156 			struct task_struct *waiter = dio->submit.waiter;  in iomap_dio_bio_end_io()
157 			WRITE_ONCE(dio->submit.waiter, NULL);  in iomap_dio_bio_end_io()
159 		} else if (dio->flags & IOMAP_DIO_WRITE) {  in iomap_dio_bio_end_io()
160 			struct inode *inode = file_inode(dio->iocb->ki_filp);  in iomap_dio_bio_end_io()
162 			INIT_WORK(&dio->aio.work, iomap_dio_complete_work);  in iomap_dio_bio_end_io()
163 			queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);  in iomap_dio_bio_end_io()
165 			iomap_dio_complete_work(&dio->aio.work);  in iomap_dio_bio_end_io()
178 iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,  in iomap_dio_zero()  argument
188 	bio->bi_private = dio;  in iomap_dio_zero()
194 	iomap_dio_submit_bio(dio, iomap, bio);  in iomap_dio_zero()
199 		struct iomap_dio *dio, struct iomap *iomap)  in iomap_dio_bio_actor()  argument
203 	unsigned int align = iov_iter_alignment(dio->submit.iter);  in iomap_dio_bio_actor()
215 		dio->flags |= IOMAP_DIO_UNWRITTEN;  in iomap_dio_bio_actor()
220 		dio->flags |= IOMAP_DIO_COW;  in iomap_dio_bio_actor()
233 		    (dio->flags & IOMAP_DIO_WRITE_FUA) &&  in iomap_dio_bio_actor()
242 	iter = *dio->submit.iter;  in iomap_dio_bio_actor()
253 			iomap_dio_zero(dio, iomap, pos - pad, pad);  in iomap_dio_bio_actor()
258 		if (dio->error) {  in iomap_dio_bio_actor()
259 			iov_iter_revert(dio->submit.iter, copied);  in iomap_dio_bio_actor()
266 		bio->bi_write_hint = dio->iocb->ki_hint;  in iomap_dio_bio_actor()
267 		bio->bi_ioprio = dio->iocb->ki_ioprio;  in iomap_dio_bio_actor()
268 		bio->bi_private = dio;  in iomap_dio_bio_actor()
284 		if (dio->flags & IOMAP_DIO_WRITE) {  in iomap_dio_bio_actor()
289 				dio->flags &= ~IOMAP_DIO_WRITE_FUA;  in iomap_dio_bio_actor()
293 			if (dio->flags & IOMAP_DIO_DIRTY)  in iomap_dio_bio_actor()
297 		iov_iter_advance(dio->submit.iter, n);  in iomap_dio_bio_actor()
299 		dio->size += n;  in iomap_dio_bio_actor()
304 		iomap_dio_submit_bio(dio, iomap, bio);  in iomap_dio_bio_actor()
315 	    ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {  in iomap_dio_bio_actor()
319 			iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);  in iomap_dio_bio_actor()
325 iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)  in iomap_dio_hole_actor()  argument
327 	length = iov_iter_zero(length, dio->submit.iter);  in iomap_dio_hole_actor()
328 	dio->size += length;  in iomap_dio_hole_actor()
334 		struct iomap_dio *dio, struct iomap *iomap)  in iomap_dio_inline_actor()  argument
336 	struct iov_iter *iter = dio->submit.iter;  in iomap_dio_inline_actor()
341 	if (dio->flags & IOMAP_DIO_WRITE) {  in iomap_dio_inline_actor()
355 	dio->size += copied;  in iomap_dio_inline_actor()
363 	struct iomap_dio *dio = data;  in iomap_dio_actor()  local
367 		if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_actor()
369 		return iomap_dio_hole_actor(length, dio);  in iomap_dio_actor()
371 		if (!(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_actor()
372 			return iomap_dio_hole_actor(length, dio);  in iomap_dio_actor()
373 		return iomap_dio_bio_actor(inode, pos, length, dio, iomap);  in iomap_dio_actor()
375 		return iomap_dio_bio_actor(inode, pos, length, dio, iomap);  in iomap_dio_actor()
377 		return iomap_dio_inline_actor(inode, pos, length, dio, iomap);  in iomap_dio_actor()
405 	struct iomap_dio *dio;  in iomap_dio_rw()  local
412 	dio = kmalloc(sizeof(*dio), GFP_KERNEL);  in iomap_dio_rw()
413 	if (!dio)  in iomap_dio_rw()
416 	dio->iocb = iocb;  in iomap_dio_rw()
417 	atomic_set(&dio->ref, 1);  in iomap_dio_rw()
418 	dio->size = 0;  in iomap_dio_rw()
419 	dio->i_size = i_size_read(inode);  in iomap_dio_rw()
420 	dio->dops = dops;  in iomap_dio_rw()
421 	dio->error = 0;  in iomap_dio_rw()
422 	dio->flags = 0;  in iomap_dio_rw()
424 	dio->submit.iter = iter;  in iomap_dio_rw()
425 	dio->submit.waiter = current;  in iomap_dio_rw()
426 	dio->submit.cookie = BLK_QC_T_NONE;  in iomap_dio_rw()
427 	dio->submit.last_queue = NULL;  in iomap_dio_rw()
430 		if (pos >= dio->i_size)  in iomap_dio_rw()
434 			dio->flags |= IOMAP_DIO_DIRTY;  in iomap_dio_rw()
437 		dio->flags |= IOMAP_DIO_WRITE;  in iomap_dio_rw()
441 			dio->flags |= IOMAP_DIO_NEED_SYNC;  in iomap_dio_rw()
450 			dio->flags |= IOMAP_DIO_WRITE_FUA;  in iomap_dio_rw()
488 		ret = iomap_apply(inode, pos, count, flags, ops, dio,  in iomap_dio_rw()
500 		if (iov_iter_rw(iter) == READ && pos >= dio->i_size)  in iomap_dio_rw()
506 		iomap_dio_set_error(dio, ret);  in iomap_dio_rw()
512 	if (dio->flags & IOMAP_DIO_WRITE_FUA)  in iomap_dio_rw()
513 		dio->flags &= ~IOMAP_DIO_NEED_SYNC;  in iomap_dio_rw()
515 	WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);  in iomap_dio_rw()
516 	WRITE_ONCE(iocb->private, dio->submit.last_queue);  in iomap_dio_rw()
533 	dio->wait_for_completion = wait_for_completion;  in iomap_dio_rw()
534 	if (!atomic_dec_and_test(&dio->ref)) {  in iomap_dio_rw()
540 			if (!READ_ONCE(dio->submit.waiter))  in iomap_dio_rw()
544 			    !dio->submit.last_queue ||  in iomap_dio_rw()
545 			    !blk_poll(dio->submit.last_queue,  in iomap_dio_rw()
546 					 dio->submit.cookie, true))  in iomap_dio_rw()
552 	return iomap_dio_complete(dio);  in iomap_dio_rw()
555 	kfree(dio);  in iomap_dio_rw()