118 struct dio {  struct
167 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)  in dio_refill_pages()  argument
169 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in dio_refill_pages()
172 	ret = iov_iter_get_pages2(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,  in dio_refill_pages()
182 		if (dio->page_errors == 0)  in dio_refill_pages()
183 			dio->page_errors = ret;  in dio_refill_pages()
185 		dio->pages[0] = page;  in dio_refill_pages()
209 static inline struct page *dio_get_page(struct dio *dio,  in dio_get_page()  argument
215 		ret = dio_refill_pages(dio, sdio);  in dio_get_page()
220 	return dio->pages[sdio->head];  in dio_get_page()
234 static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)  in dio_complete()  argument
236 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in dio_complete()
237 	loff_t offset = dio->iocb->ki_pos;  in dio_complete()
250 	if (dio->result) {  in dio_complete()
251 		transferred = dio->result;  in dio_complete()
255 		    ((offset + transferred) > dio->i_size))  in dio_complete()
256 			transferred = dio->i_size - offset;  in dio_complete()
263 		ret = dio->page_errors;  in dio_complete()
265 		ret = dio->io_error;  in dio_complete()
269 	if (dio->end_io) {  in dio_complete()
271 		err = dio->end_io(dio->iocb, offset, ret, dio->private);  in dio_complete()
290 	    dio->inode->i_mapping->nrpages) {  in dio_complete()
291 		err = invalidate_inode_pages2_range(dio->inode->i_mapping,  in dio_complete()
295 			dio_warn_stale_pagecache(dio->iocb->ki_filp);  in dio_complete()
298 	inode_dio_end(dio->inode);  in dio_complete()
306 		dio->iocb->ki_pos += transferred;  in dio_complete()
309 			ret = generic_write_sync(dio->iocb, ret);  in dio_complete()
310 		dio->iocb->ki_complete(dio->iocb, ret);  in dio_complete()
313 	kmem_cache_free(dio_cache, dio);  in dio_complete()
319 	struct dio *dio = container_of(work, struct dio, complete_work);  in dio_aio_complete_work()  local
321 	dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE);  in dio_aio_complete_work()
324 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);
331 	struct dio *dio = bio->bi_private;  in dio_bio_end_aio()  local
332 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in dio_bio_end_aio()
338 	dio_bio_complete(dio, bio);  in dio_bio_end_aio()
340 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_end_aio()
341 	remaining = --dio->refcount;  in dio_bio_end_aio()
342 	if (remaining == 1 && dio->waiter)  in dio_bio_end_aio()
343 		wake_up_process(dio->waiter);  in dio_bio_end_aio()
344 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_end_aio()
355 		if (dio->result)  in dio_bio_end_aio()
356 			defer_completion = dio->defer_completion ||  in dio_bio_end_aio()
358 					    dio->inode->i_mapping->nrpages);  in dio_bio_end_aio()
360 			INIT_WORK(&dio->complete_work, dio_aio_complete_work);  in dio_bio_end_aio()
361 			queue_work(dio->inode->i_sb->s_dio_done_wq,  in dio_bio_end_aio()
362 				   &dio->complete_work);  in dio_bio_end_aio()
364 			dio_complete(dio, 0, DIO_COMPLETE_ASYNC);  in dio_bio_end_aio()
378 	struct dio *dio = bio->bi_private;  in dio_bio_end_io()  local
381 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_end_io()
382 	bio->bi_private = dio->bio_list;  in dio_bio_end_io()
383 	dio->bio_list = bio;  in dio_bio_end_io()
384 	if (--dio->refcount == 1 && dio->waiter)  in dio_bio_end_io()
385 		wake_up_process(dio->waiter);  in dio_bio_end_io()
386 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_end_io()
390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,  in dio_bio_alloc()  argument
400 	bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL);  in dio_bio_alloc()
402 	if (dio->is_async)  in dio_bio_alloc()
417 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)  in dio_bio_submit()  argument
419 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in dio_bio_submit()
423 	bio->bi_private = dio;  in dio_bio_submit()
425 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_submit()
426 	dio->refcount++;  in dio_bio_submit()
427 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_submit()
429 	if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty)  in dio_bio_submit()
432 	dio->bio_disk = bio->bi_bdev->bd_disk;  in dio_bio_submit()
435 		sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);  in dio_bio_submit()
447 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)  in dio_cleanup()  argument
450 		put_page(dio->pages[sdio->head++]);  in dio_cleanup()
459 static struct bio *dio_await_one(struct dio *dio)  in dio_await_one()  argument
464 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_await_one()
472 	while (dio->refcount > 1 && dio->bio_list == NULL) {  in dio_await_one()
474 		dio->waiter = current;  in dio_await_one()
475 		spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_await_one()
478 		spin_lock_irqsave(&dio->bio_lock, flags);  in dio_await_one()
479 		dio->waiter = NULL;  in dio_await_one()
481 	if (dio->bio_list) {  in dio_await_one()
482 		bio = dio->bio_list;  in dio_await_one()
483 		dio->bio_list = bio->bi_private;  in dio_await_one()
485 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_await_one()
492 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)  in dio_bio_complete()  argument
495 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in dio_bio_complete()
496 	bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty;  in dio_bio_complete()
500 			dio->io_error = -EAGAIN;  in dio_bio_complete()
502 			dio->io_error = -EIO;  in dio_bio_complete()
505 	if (dio->is_async && should_dirty) {  in dio_bio_complete()
521 static void dio_await_completion(struct dio *dio)  in dio_await_completion()  argument
525 		bio = dio_await_one(dio);  in dio_await_completion()
527 			dio_bio_complete(dio, bio);  in dio_await_completion()
538 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)  in dio_bio_reap()  argument
543 		while (dio->bio_list) {  in dio_bio_reap()
548 			spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_reap()
549 			bio = dio->bio_list;  in dio_bio_reap()
550 			dio->bio_list = bio->bi_private;  in dio_bio_reap()
551 			spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_reap()
552 			ret2 = blk_status_to_errno(dio_bio_complete(dio, bio));  in dio_bio_reap()
585 static int dio_set_defer_completion(struct dio *dio)  in dio_set_defer_completion()  argument
587 	struct super_block *sb = dio->inode->i_sb;  in dio_set_defer_completion()
589 	if (dio->defer_completion)  in dio_set_defer_completion()
591 	dio->defer_completion = true;  in dio_set_defer_completion()
620 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,  in get_more_blocks()  argument
623 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in get_more_blocks()
636 	ret = dio->page_errors;  in get_more_blocks()
659 		if (dio->flags & DIO_SKIP_HOLES) {  in get_more_blocks()
660 			i_size = i_size_read(dio->inode);  in get_more_blocks()
665 		ret = (*sdio->get_block)(dio->inode, fs_startblk,  in get_more_blocks()
669 		dio->private = map_bh->b_private;  in get_more_blocks()
672 			ret = dio_set_defer_completion(dio);  in get_more_blocks()
680 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,  in dio_new_bio()  argument
686 	ret = dio_bio_reap(dio, sdio);  in dio_new_bio()
692 	dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);  in dio_new_bio()
737 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,  in dio_send_cur_page()  argument
763 			dio_bio_submit(dio, sdio);  in dio_send_cur_page()
767 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
773 		dio_bio_submit(dio, sdio);  in dio_send_cur_page()
774 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
802 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,  in submit_page_section()  argument
806 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in submit_page_section()
832 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
851 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
853 			dio_bio_submit(dio, sdio);  in submit_page_section()
869 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,  in dio_zero_block()  argument
897 	if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,  in dio_zero_block()
920 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,  in do_direct_IO()  argument
923 	const enum req_op dio_op = dio->opf & REQ_OP_MASK;  in do_direct_IO()
932 		page = dio_get_page(dio, sdio);  in do_direct_IO()
953 				ret = get_more_blocks(dio, sdio, map_bh);  in do_direct_IO()
1008 				i_size_aligned = ALIGN(i_size_read(dio->inode),  in do_direct_IO()
1019 				dio->result += 1 << blkbits;  in do_direct_IO()
1029 				dio_zero_block(dio, sdio, 0, map_bh);  in do_direct_IO()
1047 			ret = submit_page_section(dio, sdio, page,  in do_direct_IO()
1060 			dio->result += this_chunk_bytes;  in do_direct_IO()
1075 static inline int drop_refcount(struct dio *dio)  in drop_refcount()  argument
1091 	spin_lock_irqsave(&dio->bio_lock, flags);  in drop_refcount()
1092 	ret2 = --dio->refcount;  in drop_refcount()
1093 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in drop_refcount()
1134 	struct dio *dio;  in __blockdev_direct_IO()  local
1149 	dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);  in __blockdev_direct_IO()
1150 	if (!dio)  in __blockdev_direct_IO()
1157 	memset(dio, 0, offsetof(struct dio, pages));  in __blockdev_direct_IO()
1159 	dio->flags = flags;  in __blockdev_direct_IO()
1160 	if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {  in __blockdev_direct_IO()
1166 	dio->i_size = i_size_read(inode);  in __blockdev_direct_IO()
1167 	if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {  in __blockdev_direct_IO()
1180 	if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {  in __blockdev_direct_IO()
1195 		dio->is_async = false;  in __blockdev_direct_IO()
1197 		dio->is_async = false;  in __blockdev_direct_IO()
1199 		dio->is_async = true;  in __blockdev_direct_IO()
1201 	dio->inode = inode;  in __blockdev_direct_IO()
1203 		dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;  in __blockdev_direct_IO()
1205 			dio->opf |= REQ_NOWAIT;  in __blockdev_direct_IO()
1207 		dio->opf = REQ_OP_READ;  in __blockdev_direct_IO()
1214 	if (dio->is_async && iov_iter_rw(iter) == WRITE) {  in __blockdev_direct_IO()
1217 			retval = dio_set_defer_completion(dio);  in __blockdev_direct_IO()
1218 		else if (!dio->inode->i_sb->s_dio_done_wq) {  in __blockdev_direct_IO()
1224 			retval = sb_init_dio_done_wq(dio->inode->i_sb);  in __blockdev_direct_IO()
1241 	dio->end_io = end_io;  in __blockdev_direct_IO()
1246 	dio->iocb = iocb;  in __blockdev_direct_IO()
1248 	spin_lock_init(&dio->bio_lock);  in __blockdev_direct_IO()
1249 	dio->refcount = 1;  in __blockdev_direct_IO()
1251 	dio->should_dirty = user_backed_iter(iter) && iov_iter_rw(iter) == READ;  in __blockdev_direct_IO()
1266 	retval = do_direct_IO(dio, &sdio, &map_bh);  in __blockdev_direct_IO()
1268 		dio_cleanup(dio, &sdio);  in __blockdev_direct_IO()
1281 	dio_zero_block(dio, &sdio, 1, &map_bh);  in __blockdev_direct_IO()
1286 		ret2 = dio_send_cur_page(dio, &sdio, &map_bh);  in __blockdev_direct_IO()
1293 		dio_bio_submit(dio, &sdio);  in __blockdev_direct_IO()
1301 	dio_cleanup(dio, &sdio);  in __blockdev_direct_IO()
1308 	if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING))  in __blockdev_direct_IO()
1309 		inode_unlock(dio->inode);  in __blockdev_direct_IO()
1319 	if (dio->is_async && retval == 0 && dio->result &&  in __blockdev_direct_IO()
1320 	    (iov_iter_rw(iter) == READ || dio->result == count))  in __blockdev_direct_IO()
1323 		dio_await_completion(dio);  in __blockdev_direct_IO()
1325 	if (drop_refcount(dio) == 0) {  in __blockdev_direct_IO()
1326 		retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE);  in __blockdev_direct_IO()
1333 	if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ)  in __blockdev_direct_IO()
1336 	kmem_cache_free(dio_cache, dio);  in __blockdev_direct_IO()
1343 	dio_cache = KMEM_CACHE(dio, SLAB_PANIC);  in dio_init()