Lines Matching refs:dio
1440 static ssize_t iomap_dio_complete(struct iomap_dio *dio) in iomap_dio_complete() argument
1442 struct kiocb *iocb = dio->iocb; in iomap_dio_complete()
1447 if (dio->end_io) { in iomap_dio_complete()
1448 ret = dio->end_io(iocb, in iomap_dio_complete()
1449 dio->error ? dio->error : dio->size, in iomap_dio_complete()
1450 dio->flags); in iomap_dio_complete()
1452 ret = dio->error; in iomap_dio_complete()
1456 ret = dio->size; in iomap_dio_complete()
1458 if (offset + ret > dio->i_size && in iomap_dio_complete()
1459 !(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_complete()
1460 ret = dio->i_size - offset; in iomap_dio_complete()
1476 if (!dio->error && in iomap_dio_complete()
1477 (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { in iomap_dio_complete()
1481 (offset + dio->size - 1) >> PAGE_SHIFT); in iomap_dio_complete()
1490 if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) in iomap_dio_complete()
1494 kfree(dio); in iomap_dio_complete()
1501 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); in iomap_dio_complete_work() local
1502 struct kiocb *iocb = dio->iocb; in iomap_dio_complete_work()
1504 iocb->ki_complete(iocb, iomap_dio_complete(dio), 0); in iomap_dio_complete_work()
1512 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) in iomap_dio_set_error() argument
1514 cmpxchg(&dio->error, 0, ret); in iomap_dio_set_error()
1519 struct iomap_dio *dio = bio->bi_private; in iomap_dio_bio_end_io() local
1520 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY); in iomap_dio_bio_end_io()
1523 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); in iomap_dio_bio_end_io()
1525 if (atomic_dec_and_test(&dio->ref)) { in iomap_dio_bio_end_io()
1526 if (dio->wait_for_completion) { in iomap_dio_bio_end_io()
1527 struct task_struct *waiter = dio->submit.waiter; in iomap_dio_bio_end_io()
1528 WRITE_ONCE(dio->submit.waiter, NULL); in iomap_dio_bio_end_io()
1530 } else if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_end_io()
1531 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_bio_end_io()
1533 INIT_WORK(&dio->aio.work, iomap_dio_complete_work); in iomap_dio_bio_end_io()
1534 queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); in iomap_dio_bio_end_io()
1536 iomap_dio_complete_work(&dio->aio.work); in iomap_dio_bio_end_io()
1553 iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, in iomap_dio_zero() argument
1562 bio->bi_private = dio; in iomap_dio_zero()
1569 atomic_inc(&dio->ref); in iomap_dio_zero()
1575 struct iomap_dio *dio, struct iomap *iomap) in iomap_dio_bio_actor() argument
1579 unsigned int align = iov_iter_alignment(dio->submit.iter); in iomap_dio_bio_actor()
1591 dio->flags |= IOMAP_DIO_UNWRITTEN; in iomap_dio_bio_actor()
1596 dio->flags |= IOMAP_DIO_COW; in iomap_dio_bio_actor()
1608 (dio->flags & IOMAP_DIO_WRITE_FUA) && in iomap_dio_bio_actor()
1617 iter = *dio->submit.iter; in iomap_dio_bio_actor()
1628 iomap_dio_zero(dio, iomap, pos - pad, pad); in iomap_dio_bio_actor()
1633 if (dio->error) { in iomap_dio_bio_actor()
1634 iov_iter_revert(dio->submit.iter, copied); in iomap_dio_bio_actor()
1641 bio->bi_write_hint = dio->iocb->ki_hint; in iomap_dio_bio_actor()
1642 bio->bi_ioprio = dio->iocb->ki_ioprio; in iomap_dio_bio_actor()
1643 bio->bi_private = dio; in iomap_dio_bio_actor()
1653 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_actor()
1658 dio->flags &= ~IOMAP_DIO_WRITE_FUA; in iomap_dio_bio_actor()
1662 if (dio->flags & IOMAP_DIO_DIRTY) in iomap_dio_bio_actor()
1666 iov_iter_advance(dio->submit.iter, n); in iomap_dio_bio_actor()
1668 dio->size += n; in iomap_dio_bio_actor()
1674 atomic_inc(&dio->ref); in iomap_dio_bio_actor()
1676 dio->submit.last_queue = bdev_get_queue(iomap->bdev); in iomap_dio_bio_actor()
1677 dio->submit.cookie = submit_bio(bio); in iomap_dio_bio_actor()
1684 iomap_dio_zero(dio, iomap, pos, fs_block_size - pad); in iomap_dio_bio_actor()
1690 iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio) in iomap_dio_hole_actor() argument
1692 length = iov_iter_zero(length, dio->submit.iter); in iomap_dio_hole_actor()
1693 dio->size += length; in iomap_dio_hole_actor()
1699 struct iomap_dio *dio, struct iomap *iomap) in iomap_dio_inline_actor() argument
1701 struct iov_iter *iter = dio->submit.iter; in iomap_dio_inline_actor()
1706 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_inline_actor()
1720 dio->size += copied; in iomap_dio_inline_actor()
1728 struct iomap_dio *dio = data; in iomap_dio_actor() local
1732 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_actor()
1734 return iomap_dio_hole_actor(length, dio); in iomap_dio_actor()
1736 if (!(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_actor()
1737 return iomap_dio_hole_actor(length, dio); in iomap_dio_actor()
1738 return iomap_dio_bio_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
1740 return iomap_dio_bio_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
1742 return iomap_dio_inline_actor(inode, pos, length, dio, iomap); in iomap_dio_actor()
1769 struct iomap_dio *dio; in iomap_dio_rw() local
1776 dio = kmalloc(sizeof(*dio), GFP_KERNEL); in iomap_dio_rw()
1777 if (!dio) in iomap_dio_rw()
1780 dio->iocb = iocb; in iomap_dio_rw()
1781 atomic_set(&dio->ref, 1); in iomap_dio_rw()
1782 dio->size = 0; in iomap_dio_rw()
1783 dio->i_size = i_size_read(inode); in iomap_dio_rw()
1784 dio->end_io = end_io; in iomap_dio_rw()
1785 dio->error = 0; in iomap_dio_rw()
1786 dio->flags = 0; in iomap_dio_rw()
1787 dio->wait_for_completion = is_sync_kiocb(iocb); in iomap_dio_rw()
1789 dio->submit.iter = iter; in iomap_dio_rw()
1790 dio->submit.waiter = current; in iomap_dio_rw()
1791 dio->submit.cookie = BLK_QC_T_NONE; in iomap_dio_rw()
1792 dio->submit.last_queue = NULL; in iomap_dio_rw()
1795 if (pos >= dio->i_size) in iomap_dio_rw()
1799 dio->flags |= IOMAP_DIO_DIRTY; in iomap_dio_rw()
1802 dio->flags |= IOMAP_DIO_WRITE; in iomap_dio_rw()
1806 dio->flags |= IOMAP_DIO_NEED_SYNC; in iomap_dio_rw()
1815 dio->flags |= IOMAP_DIO_WRITE_FUA; in iomap_dio_rw()
1842 if (iov_iter_rw(iter) == WRITE && !dio->wait_for_completion && in iomap_dio_rw()
1853 ret = iomap_apply(inode, pos, count, flags, ops, dio, in iomap_dio_rw()
1858 dio->wait_for_completion = true; in iomap_dio_rw()
1865 if (iov_iter_rw(iter) == READ && pos >= dio->i_size) in iomap_dio_rw()
1871 iomap_dio_set_error(dio, ret); in iomap_dio_rw()
1877 if (dio->flags & IOMAP_DIO_WRITE_FUA) in iomap_dio_rw()
1878 dio->flags &= ~IOMAP_DIO_NEED_SYNC; in iomap_dio_rw()
1880 if (!atomic_dec_and_test(&dio->ref)) { in iomap_dio_rw()
1881 if (!dio->wait_for_completion) in iomap_dio_rw()
1886 if (!READ_ONCE(dio->submit.waiter)) in iomap_dio_rw()
1890 !dio->submit.last_queue || in iomap_dio_rw()
1891 !blk_poll(dio->submit.last_queue, in iomap_dio_rw()
1892 dio->submit.cookie)) in iomap_dio_rw()
1898 ret = iomap_dio_complete(dio); in iomap_dio_rw()
1903 kfree(dio); in iomap_dio_rw()