Lines Matching refs:fio
410 static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) in f2fs_io_flags() argument
416 if (fio->op != REQ_OP_WRITE) in f2fs_io_flags()
418 if (fio->type == DATA) in f2fs_io_flags()
419 io_flag = fio->sbi->data_io_flag; in f2fs_io_flags()
420 else if (fio->type == NODE) in f2fs_io_flags()
421 io_flag = fio->sbi->node_io_flag; in f2fs_io_flags()
434 if ((1 << fio->temp) & meta_flag) in f2fs_io_flags()
436 if ((1 << fio->temp) & fua_flag) in f2fs_io_flags()
441 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) in __bio_alloc() argument
443 struct f2fs_sb_info *sbi = fio->sbi; in __bio_alloc()
448 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); in __bio_alloc()
450 fio->op | fio->op_flags | f2fs_io_flags(fio), in __bio_alloc()
453 if (is_read_io(fio->op)) { in __bio_alloc()
462 if (fio->io_wbc) in __bio_alloc()
463 wbc_init_bio(fio->io_wbc, bio); in __bio_alloc()
470 const struct f2fs_io_info *fio, in f2fs_set_bio_crypt_ctx() argument
477 if (!fio || !fio->encrypted_page) in f2fs_set_bio_crypt_ctx()
483 const struct f2fs_io_info *fio) in f2fs_crypt_mergeable_bio() argument
489 if (fio && fio->encrypted_page) in f2fs_crypt_mergeable_bio()
556 struct f2fs_io_info *fio = &io->fio; in __submit_merged_bio() local
561 if (is_read_io(fio->op)) in __submit_merged_bio()
562 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
564 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
566 __submit_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
645 io->fio.type = META_FLUSH; in __f2fs_submit_merged_write()
702 int f2fs_submit_page_bio(struct f2fs_io_info *fio) in f2fs_submit_page_bio() argument
705 struct page *page = fio->encrypted_page ? in f2fs_submit_page_bio()
706 fio->encrypted_page : fio->page; in f2fs_submit_page_bio()
708 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_submit_page_bio()
709 fio->is_por ? META_POR : (__is_meta_io(fio) ? in f2fs_submit_page_bio()
711 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_submit_page_bio()
715 trace_f2fs_submit_page_bio(page, fio); in f2fs_submit_page_bio()
718 bio = __bio_alloc(fio, 1); in f2fs_submit_page_bio()
720 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, in f2fs_submit_page_bio()
721 fio->page->index, fio, GFP_NOIO); in f2fs_submit_page_bio()
728 if (fio->io_wbc && !is_read_io(fio->op)) in f2fs_submit_page_bio()
729 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); in f2fs_submit_page_bio()
731 inc_page_count(fio->sbi, is_read_io(fio->op) ? in f2fs_submit_page_bio()
732 __read_io_type(page) : WB_DATA_TYPE(fio->page)); in f2fs_submit_page_bio()
734 __submit_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
750 struct f2fs_io_info *fio) in io_type_is_mergeable() argument
752 if (io->fio.op != fio->op) in io_type_is_mergeable()
754 return io->fio.op_flags == fio->op_flags; in io_type_is_mergeable()
759 struct f2fs_io_info *fio, in io_is_mergeable() argument
763 if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) { in io_is_mergeable()
775 return io_type_is_mergeable(io, fio); in io_is_mergeable()
802 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, in add_ipu_page() argument
805 struct f2fs_sb_info *sbi = fio->sbi; in add_ipu_page()
823 *fio->last_block, in add_ipu_page()
824 fio->new_blkaddr)); in add_ipu_page()
826 fio->page->mapping->host, in add_ipu_page()
827 fio->page->index, fio) && in add_ipu_page()
906 int f2fs_merge_page_bio(struct f2fs_io_info *fio) in f2fs_merge_page_bio() argument
908 struct bio *bio = *fio->bio; in f2fs_merge_page_bio()
909 struct page *page = fio->encrypted_page ? in f2fs_merge_page_bio()
910 fio->encrypted_page : fio->page; in f2fs_merge_page_bio()
912 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_merge_page_bio()
913 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) { in f2fs_merge_page_bio()
914 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_merge_page_bio()
918 trace_f2fs_submit_page_bio(page, fio); in f2fs_merge_page_bio()
920 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, in f2fs_merge_page_bio()
921 fio->new_blkaddr)) in f2fs_merge_page_bio()
922 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); in f2fs_merge_page_bio()
925 bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_merge_page_bio()
926 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, in f2fs_merge_page_bio()
927 fio->page->index, fio, GFP_NOIO); in f2fs_merge_page_bio()
929 add_bio_entry(fio->sbi, bio, page, fio->temp); in f2fs_merge_page_bio()
931 if (add_ipu_page(fio, &bio, page)) in f2fs_merge_page_bio()
935 if (fio->io_wbc) in f2fs_merge_page_bio()
936 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); in f2fs_merge_page_bio()
938 inc_page_count(fio->sbi, WB_DATA_TYPE(page)); in f2fs_merge_page_bio()
940 *fio->last_block = fio->new_blkaddr; in f2fs_merge_page_bio()
941 *fio->bio = bio; in f2fs_merge_page_bio()
946 void f2fs_submit_page_write(struct f2fs_io_info *fio) in f2fs_submit_page_write() argument
948 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_submit_page_write()
949 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); in f2fs_submit_page_write()
950 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; in f2fs_submit_page_write()
953 f2fs_bug_on(sbi, is_read_io(fio->op)); in f2fs_submit_page_write()
957 if (fio->in_list) { in f2fs_submit_page_write()
963 fio = list_first_entry(&io->io_list, in f2fs_submit_page_write()
965 list_del(&fio->list); in f2fs_submit_page_write()
969 verify_fio_blkaddr(fio); in f2fs_submit_page_write()
971 if (fio->encrypted_page) in f2fs_submit_page_write()
972 bio_page = fio->encrypted_page; in f2fs_submit_page_write()
973 else if (fio->compressed_page) in f2fs_submit_page_write()
974 bio_page = fio->compressed_page; in f2fs_submit_page_write()
976 bio_page = fio->page; in f2fs_submit_page_write()
979 fio->submitted = true; in f2fs_submit_page_write()
984 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, in f2fs_submit_page_write()
985 fio->new_blkaddr) || in f2fs_submit_page_write()
986 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host, in f2fs_submit_page_write()
987 bio_page->index, fio))) in f2fs_submit_page_write()
992 (fio->type == DATA || fio->type == NODE) && in f2fs_submit_page_write()
993 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { in f2fs_submit_page_write()
995 fio->retry = true; in f2fs_submit_page_write()
998 io->bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_submit_page_write()
999 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, in f2fs_submit_page_write()
1000 bio_page->index, fio, GFP_NOIO); in f2fs_submit_page_write()
1001 io->fio = *fio; in f2fs_submit_page_write()
1009 if (fio->io_wbc) in f2fs_submit_page_write()
1010 wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE); in f2fs_submit_page_write()
1012 io->last_block_in_bio = fio->new_blkaddr; in f2fs_submit_page_write()
1014 trace_f2fs_submit_page_write(fio->page, fio); in f2fs_submit_page_write()
1016 if (fio->in_list) in f2fs_submit_page_write()
2471 int f2fs_encrypt_one_page(struct f2fs_io_info *fio) in f2fs_encrypt_one_page() argument
2473 struct inode *inode = fio->page->mapping->host; in f2fs_encrypt_one_page()
2480 page = fio->compressed_page ? fio->compressed_page : fio->page; in f2fs_encrypt_one_page()
2483 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); in f2fs_encrypt_one_page()
2489 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page, in f2fs_encrypt_one_page()
2491 if (IS_ERR(fio->encrypted_page)) { in f2fs_encrypt_one_page()
2493 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { in f2fs_encrypt_one_page()
2494 f2fs_flush_merged_writes(fio->sbi); in f2fs_encrypt_one_page()
2499 return PTR_ERR(fio->encrypted_page); in f2fs_encrypt_one_page()
2502 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); in f2fs_encrypt_one_page()
2506 page_address(fio->encrypted_page), PAGE_SIZE); in f2fs_encrypt_one_page()
2513 struct f2fs_io_info *fio) in check_inplace_update_policy() argument
2536 fio && fio->op == REQ_OP_WRITE && in check_inplace_update_policy()
2537 !(fio->op_flags & REQ_SYNC) && in check_inplace_update_policy()
2546 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && in check_inplace_update_policy()
2547 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in check_inplace_update_policy()
2553 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) in f2fs_should_update_inplace() argument
2566 return check_inplace_update_policy(inode, fio); in f2fs_should_update_inplace()
2569 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) in f2fs_should_update_outplace() argument
2576 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) in f2fs_should_update_outplace()
2594 if (fio) { in f2fs_should_update_outplace()
2595 if (page_private_gcing(fio->page)) in f2fs_should_update_outplace()
2597 if (page_private_dummy(fio->page)) in f2fs_should_update_outplace()
2600 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in f2fs_should_update_outplace()
2606 static inline bool need_inplace_update(struct f2fs_io_info *fio) in need_inplace_update() argument
2608 struct inode *inode = fio->page->mapping->host; in need_inplace_update()
2610 if (f2fs_should_update_outplace(inode, fio)) in need_inplace_update()
2613 return f2fs_should_update_inplace(inode, fio); in need_inplace_update()
2616 int f2fs_do_write_data_page(struct f2fs_io_info *fio) in f2fs_do_write_data_page() argument
2618 struct page *page = fio->page; in f2fs_do_write_data_page()
2632 if (need_inplace_update(fio) && in f2fs_do_write_data_page()
2634 fio->old_blkaddr = ei.blk + page->index - ei.fofs; in f2fs_do_write_data_page()
2636 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2638 f2fs_handle_error(fio->sbi, in f2fs_do_write_data_page()
2644 fio->need_lock = LOCK_DONE; in f2fs_do_write_data_page()
2649 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) in f2fs_do_write_data_page()
2656 fio->old_blkaddr = dn.data_blkaddr; in f2fs_do_write_data_page()
2659 if (fio->old_blkaddr == NULL_ADDR) { in f2fs_do_write_data_page()
2665 if (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2666 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2669 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_do_write_data_page()
2678 (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2679 need_inplace_update(fio))) { in f2fs_do_write_data_page()
2680 err = f2fs_encrypt_one_page(fio); in f2fs_do_write_data_page()
2687 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2688 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2689 err = f2fs_inplace_write_data(fio); in f2fs_do_write_data_page()
2692 fscrypt_finalize_bounce_page(&fio->encrypted_page); in f2fs_do_write_data_page()
2698 trace_f2fs_do_write_data_page(fio->page, IPU); in f2fs_do_write_data_page()
2702 if (fio->need_lock == LOCK_RETRY) { in f2fs_do_write_data_page()
2703 if (!f2fs_trylock_op(fio->sbi)) { in f2fs_do_write_data_page()
2707 fio->need_lock = LOCK_REQ; in f2fs_do_write_data_page()
2710 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); in f2fs_do_write_data_page()
2714 fio->version = ni.version; in f2fs_do_write_data_page()
2716 err = f2fs_encrypt_one_page(fio); in f2fs_do_write_data_page()
2723 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) in f2fs_do_write_data_page()
2724 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); in f2fs_do_write_data_page()
2727 f2fs_outplace_write_data(&dn, fio); in f2fs_do_write_data_page()
2735 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2736 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2757 struct f2fs_io_info fio = { in f2fs_write_single_data_page() local
2821 fio.need_lock = LOCK_DONE; in f2fs_write_single_data_page()
2822 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2845 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2847 fio.need_lock = LOCK_REQ; in f2fs_write_single_data_page()
2848 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2890 *submitted = fio.submitted ? 1 : 0; in f2fs_write_single_data_page()