Lines Matching +full:cold +full:- +full:temp
1 // SPDX-License-Identifier: GPL-2.0
17 #include <linux/blk-crypto.h>
53 struct address_space *mapping = page->mapping; in __is_cp_guaranteed()
60 inode = mapping->host; in __is_cp_guaranteed()
63 if (inode->i_ino == F2FS_META_INO(sbi) || in __is_cp_guaranteed()
64 inode->i_ino == F2FS_NODE_INO(sbi) || in __is_cp_guaranteed()
65 S_ISDIR(inode->i_mode)) in __is_cp_guaranteed()
70 if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || in __is_cp_guaranteed()
81 struct inode *inode = mapping->host; in __read_io_type()
84 if (inode->i_ino == F2FS_META_INO(sbi)) in __read_io_type()
87 if (inode->i_ino == F2FS_NODE_INO(sbi)) in __read_io_type()
98 STEP_DECRYPT = 0, /* compile out the decryption-related code */
103 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
108 STEP_VERITY = 0, /* compile out the verity-related code */
129 * This marks pages up-to-date only if there was no error in the bio (I/O error,
130 * decryption error, or verity error), as indicated by bio->bi_status.
132 * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk)
133 * aren't marked up-to-date here, as decompression is done on a per-compression-
134 * cluster basis rather than a per-bio basis. Instead, we only must do two
144 struct bio_post_read_ctx *ctx = bio->bi_private; in f2fs_finish_read_bio()
147 struct page *page = bv->bv_page; in f2fs_finish_read_bio()
150 if (ctx && !ctx->decompression_attempted) in f2fs_finish_read_bio()
157 if (bio->bi_status) in f2fs_finish_read_bio()
174 struct bio *bio = ctx->bio; in f2fs_verify_bio()
175 bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); in f2fs_verify_bio()
182 * mempool first. This assumes that verity is the last post-read step. in f2fs_verify_bio()
185 bio->bi_private = NULL; in f2fs_verify_bio()
188 * Verify the bio's pages with fs-verity. Exclude compressed pages, in f2fs_verify_bio()
196 struct page *page = bv->bv_page; in f2fs_verify_bio()
200 bio->bi_status = BLK_STS_IOERR; in f2fs_verify_bio()
212 * If the bio's data needs to be verified with fs-verity, then enqueue the
222 struct bio_post_read_ctx *ctx = bio->bi_private; in f2fs_verify_and_finish_bio()
224 if (ctx && (ctx->enabled_steps & STEP_VERITY)) { in f2fs_verify_and_finish_bio()
225 INIT_WORK(&ctx->work, f2fs_verify_bio); in f2fs_verify_and_finish_bio()
226 fsverity_enqueue_verify_work(&ctx->work); in f2fs_verify_and_finish_bio()
234 * remaining page was read by @ctx->bio.
239 * is done on a per-cluster basis, not a per-bio basis.
247 block_t blkaddr = ctx->fs_blkaddr; in f2fs_handle_step_decompress()
249 bio_for_each_segment_all(bv, ctx->bio, iter_all) { in f2fs_handle_step_decompress()
250 struct page *page = bv->bv_page; in f2fs_handle_step_decompress()
261 ctx->decompression_attempted = true; in f2fs_handle_step_decompress()
265 * the per-bio verity work is unnecessary, as verity will be fully in f2fs_handle_step_decompress()
269 ctx->enabled_steps &= ~STEP_VERITY; in f2fs_handle_step_decompress()
276 struct bio *bio = ctx->bio; in f2fs_post_read_work()
278 if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { in f2fs_post_read_work()
283 if (ctx->enabled_steps & STEP_DECOMPRESS) in f2fs_post_read_work()
296 ctx = bio->bi_private; in f2fs_read_end_io()
299 bio->bi_status = BLK_STS_IOERR; in f2fs_read_end_io()
301 if (bio->bi_status) { in f2fs_read_end_io()
307 unsigned int enabled_steps = ctx->enabled_steps & in f2fs_read_end_io()
318 INIT_WORK(&ctx->work, f2fs_post_read_work); in f2fs_read_end_io()
319 queue_work(ctx->sbi->post_read_wq, &ctx->work); in f2fs_read_end_io()
334 sbi = bio->bi_private; in f2fs_write_end_io()
337 bio->bi_status = BLK_STS_IOERR; in f2fs_write_end_io()
340 struct page *page = bvec->bv_page; in f2fs_write_end_io()
346 mempool_free(page, sbi->write_io_dummy); in f2fs_write_end_io()
348 if (unlikely(bio->bi_status)) in f2fs_write_end_io()
363 if (unlikely(bio->bi_status)) { in f2fs_write_end_io()
364 mapping_set_error(page->mapping, -EIO); in f2fs_write_end_io()
370 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && in f2fs_write_end_io()
371 page->index != nid_of_node(page)); in f2fs_write_end_io()
380 wq_has_sleeper(&sbi->cp_wait)) in f2fs_write_end_io()
381 wake_up(&sbi->cp_wait); in f2fs_write_end_io()
389 struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; in f2fs_zone_write_end_io()
391 bio->bi_private = io->bi_private; in f2fs_zone_write_end_io()
392 complete(&io->zone_wait); in f2fs_zone_write_end_io()
400 struct block_device *bdev = sbi->sb->s_bdev; in f2fs_target_device()
404 for (i = 0; i < sbi->s_ndevs; i++) { in f2fs_target_device()
407 blk_addr -= FDEV(i).start_blk; in f2fs_target_device()
426 for (i = 0; i < sbi->s_ndevs; i++) in f2fs_target_device_index()
434 unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); in f2fs_io_flags()
438 if (fio->op != REQ_OP_WRITE) in f2fs_io_flags()
440 if (fio->type == DATA) in f2fs_io_flags()
441 io_flag = fio->sbi->data_io_flag; in f2fs_io_flags()
442 else if (fio->type == NODE) in f2fs_io_flags()
443 io_flag = fio->sbi->node_io_flag; in f2fs_io_flags()
451 * data/node io flag bits per temp: in f2fs_io_flags()
454 * Cold | Warm | Hot | Cold | Warm | Hot | in f2fs_io_flags()
456 if (BIT(fio->temp) & meta_flag) in f2fs_io_flags()
458 if (BIT(fio->temp) & fua_flag) in f2fs_io_flags()
465 struct f2fs_sb_info *sbi = fio->sbi; in __bio_alloc()
470 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); in __bio_alloc()
472 fio->op | fio->op_flags | f2fs_io_flags(fio), in __bio_alloc()
474 bio->bi_iter.bi_sector = sector; in __bio_alloc()
475 if (is_read_io(fio->op)) { in __bio_alloc()
476 bio->bi_end_io = f2fs_read_end_io; in __bio_alloc()
477 bio->bi_private = NULL; in __bio_alloc()
479 bio->bi_end_io = f2fs_write_end_io; in __bio_alloc()
480 bio->bi_private = sbi; in __bio_alloc()
484 if (fio->io_wbc) in __bio_alloc()
485 wbc_init_bio(fio->io_wbc, bio); in __bio_alloc()
496 * The f2fs garbage collector sets ->encrypted_page when it wants to in f2fs_set_bio_crypt_ctx()
499 if (!fio || !fio->encrypted_page) in f2fs_set_bio_crypt_ctx()
508 * The f2fs garbage collector sets ->encrypted_page when it wants to in f2fs_crypt_mergeable_bio()
511 if (fio && fio->encrypted_page) in f2fs_crypt_mergeable_bio()
521 trace_f2fs_submit_read_bio(sbi->sb, type, bio); in f2fs_submit_read_bio()
530 (bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS) % F2FS_IO_SIZE(sbi); in f2fs_align_write_bio()
538 mempool_alloc(sbi->write_io_dummy, in f2fs_align_write_bio()
558 if (f2fs_lfs_mode(sbi) && current->plug) in f2fs_submit_write_bio()
559 blk_finish_plug(current->plug); in f2fs_submit_write_bio()
572 trace_f2fs_submit_write_bio(sbi->sb, type, bio); in f2fs_submit_write_bio()
579 struct f2fs_io_info *fio = &io->fio; in __submit_merged_bio()
581 if (!io->bio) in __submit_merged_bio()
584 if (is_read_io(fio->op)) { in __submit_merged_bio()
585 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
586 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
588 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
589 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
591 io->bio = NULL; in __submit_merged_bio()
607 struct page *target = bvec->bv_page; in __has_merged_page()
620 if (inode && inode == target->mapping->host) in __has_merged_page()
639 sbi->write_io[i] = f2fs_kmalloc(sbi, in f2fs_init_write_merge_io()
642 if (!sbi->write_io[i]) in f2fs_init_write_merge_io()
643 return -ENOMEM; in f2fs_init_write_merge_io()
646 init_f2fs_rwsem(&sbi->write_io[i][j].io_rwsem); in f2fs_init_write_merge_io()
647 sbi->write_io[i][j].sbi = sbi; in f2fs_init_write_merge_io()
648 sbi->write_io[i][j].bio = NULL; in f2fs_init_write_merge_io()
649 spin_lock_init(&sbi->write_io[i][j].io_lock); in f2fs_init_write_merge_io()
650 INIT_LIST_HEAD(&sbi->write_io[i][j].io_list); in f2fs_init_write_merge_io()
651 INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list); in f2fs_init_write_merge_io()
652 init_f2fs_rwsem(&sbi->write_io[i][j].bio_list_lock); in f2fs_init_write_merge_io()
654 init_completion(&sbi->write_io[i][j].zone_wait); in f2fs_init_write_merge_io()
655 sbi->write_io[i][j].zone_pending_bio = NULL; in f2fs_init_write_merge_io()
656 sbi->write_io[i][j].bi_private = NULL; in f2fs_init_write_merge_io()
665 enum page_type type, enum temp_type temp) in __f2fs_submit_merged_write() argument
668 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; in __f2fs_submit_merged_write()
670 f2fs_down_write(&io->io_rwsem); in __f2fs_submit_merged_write()
672 if (!io->bio) in __f2fs_submit_merged_write()
677 io->fio.type = META_FLUSH; in __f2fs_submit_merged_write()
678 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; in __f2fs_submit_merged_write()
680 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; in __f2fs_submit_merged_write()
684 f2fs_up_write(&io->io_rwsem); in __f2fs_submit_merged_write()
691 enum temp_type temp; in __submit_merged_write_cond() local
694 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { in __submit_merged_write_cond()
697 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; in __submit_merged_write_cond()
699 f2fs_down_read(&io->io_rwsem); in __submit_merged_write_cond()
700 ret = __has_merged_page(io->bio, inode, page, ino); in __submit_merged_write_cond()
701 f2fs_up_read(&io->io_rwsem); in __submit_merged_write_cond()
704 __f2fs_submit_merged_write(sbi, type, temp); in __submit_merged_write_cond()
706 /* TODO: use HOT temp only for meta pages now. */ in __submit_merged_write_cond()
738 struct page *page = fio->encrypted_page ? in f2fs_submit_page_bio()
739 fio->encrypted_page : fio->page; in f2fs_submit_page_bio()
741 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_submit_page_bio()
742 fio->is_por ? META_POR : (__is_meta_io(fio) ? in f2fs_submit_page_bio()
744 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_submit_page_bio()
745 return -EFSCORRUPTED; in f2fs_submit_page_bio()
753 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, in f2fs_submit_page_bio()
754 fio->page->index, fio, GFP_NOIO); in f2fs_submit_page_bio()
758 return -EFAULT; in f2fs_submit_page_bio()
761 if (fio->io_wbc && !is_read_io(fio->op)) in f2fs_submit_page_bio()
762 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); in f2fs_submit_page_bio()
764 inc_page_count(fio->sbi, is_read_io(fio->op) ? in f2fs_submit_page_bio()
765 __read_io_type(page) : WB_DATA_TYPE(fio->page)); in f2fs_submit_page_bio()
768 f2fs_submit_read_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
770 f2fs_submit_write_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
777 if (unlikely(sbi->max_io_bytes && in page_is_mergeable()
778 bio->bi_iter.bi_size >= sbi->max_io_bytes)) in page_is_mergeable()
782 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); in page_is_mergeable()
788 if (io->fio.op != fio->op) in io_type_is_mergeable()
790 return io->fio.op_flags == fio->op_flags; in io_type_is_mergeable()
799 if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) { in io_is_mergeable()
801 F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size); in io_is_mergeable()
803 unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt; in io_is_mergeable()
815 struct page *page, enum temp_type temp) in add_bio_entry() argument
817 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; in add_bio_entry()
821 be->bio = bio; in add_bio_entry()
827 f2fs_down_write(&io->bio_list_lock); in add_bio_entry()
828 list_add_tail(&be->list, &io->bio_list); in add_bio_entry()
829 f2fs_up_write(&io->bio_list_lock); in add_bio_entry()
834 list_del(&be->list); in del_bio_entry()
841 struct f2fs_sb_info *sbi = fio->sbi; in add_ipu_page()
842 enum temp_type temp; in add_ipu_page() local
844 int ret = -EAGAIN; in add_ipu_page()
846 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { in add_ipu_page()
847 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; in add_ipu_page()
848 struct list_head *head = &io->bio_list; in add_ipu_page()
851 f2fs_down_write(&io->bio_list_lock); in add_ipu_page()
853 if (be->bio != *bio) in add_ipu_page()
859 *fio->last_block, in add_ipu_page()
860 fio->new_blkaddr)); in add_ipu_page()
862 fio->page->mapping->host, in add_ipu_page()
863 fio->page->index, fio) && in add_ipu_page()
875 f2fs_up_write(&io->bio_list_lock); in add_ipu_page()
889 enum temp_type temp; in f2fs_submit_merged_ipu_write() local
895 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { in f2fs_submit_merged_ipu_write()
896 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; in f2fs_submit_merged_ipu_write()
897 struct list_head *head = &io->bio_list; in f2fs_submit_merged_ipu_write()
903 f2fs_down_read(&io->bio_list_lock); in f2fs_submit_merged_ipu_write()
906 found = (target == be->bio); in f2fs_submit_merged_ipu_write()
908 found = __has_merged_page(be->bio, NULL, in f2fs_submit_merged_ipu_write()
913 f2fs_up_read(&io->bio_list_lock); in f2fs_submit_merged_ipu_write()
920 f2fs_down_write(&io->bio_list_lock); in f2fs_submit_merged_ipu_write()
923 found = (target == be->bio); in f2fs_submit_merged_ipu_write()
925 found = __has_merged_page(be->bio, NULL, in f2fs_submit_merged_ipu_write()
928 target = be->bio; in f2fs_submit_merged_ipu_write()
933 f2fs_up_write(&io->bio_list_lock); in f2fs_submit_merged_ipu_write()
946 struct bio *bio = *fio->bio; in f2fs_merge_page_bio()
947 struct page *page = fio->encrypted_page ? in f2fs_merge_page_bio()
948 fio->encrypted_page : fio->page; in f2fs_merge_page_bio()
950 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_merge_page_bio()
952 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_merge_page_bio()
953 return -EFSCORRUPTED; in f2fs_merge_page_bio()
958 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, in f2fs_merge_page_bio()
959 fio->new_blkaddr)) in f2fs_merge_page_bio()
960 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); in f2fs_merge_page_bio()
964 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, in f2fs_merge_page_bio()
965 fio->page->index, fio, GFP_NOIO); in f2fs_merge_page_bio()
967 add_bio_entry(fio->sbi, bio, page, fio->temp); in f2fs_merge_page_bio()
973 if (fio->io_wbc) in f2fs_merge_page_bio()
974 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); in f2fs_merge_page_bio()
976 inc_page_count(fio->sbi, WB_DATA_TYPE(page)); in f2fs_merge_page_bio()
978 *fio->last_block = fio->new_blkaddr; in f2fs_merge_page_bio()
979 *fio->bio = bio; in f2fs_merge_page_bio()
996 blkaddr -= FDEV(devi).start_blk; in is_end_zone_blkaddr()
1000 (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); in is_end_zone_blkaddr()
1006 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_submit_page_write()
1007 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); in f2fs_submit_page_write()
1008 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; in f2fs_submit_page_write()
1011 f2fs_bug_on(sbi, is_read_io(fio->op)); in f2fs_submit_page_write()
1013 f2fs_down_write(&io->io_rwsem); in f2fs_submit_page_write()
1016 if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { in f2fs_submit_page_write()
1017 wait_for_completion_io(&io->zone_wait); in f2fs_submit_page_write()
1018 bio_put(io->zone_pending_bio); in f2fs_submit_page_write()
1019 io->zone_pending_bio = NULL; in f2fs_submit_page_write()
1020 io->bi_private = NULL; in f2fs_submit_page_write()
1025 if (fio->in_list) { in f2fs_submit_page_write()
1026 spin_lock(&io->io_lock); in f2fs_submit_page_write()
1027 if (list_empty(&io->io_list)) { in f2fs_submit_page_write()
1028 spin_unlock(&io->io_lock); in f2fs_submit_page_write()
1031 fio = list_first_entry(&io->io_list, in f2fs_submit_page_write()
1033 list_del(&fio->list); in f2fs_submit_page_write()
1034 spin_unlock(&io->io_lock); in f2fs_submit_page_write()
1039 if (fio->encrypted_page) in f2fs_submit_page_write()
1040 bio_page = fio->encrypted_page; in f2fs_submit_page_write()
1041 else if (fio->compressed_page) in f2fs_submit_page_write()
1042 bio_page = fio->compressed_page; in f2fs_submit_page_write()
1044 bio_page = fio->page; in f2fs_submit_page_write()
1047 fio->submitted = 1; in f2fs_submit_page_write()
1051 if (io->bio && in f2fs_submit_page_write()
1052 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, in f2fs_submit_page_write()
1053 fio->new_blkaddr) || in f2fs_submit_page_write()
1054 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host, in f2fs_submit_page_write()
1055 bio_page->index, fio))) in f2fs_submit_page_write()
1058 if (io->bio == NULL) { in f2fs_submit_page_write()
1060 (fio->type == DATA || fio->type == NODE) && in f2fs_submit_page_write()
1061 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { in f2fs_submit_page_write()
1063 fio->retry = 1; in f2fs_submit_page_write()
1066 io->bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_submit_page_write()
1067 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, in f2fs_submit_page_write()
1068 bio_page->index, fio, GFP_NOIO); in f2fs_submit_page_write()
1069 io->fio = *fio; in f2fs_submit_page_write()
1072 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { in f2fs_submit_page_write()
1077 if (fio->io_wbc) in f2fs_submit_page_write()
1078 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); in f2fs_submit_page_write()
1080 io->last_block_in_bio = fio->new_blkaddr; in f2fs_submit_page_write()
1082 trace_f2fs_submit_page_write(fio->page, fio); in f2fs_submit_page_write()
1084 if (fio->in_list) in f2fs_submit_page_write()
1089 is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { in f2fs_submit_page_write()
1090 bio_get(io->bio); in f2fs_submit_page_write()
1091 reinit_completion(&io->zone_wait); in f2fs_submit_page_write()
1092 io->bi_private = io->bio->bi_private; in f2fs_submit_page_write()
1093 io->bio->bi_private = io; in f2fs_submit_page_write()
1094 io->bio->bi_end_io = f2fs_zone_write_end_io; in f2fs_submit_page_write()
1095 io->zone_pending_bio = io->bio; in f2fs_submit_page_write()
1102 f2fs_up_write(&io->io_rwsem); in f2fs_submit_page_write()
1120 return ERR_PTR(-ENOMEM); in f2fs_grab_read_bio()
1121 bio->bi_iter.bi_sector = sector; in f2fs_grab_read_bio()
1123 bio->bi_end_io = f2fs_read_end_io; in f2fs_grab_read_bio()
1141 ctx->bio = bio; in f2fs_grab_read_bio()
1142 ctx->sbi = sbi; in f2fs_grab_read_bio()
1143 ctx->enabled_steps = post_read_steps; in f2fs_grab_read_bio()
1144 ctx->fs_blkaddr = blkaddr; in f2fs_grab_read_bio()
1145 ctx->decompression_attempted = false; in f2fs_grab_read_bio()
1146 bio->bi_private = ctx; in f2fs_grab_read_bio()
1162 page->index, for_write); in f2fs_submit_page_read()
1171 if (bio->bi_private) in f2fs_submit_page_read()
1172 mempool_free(bio->bi_private, bio_post_read_ctx_pool); in f2fs_submit_page_read()
1174 return -EFAULT; in f2fs_submit_page_read()
1184 struct f2fs_node *rn = F2FS_NODE(dn->node_page); in __set_data_blkaddr()
1188 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode)) in __set_data_blkaddr()
1189 base = get_extra_isize(dn->inode); in __set_data_blkaddr()
1193 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); in __set_data_blkaddr()
1198 * ->data_page
1199 * ->node_page
1204 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); in f2fs_set_data_blkaddr()
1206 if (set_page_dirty(dn->node_page)) in f2fs_set_data_blkaddr()
1207 dn->node_changed = true; in f2fs_set_data_blkaddr()
1212 dn->data_blkaddr = blkaddr; in f2fs_update_data_blkaddr()
1217 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
1220 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); in f2fs_reserve_new_blocks()
1226 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) in f2fs_reserve_new_blocks()
1227 return -EPERM; in f2fs_reserve_new_blocks()
1228 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) in f2fs_reserve_new_blocks()
1231 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, in f2fs_reserve_new_blocks()
1232 dn->ofs_in_node, count); in f2fs_reserve_new_blocks()
1234 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); in f2fs_reserve_new_blocks()
1236 for (; count > 0; dn->ofs_in_node++) { in f2fs_reserve_new_blocks()
1240 dn->data_blkaddr = NEW_ADDR; in f2fs_reserve_new_blocks()
1242 count--; in f2fs_reserve_new_blocks()
1246 if (set_page_dirty(dn->node_page)) in f2fs_reserve_new_blocks()
1247 dn->node_changed = true; in f2fs_reserve_new_blocks()
1251 /* Should keep dn->ofs_in_node unchanged */
1254 unsigned int ofs_in_node = dn->ofs_in_node; in f2fs_reserve_new_block()
1258 dn->ofs_in_node = ofs_in_node; in f2fs_reserve_new_block()
1264 bool need_put = dn->inode_page ? false : true; in f2fs_reserve_block()
1271 if (dn->data_blkaddr == NULL_ADDR) in f2fs_reserve_block()
1282 struct address_space *mapping = inode->i_mapping; in f2fs_get_read_data_page()
1289 return ERR_PTR(-ENOMEM); in f2fs_get_read_data_page()
1295 err = -EFSCORRUPTED; in f2fs_get_read_data_page()
1306 if (err == -ENOENT && next_pgofs) in f2fs_get_read_data_page()
1313 err = -ENOENT; in f2fs_get_read_data_page()
1322 err = -EFSCORRUPTED; in f2fs_get_read_data_page()
1335 * new inode page couldn't be allocated due to -ENOSPC. in f2fs_get_read_data_page()
1337 * see, f2fs_add_link -> f2fs_get_new_data_page -> in f2fs_get_read_data_page()
1362 struct address_space *mapping = inode->i_mapping; in f2fs_find_data_page()
1380 return ERR_PTR(-EIO); in f2fs_find_data_page()
1393 struct address_space *mapping = inode->i_mapping; in f2fs_get_lock_data_page()
1402 if (unlikely(page->mapping != mapping || !PageUptodate(page))) { in f2fs_get_lock_data_page()
1404 return ERR_PTR(-EIO); in f2fs_get_lock_data_page()
1411 * A new zero-filled data page is allocated in the page cache.
1421 struct address_space *mapping = inode->i_mapping; in f2fs_get_new_data_page()
1433 return ERR_PTR(-ENOMEM); in f2fs_get_new_data_page()
1470 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); in __allocate_data_block()
1477 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) in __allocate_data_block()
1478 return -EPERM; in __allocate_data_block()
1480 err = f2fs_get_node_info(sbi, dn->nid, &ni, false); in __allocate_data_block()
1484 dn->data_blkaddr = f2fs_data_blkaddr(dn); in __allocate_data_block()
1485 if (dn->data_blkaddr == NULL_ADDR) { in __allocate_data_block()
1486 err = inc_valid_block_count(sbi, dn->inode, &count); in __allocate_data_block()
1491 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); in __allocate_data_block()
1492 old_blkaddr = dn->data_blkaddr; in __allocate_data_block()
1493 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, in __allocate_data_block()
1500 f2fs_update_data_blkaddr(dn, dn->data_blkaddr); in __allocate_data_block()
1507 f2fs_down_read(&sbi->node_change); in f2fs_map_lock()
1515 f2fs_up_read(&sbi->node_change); in f2fs_map_unlock()
1522 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); in f2fs_get_block_locked()
1526 if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, in f2fs_get_block_locked()
1527 &dn->data_blkaddr)) in f2fs_get_block_locked()
1542 * -ENOENT due to filesystem has been shutdown or cp_error, return in f2fs_map_no_dnode()
1543 * -EIO in that case. in f2fs_map_no_dnode()
1545 if (map->m_may_create && in f2fs_map_no_dnode()
1547 return -EIO; in f2fs_map_no_dnode()
1549 if (map->m_next_pgofs) in f2fs_map_no_dnode()
1550 *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); in f2fs_map_no_dnode()
1551 if (map->m_next_extent) in f2fs_map_no_dnode()
1552 *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); in f2fs_map_no_dnode()
1560 unsigned int maxblocks = map->m_len; in f2fs_map_blocks_cached()
1561 pgoff_t pgoff = (pgoff_t)map->m_lblk; in f2fs_map_blocks_cached()
1567 map->m_pblk = ei.blk + pgoff - ei.fofs; in f2fs_map_blocks_cached()
1568 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); in f2fs_map_blocks_cached()
1569 map->m_flags = F2FS_MAP_MAPPED; in f2fs_map_blocks_cached()
1570 if (map->m_next_extent) in f2fs_map_blocks_cached()
1571 *map->m_next_extent = pgoff + map->m_len; in f2fs_map_blocks_cached()
1576 map->m_pblk, map->m_len); in f2fs_map_blocks_cached()
1579 int bidx = f2fs_target_device_index(sbi, map->m_pblk); in f2fs_map_blocks_cached()
1580 struct f2fs_dev_info *dev = &sbi->devs[bidx]; in f2fs_map_blocks_cached()
1582 map->m_bdev = dev->bdev; in f2fs_map_blocks_cached()
1583 map->m_pblk -= dev->start_blk; in f2fs_map_blocks_cached()
1584 map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); in f2fs_map_blocks_cached()
1586 map->m_bdev = inode->i_sb->s_bdev; in f2fs_map_blocks_cached()
1598 unsigned int maxblocks = map->m_len; in f2fs_map_blocks()
1601 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; in f2fs_map_blocks()
1614 if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) in f2fs_map_blocks()
1617 map->m_bdev = inode->i_sb->s_bdev; in f2fs_map_blocks()
1618 map->m_multidev_dio = in f2fs_map_blocks()
1621 map->m_len = 0; in f2fs_map_blocks()
1622 map->m_flags = 0; in f2fs_map_blocks()
1625 pgofs = (pgoff_t)map->m_lblk; in f2fs_map_blocks()
1629 if (map->m_may_create) in f2fs_map_blocks()
1637 map->m_pblk = 0; in f2fs_map_blocks()
1638 if (err == -ENOENT) in f2fs_map_blocks()
1653 err = -EFSCORRUPTED; in f2fs_map_blocks()
1658 /* use out-place-update for direct IO under LFS mode */ in f2fs_map_blocks()
1659 if (map->m_may_create && in f2fs_map_blocks()
1662 err = -EIO; in f2fs_map_blocks()
1675 err = __allocate_data_block(&dn, map->m_seg_type); in f2fs_map_blocks()
1684 err = -EIO; in f2fs_map_blocks()
1690 map->m_flags |= F2FS_MAP_NEW; in f2fs_map_blocks()
1696 err = -EFSCORRUPTED; in f2fs_map_blocks()
1706 map->m_pblk = 0; in f2fs_map_blocks()
1710 if (map->m_next_pgofs) in f2fs_map_blocks()
1711 *map->m_next_pgofs = pgofs + 1; in f2fs_map_blocks()
1717 if (map->m_next_pgofs) in f2fs_map_blocks()
1718 *map->m_next_pgofs = pgofs + 1; in f2fs_map_blocks()
1726 if (map->m_multidev_dio) in f2fs_map_blocks()
1729 if (map->m_len == 0) { in f2fs_map_blocks()
1732 map->m_flags |= F2FS_MAP_DELALLOC; in f2fs_map_blocks()
1733 map->m_flags |= F2FS_MAP_MAPPED; in f2fs_map_blocks()
1735 map->m_pblk = blkaddr; in f2fs_map_blocks()
1736 map->m_len = 1; in f2fs_map_blocks()
1738 if (map->m_multidev_dio) in f2fs_map_blocks()
1739 map->m_bdev = FDEV(bidx).bdev; in f2fs_map_blocks()
1740 } else if ((map->m_pblk != NEW_ADDR && in f2fs_map_blocks()
1741 blkaddr == (map->m_pblk + ofs)) || in f2fs_map_blocks()
1742 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) || in f2fs_map_blocks()
1744 if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) in f2fs_map_blocks()
1747 map->m_len++; in f2fs_map_blocks()
1765 map->m_len += dn.ofs_in_node - ofs_in_node; in f2fs_map_blocks()
1767 err = -ENOSPC; in f2fs_map_blocks()
1779 if (map->m_flags & F2FS_MAP_MAPPED) { in f2fs_map_blocks()
1780 unsigned int ofs = start_pgofs - map->m_lblk; in f2fs_map_blocks()
1783 start_pgofs, map->m_pblk + ofs, in f2fs_map_blocks()
1784 map->m_len - ofs); in f2fs_map_blocks()
1790 if (map->m_may_create) { in f2fs_map_blocks()
1798 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) { in f2fs_map_blocks()
1804 map->m_pblk, map->m_len); in f2fs_map_blocks()
1806 if (map->m_multidev_dio) { in f2fs_map_blocks()
1807 block_t blk_addr = map->m_pblk; in f2fs_map_blocks()
1809 bidx = f2fs_target_device_index(sbi, map->m_pblk); in f2fs_map_blocks()
1811 map->m_bdev = FDEV(bidx).bdev; in f2fs_map_blocks()
1812 map->m_pblk -= FDEV(bidx).start_blk; in f2fs_map_blocks()
1814 if (map->m_may_create) in f2fs_map_blocks()
1815 f2fs_update_device_state(sbi, inode->i_ino, in f2fs_map_blocks()
1816 blk_addr, map->m_len); in f2fs_map_blocks()
1818 f2fs_bug_on(sbi, blk_addr + map->m_len > in f2fs_map_blocks()
1824 if (map->m_flags & F2FS_MAP_MAPPED) { in f2fs_map_blocks()
1825 unsigned int ofs = start_pgofs - map->m_lblk; in f2fs_map_blocks()
1828 start_pgofs, map->m_pblk + ofs, in f2fs_map_blocks()
1829 map->m_len - ofs); in f2fs_map_blocks()
1831 if (map->m_next_extent) in f2fs_map_blocks()
1832 *map->m_next_extent = pgofs + 1; in f2fs_map_blocks()
1836 if (map->m_may_create) { in f2fs_map_blocks()
1862 map.m_len = last_lblk - map.m_lblk; in f2fs_overwrite_io()
1873 return (bytes >> inode->i_blkbits); in bytes_to_blks()
1878 return (blks << inode->i_blkbits); in blks_to_bytes()
1889 nid_t xnid = F2FS_I(inode)->i_xattr_nid; in f2fs_xattr_fiemap()
1896 inode->i_ino, false); in f2fs_xattr_fiemap()
1898 return -ENOMEM; in f2fs_xattr_fiemap()
1900 err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); in f2fs_xattr_fiemap()
1908 sizeof(__le32) * (DEF_ADDRS_PER_INODE - in f2fs_xattr_fiemap()
1930 return -ENOMEM; in f2fs_xattr_fiemap()
1939 len = inode->i_sb->s_blocksize; in f2fs_xattr_fiemap()
1983 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; in f2fs_fiemap()
1987 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { in f2fs_fiemap()
2001 ret = -EFBIG; in f2fs_fiemap()
2005 if (len > maxbytes || (maxbytes - len) < start) in f2fs_fiemap()
2006 len = maxbytes - start; in f2fs_fiemap()
2008 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { in f2fs_fiemap()
2015 if (ret != -EAGAIN) in f2fs_fiemap()
2023 last_blk = bytes_to_blks(inode, start + len - 1); in f2fs_fiemap()
2034 map.m_len = cluster_size - count_in_cluster; in f2fs_fiemap()
2081 unsigned int appended_blks = cluster_size - in f2fs_fiemap()
2110 ret = -EINTR; in f2fs_fiemap()
2124 return inode->i_sb->s_maxbytes; in f2fs_readpage_limit()
2147 f2fs_readpage_limit(inode) + blocksize - 1); in f2fs_read_single_page()
2157 if ((map->m_flags & F2FS_MAP_MAPPED) && in f2fs_read_single_page()
2158 block_in_file > map->m_lblk && in f2fs_read_single_page()
2159 block_in_file < (map->m_lblk + map->m_len)) in f2fs_read_single_page()
2166 map->m_lblk = block_in_file; in f2fs_read_single_page()
2167 map->m_len = last_block - block_in_file; in f2fs_read_single_page()
2173 if ((map->m_flags & F2FS_MAP_MAPPED)) { in f2fs_read_single_page()
2174 block_nr = map->m_pblk + block_in_file - map->m_lblk; in f2fs_read_single_page()
2179 ret = -EFSCORRUPTED; in f2fs_read_single_page()
2187 if (f2fs_need_verity(inode, page->index) && in f2fs_read_single_page()
2189 ret = -EIO; in f2fs_read_single_page()
2204 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { in f2fs_read_single_page()
2211 is_readahead ? REQ_RAHEAD : 0, page->index, in f2fs_read_single_page()
2244 struct inode *inode = cc->inode; in f2fs_read_multi_pages()
2247 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; in f2fs_read_multi_pages()
2259 f2fs_readpage_limit(inode) + blocksize - 1); in f2fs_read_multi_pages()
2262 for (i = 0; i < cc->cluster_size; i++) { in f2fs_read_multi_pages()
2263 struct page *page = cc->rpages[i]; in f2fs_read_multi_pages()
2267 if ((sector_t)page->index >= last_block_in_file) { in f2fs_read_multi_pages()
2277 cc->rpages[i] = NULL; in f2fs_read_multi_pages()
2278 cc->nr_rpages--; in f2fs_read_multi_pages()
2297 ret = -EIO; in f2fs_read_multi_pages()
2303 for (i = 1; i < cc->cluster_size; i++) { in f2fs_read_multi_pages()
2308 ei.blk + i - 1; in f2fs_read_multi_pages()
2314 ret = -EFAULT; in f2fs_read_multi_pages()
2317 cc->nr_cpages++; in f2fs_read_multi_pages()
2324 if (cc->nr_cpages == 0) { in f2fs_read_multi_pages()
2335 for (i = 0; i < cc->nr_cpages; i++) { in f2fs_read_multi_pages()
2336 struct page *page = dic->cpages[i]; in f2fs_read_multi_pages()
2347 if (atomic_dec_and_test(&dic->remaining_pages)) in f2fs_read_multi_pages()
2354 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { in f2fs_read_multi_pages()
2363 page->index, for_write); in f2fs_read_multi_pages()
2377 ctx->enabled_steps |= STEP_DECOMPRESS; in f2fs_read_multi_pages()
2378 refcount_inc(&dic->refcnt); in f2fs_read_multi_pages()
2395 for (i = 0; i < cc->cluster_size; i++) { in f2fs_read_multi_pages()
2396 if (cc->rpages[i]) { in f2fs_read_multi_pages()
2397 ClearPageUptodate(cc->rpages[i]); in f2fs_read_multi_pages()
2398 unlock_page(cc->rpages[i]); in f2fs_read_multi_pages()
2419 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, in f2fs_mpage_readpages()
2420 .cluster_size = F2FS_I(inode)->i_cluster_size, in f2fs_mpage_readpages()
2442 for (; nr_pages; nr_pages--) { in f2fs_mpage_readpages()
2445 prefetchw(&page->flags); in f2fs_mpage_readpages()
2451 if (!f2fs_cluster_can_merge_page(&cc, page->index)) { in f2fs_mpage_readpages()
2462 page->index >> cc.log_cluster_size) { in f2fs_mpage_readpages()
2466 ret = f2fs_is_compressed_cluster(inode, page->index); in f2fs_mpage_readpages()
2471 page->index >> cc.log_cluster_size; in f2fs_mpage_readpages()
2523 struct page *page = &folio->page; in f2fs_read_data_folio()
2524 struct inode *inode = page_file_mapping(page)->host; in f2fs_read_data_folio()
2525 int ret = -EAGAIN; in f2fs_read_data_folio()
2531 return -EOPNOTSUPP; in f2fs_read_data_folio()
2537 if (ret == -EAGAIN) in f2fs_read_data_folio()
2544 struct inode *inode = rac->mapping->host; in f2fs_readahead()
2560 struct inode *inode = fio->page->mapping->host; in f2fs_encrypt_one_page()
2567 page = fio->compressed_page ? fio->compressed_page : fio->page; in f2fs_encrypt_one_page()
2570 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); in f2fs_encrypt_one_page()
2576 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page, in f2fs_encrypt_one_page()
2578 if (IS_ERR(fio->encrypted_page)) { in f2fs_encrypt_one_page()
2580 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { in f2fs_encrypt_one_page()
2581 f2fs_flush_merged_writes(fio->sbi); in f2fs_encrypt_one_page()
2586 return PTR_ERR(fio->encrypted_page); in f2fs_encrypt_one_page()
2589 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); in f2fs_encrypt_one_page()
2593 page_address(fio->encrypted_page), PAGE_SIZE); in f2fs_encrypt_one_page()
2611 if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) in check_inplace_update_policy()
2614 utilization(sbi) > SM_I(sbi)->min_ipu_util) in check_inplace_update_policy()
2620 if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && in check_inplace_update_policy()
2621 !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) in check_inplace_update_policy()
2629 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in check_inplace_update_policy()
2644 /* if this is cold file, we should overwrite to avoid fragmentation */ in f2fs_should_update_inplace()
2662 if (S_ISDIR(inode->i_mode)) in f2fs_should_update_outplace()
2677 if (page_private_gcing(fio->page)) in f2fs_should_update_outplace()
2679 if (page_private_dummy(fio->page)) in f2fs_should_update_outplace()
2682 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in f2fs_should_update_outplace()
2690 struct inode *inode = fio->page->mapping->host; in need_inplace_update()
2700 struct page *page = fio->page; in f2fs_do_write_data_page()
2701 struct inode *inode = page->mapping->host; in f2fs_do_write_data_page()
2709 set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0); in f2fs_do_write_data_page()
2714 f2fs_lookup_read_extent_cache_block(inode, page->index, in f2fs_do_write_data_page()
2715 &fio->old_blkaddr)) { in f2fs_do_write_data_page()
2716 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2718 f2fs_handle_error(fio->sbi, in f2fs_do_write_data_page()
2720 return -EFSCORRUPTED; in f2fs_do_write_data_page()
2724 fio->need_lock = LOCK_DONE; in f2fs_do_write_data_page()
2728 /* Deadlock due to between page->lock and f2fs_lock_op */ in f2fs_do_write_data_page()
2729 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) in f2fs_do_write_data_page()
2730 return -EAGAIN; in f2fs_do_write_data_page()
2732 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); in f2fs_do_write_data_page()
2736 fio->old_blkaddr = dn.data_blkaddr; in f2fs_do_write_data_page()
2739 if (fio->old_blkaddr == NULL_ADDR) { in f2fs_do_write_data_page()
2745 if (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2746 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2748 err = -EFSCORRUPTED; in f2fs_do_write_data_page()
2749 f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); in f2fs_do_write_data_page()
2755 * it had better in-place writes for updated data. in f2fs_do_write_data_page()
2758 (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2766 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2767 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2771 fscrypt_finalize_bounce_page(&fio->encrypted_page); in f2fs_do_write_data_page()
2777 trace_f2fs_do_write_data_page(fio->page, IPU); in f2fs_do_write_data_page()
2781 if (fio->need_lock == LOCK_RETRY) { in f2fs_do_write_data_page()
2782 if (!f2fs_trylock_op(fio->sbi)) { in f2fs_do_write_data_page()
2783 err = -EAGAIN; in f2fs_do_write_data_page()
2786 fio->need_lock = LOCK_REQ; in f2fs_do_write_data_page()
2789 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); in f2fs_do_write_data_page()
2793 fio->version = ni.version; in f2fs_do_write_data_page()
2801 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) in f2fs_do_write_data_page()
2802 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); in f2fs_do_write_data_page()
2808 if (page->index == 0) in f2fs_do_write_data_page()
2813 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2814 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2826 struct inode *inode = page->mapping->host; in f2fs_write_single_data_page()
2831 loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT; in f2fs_write_single_data_page()
2838 .ino = inode->i_ino, in f2fs_write_single_data_page()
2859 mapping_set_error(page->mapping, -EIO); in f2fs_write_single_data_page()
2864 if (S_ISDIR(inode->i_mode) && in f2fs_write_single_data_page()
2868 /* keep data pages in remount-ro mode */ in f2fs_write_single_data_page()
2877 if (page->index < end_index || in f2fs_write_single_data_page()
2883 * If the offset is out-of-range of file size, in f2fs_write_single_data_page()
2886 offset = i_size & (PAGE_SIZE - 1); in f2fs_write_single_data_page()
2887 if ((page->index >= end_index + 1) || !offset) in f2fs_write_single_data_page()
2896 if (S_ISDIR(inode->i_mode) || quota_inode) { in f2fs_write_single_data_page()
2903 f2fs_down_read(&sbi->node_write); in f2fs_write_single_data_page()
2909 f2fs_up_read(&sbi->node_write); in f2fs_write_single_data_page()
2914 if (!wbc->for_reclaim) in f2fs_write_single_data_page()
2921 err = -EAGAIN; in f2fs_write_single_data_page()
2928 if (err == -EAGAIN) { in f2fs_write_single_data_page()
2930 if (err == -EAGAIN) { in f2fs_write_single_data_page()
2939 spin_lock(&F2FS_I(inode)->i_size_lock); in f2fs_write_single_data_page()
2940 if (F2FS_I(inode)->last_disk_size < psize) in f2fs_write_single_data_page()
2941 F2FS_I(inode)->last_disk_size = psize; in f2fs_write_single_data_page()
2942 spin_unlock(&F2FS_I(inode)->i_size_lock); in f2fs_write_single_data_page()
2946 if (err && err != -ENOENT) in f2fs_write_single_data_page()
2956 if (wbc->for_reclaim) { in f2fs_write_single_data_page()
2963 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && in f2fs_write_single_data_page()
2964 !F2FS_I(inode)->wb_task && allow_balance) in f2fs_write_single_data_page()
2983 * -> mapping_set_error() -> set_bit(AS_EIO, ...). in f2fs_write_single_data_page()
2987 if (!err || wbc->for_reclaim) in f2fs_write_single_data_page()
2997 struct inode *inode = page->mapping->host; in f2fs_write_data_page()
3003 if (f2fs_is_compressed_cluster(inode, page->index)) { in f2fs_write_data_page()
3016 * This function was copied from write_cache_pages from mm/page-writeback.c.
3017 * The major change is making write step of cold data page separately from
3032 struct inode *inode = mapping->host; in f2fs_write_cache_pages()
3035 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, in f2fs_write_cache_pages()
3036 .cluster_size = F2FS_I(inode)->i_cluster_size, in f2fs_write_cache_pages()
3044 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, in f2fs_write_cache_pages()
3061 if (get_dirty_pages(mapping->host) <= in f2fs_write_cache_pages()
3062 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) in f2fs_write_cache_pages()
3063 set_inode_flag(mapping->host, FI_HOT_DATA); in f2fs_write_cache_pages()
3065 clear_inode_flag(mapping->host, FI_HOT_DATA); in f2fs_write_cache_pages()
3067 if (wbc->range_cyclic) { in f2fs_write_cache_pages()
3068 index = mapping->writeback_index; /* prev offset */ in f2fs_write_cache_pages()
3069 end = -1; in f2fs_write_cache_pages()
3071 index = wbc->range_start >> PAGE_SHIFT; in f2fs_write_cache_pages()
3072 end = wbc->range_end >> PAGE_SHIFT; in f2fs_write_cache_pages()
3073 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) in f2fs_write_cache_pages()
3076 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) in f2fs_write_cache_pages()
3082 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) in f2fs_write_cache_pages()
3105 index = folio->index + idx + 1; in f2fs_write_cache_pages()
3134 folio->index)) { in f2fs_write_cache_pages()
3154 folio->index, &fsdata); in f2fs_write_cache_pages()
3161 fsdata, folio->index, 1) || in f2fs_write_cache_pages()
3171 if (atomic_read(&sbi->wb_sync_req[DATA]) && in f2fs_write_cache_pages()
3172 wbc->sync_mode == WB_SYNC_NONE) { in f2fs_write_cache_pages()
3179 done_index = folio->index; in f2fs_write_cache_pages()
3183 if (unlikely(folio->mapping != mapping)) { in f2fs_write_cache_pages()
3195 if (wbc->sync_mode == WB_SYNC_NONE) in f2fs_write_cache_pages()
3197 f2fs_wait_on_page_writeback(&folio->page, DATA, true, true); in f2fs_write_cache_pages()
3206 f2fs_compress_ctx_add_page(&cc, &folio->page); in f2fs_write_cache_pages()
3210 ret = f2fs_write_single_data_page(&folio->page, in f2fs_write_cache_pages()
3219 wbc->nr_to_write -= submitted; in f2fs_write_cache_pages()
3229 } else if (ret == -EAGAIN) { in f2fs_write_cache_pages()
3231 if (wbc->sync_mode == WB_SYNC_ALL) { in f2fs_write_cache_pages()
3243 if (wbc->nr_to_write <= 0 && in f2fs_write_cache_pages()
3244 wbc->sync_mode == WB_SYNC_NONE) { in f2fs_write_cache_pages()
3260 wbc->nr_to_write -= submitted; in f2fs_write_cache_pages()
3271 end = -1; in f2fs_write_cache_pages()
3274 if (wbc->range_cyclic && !done) in f2fs_write_cache_pages()
3276 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) in f2fs_write_cache_pages()
3277 mapping->writeback_index = done_index; in f2fs_write_cache_pages()
3280 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, in f2fs_write_cache_pages()
3293 if (F2FS_I(inode)->wb_task) in __should_serialize_io()
3296 if (!S_ISREG(inode->i_mode)) in __should_serialize_io()
3303 if (wbc->sync_mode != WB_SYNC_ALL) in __should_serialize_io()
3305 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) in __should_serialize_io()
3314 struct inode *inode = mapping->host; in __f2fs_write_data_pages()
3321 if (!mapping->a_ops->writepage) in __f2fs_write_data_pages()
3325 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) in __f2fs_write_data_pages()
3332 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && in __f2fs_write_data_pages()
3333 wbc->sync_mode == WB_SYNC_NONE && in __f2fs_write_data_pages()
3342 trace_f2fs_writepages(mapping->host, wbc, DATA); in __f2fs_write_data_pages()
3345 if (wbc->sync_mode == WB_SYNC_ALL) in __f2fs_write_data_pages()
3346 atomic_inc(&sbi->wb_sync_req[DATA]); in __f2fs_write_data_pages()
3347 else if (atomic_read(&sbi->wb_sync_req[DATA])) { in __f2fs_write_data_pages()
3349 if (current->plug) in __f2fs_write_data_pages()
3350 blk_finish_plug(current->plug); in __f2fs_write_data_pages()
3355 mutex_lock(&sbi->writepages); in __f2fs_write_data_pages()
3364 mutex_unlock(&sbi->writepages); in __f2fs_write_data_pages()
3366 if (wbc->sync_mode == WB_SYNC_ALL) in __f2fs_write_data_pages()
3367 atomic_dec(&sbi->wb_sync_req[DATA]); in __f2fs_write_data_pages()
3369 * if some pages were truncated, we cannot guarantee its mapping->host in __f2fs_write_data_pages()
3377 wbc->pages_skipped += get_dirty_pages(inode); in __f2fs_write_data_pages()
3378 trace_f2fs_writepages(mapping->host, wbc, DATA); in __f2fs_write_data_pages()
3385 struct inode *inode = mapping->host; in f2fs_write_data_pages()
3388 F2FS_I(inode)->cp_task == current ? in f2fs_write_data_pages()
3399 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ in f2fs_write_failed()
3401 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); in f2fs_write_failed()
3402 filemap_invalidate_lock(inode->i_mapping); in f2fs_write_failed()
3407 filemap_invalidate_unlock(inode->i_mapping); in f2fs_write_failed()
3408 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); in f2fs_write_failed()
3416 struct inode *inode = page->mapping->host; in prepare_write_begin()
3417 pgoff_t index = page->index; in prepare_write_begin()
3444 ipage = f2fs_get_node_page(sbi, inode->i_ino); in prepare_write_begin()
3456 if (inode->i_nlink) in prepare_write_begin()
3502 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino); in __find_data_block()
3532 ipage = f2fs_get_node_page(sbi, inode->i_ino); in __reserve_data_block()
3556 struct inode *inode = page->mapping->host; in prepare_atomic_write_begin()
3557 struct inode *cow_inode = F2FS_I(inode)->cow_inode; in prepare_atomic_write_begin()
3558 pgoff_t index = page->index; in prepare_atomic_write_begin()
3598 struct inode *inode = mapping->host; in f2fs_write_begin()
3610 err = -ENOSPC; in f2fs_write_begin()
3617 * lock_page(page #0) -> lock_page(inode_page) in f2fs_write_begin()
3653 err = -ENOMEM; in f2fs_write_begin()
3675 if (page->mapping != mapping) { in f2fs_write_begin()
3687 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && in f2fs_write_begin()
3699 err = -EFSCORRUPTED; in f2fs_write_begin()
3704 F2FS_I(inode)->cow_inode : inode, page, in f2fs_write_begin()
3710 if (unlikely(page->mapping != mapping)) { in f2fs_write_begin()
3715 err = -EIO; in f2fs_write_begin()
3732 struct inode *inode = page->mapping->host; in f2fs_write_end()
3751 f2fs_compress_write_end(inode, fsdata, page->index, copied); in f2fs_write_end()
3770 f2fs_i_size_write(F2FS_I(inode)->cow_inode, in f2fs_write_end()
3781 struct inode *inode = folio->mapping->host; in f2fs_invalidate_folio()
3784 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && in f2fs_invalidate_folio()
3789 if (inode->i_ino == F2FS_META_INO(sbi)) { in f2fs_invalidate_folio()
3791 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { in f2fs_invalidate_folio()
3798 clear_page_private_all(&folio->page); in f2fs_invalidate_folio()
3807 clear_page_private_all(&folio->page); in f2fs_release_folio()
3814 struct inode *inode = mapping->host; in f2fs_dirty_data_folio()
3816 trace_f2fs_set_page_dirty(&folio->page, DATA); in f2fs_dirty_data_folio()
3837 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); in f2fs_bmap_compress()
3845 dn.ofs_in_node += block - start_idx; in f2fs_bmap_compress()
3861 struct inode *inode = mapping->host; in f2fs_bmap()
3905 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); in f2fs_migrate_blocks()
3906 filemap_invalidate_lock(inode->i_mapping); in f2fs_migrate_blocks()
3912 f2fs_down_write(&sbi->pin_sem); in f2fs_migrate_blocks()
3926 f2fs_up_write(&sbi->pin_sem); in f2fs_migrate_blocks()
3937 ret = filemap_fdatawrite(inode->i_mapping); in f2fs_migrate_blocks()
3939 f2fs_up_write(&sbi->pin_sem); in f2fs_migrate_blocks()
3950 filemap_invalidate_unlock(inode->i_mapping); in f2fs_migrate_blocks()
3951 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); in f2fs_migrate_blocks()
3959 struct address_space *mapping = swap_file->f_mapping; in check_swap_activate()
3960 struct inode *inode = mapping->host; in check_swap_activate()
3965 sector_t lowest_pblock = -1; in check_swap_activate()
3970 unsigned int sec_blks_mask = BLKS_PER_SEC(sbi) - 1; in check_swap_activate()
3981 while (cur_lblock < last_lblock && cur_lblock < sis->max) { in check_swap_activate()
3988 map.m_len = last_lblock - cur_lblock; in check_swap_activate()
4001 ret = -EINVAL; in check_swap_activate()
4008 if ((pblock - SM_I(sbi)->main_blkaddr) & sec_blks_mask || in check_swap_activate()
4013 if (cur_lblock + nr_pblocks > sis->max) in check_swap_activate()
4014 nr_pblocks -= blks_per_sec; in check_swap_activate()
4030 if (cur_lblock + nr_pblocks >= sis->max) in check_swap_activate()
4031 nr_pblocks = sis->max - cur_lblock; in check_swap_activate()
4036 if (pblock + nr_pblocks - 1 > highest_pblock) in check_swap_activate()
4037 highest_pblock = pblock + nr_pblocks - 1; in check_swap_activate()
4041 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks in check_swap_activate()
4050 *span = 1 + highest_pblock - lowest_pblock; in check_swap_activate()
4053 sis->max = cur_lblock; in check_swap_activate()
4054 sis->pages = cur_lblock - 1; in check_swap_activate()
4055 sis->highest_bit = cur_lblock - 1; in check_swap_activate()
4069 if (!S_ISREG(inode->i_mode)) in f2fs_swap_activate()
4070 return -EINVAL; in f2fs_swap_activate()
4072 if (f2fs_readonly(F2FS_I_SB(inode)->sb)) in f2fs_swap_activate()
4073 return -EROFS; in f2fs_swap_activate()
4078 return -EINVAL; in f2fs_swap_activate()
4086 return -EINVAL; in f2fs_swap_activate()
4111 return -EOPNOTSUPP; in f2fs_swap_activate()
4140 xa_lock_irqsave(&mapping->i_pages, flags); in f2fs_clear_page_cache_dirty_tag()
4141 __xa_clear_mark(&mapping->i_pages, page_index(page), in f2fs_clear_page_cache_dirty_tag()
4143 xa_unlock_irqrestore(&mapping->i_pages, flags); in f2fs_clear_page_cache_dirty_tag()
4163 return -ENOMEM; in f2fs_init_post_read_processing()
4179 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", in f2fs_init_post_read_wq()
4182 return sbi->post_read_wq ? 0 : -ENOMEM; in f2fs_init_post_read_wq()
4187 if (sbi->post_read_wq) in f2fs_destroy_post_read_wq()
4188 destroy_workqueue(sbi->post_read_wq); in f2fs_destroy_post_read_wq()
4195 return bio_entry_slab ? 0 : -ENOMEM; in f2fs_init_bio_entry_cache()
4212 map.m_len = bytes_to_blks(inode, offset + length - 1) - map.m_lblk + 1; in f2fs_iomap_begin()
4214 map.m_seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint); in f2fs_iomap_begin()
4222 iomap->offset = blks_to_bytes(inode, map.m_lblk); in f2fs_iomap_begin()
4236 return -EINVAL; in f2fs_iomap_begin()
4238 return -EINVAL; in f2fs_iomap_begin()
4241 iomap->length = blks_to_bytes(inode, map.m_len); in f2fs_iomap_begin()
4242 iomap->type = IOMAP_MAPPED; in f2fs_iomap_begin()
4243 iomap->flags |= IOMAP_F_MERGED; in f2fs_iomap_begin()
4244 iomap->bdev = map.m_bdev; in f2fs_iomap_begin()
4245 iomap->addr = blks_to_bytes(inode, map.m_pblk); in f2fs_iomap_begin()
4248 return -ENOTBLK; in f2fs_iomap_begin()
4249 iomap->length = blks_to_bytes(inode, next_pgofs) - in f2fs_iomap_begin()
4250 iomap->offset; in f2fs_iomap_begin()
4251 iomap->type = IOMAP_HOLE; in f2fs_iomap_begin()
4252 iomap->addr = IOMAP_NULL_ADDR; in f2fs_iomap_begin()
4256 iomap->flags |= IOMAP_F_NEW; in f2fs_iomap_begin()
4257 if ((inode->i_state & I_DIRTY_DATASYNC) || in f2fs_iomap_begin()
4259 iomap->flags |= IOMAP_F_DIRTY; in f2fs_iomap_begin()