Lines Matching refs:sbi
169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) in f2fs_need_SSR() argument
171 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); in f2fs_need_SSR()
172 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); in f2fs_need_SSR()
173 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); in f2fs_need_SSR()
175 if (f2fs_lfs_mode(sbi)) in f2fs_need_SSR()
177 if (sbi->gc_mode == GC_URGENT_HIGH) in f2fs_need_SSR()
179 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in f2fs_need_SSR()
182 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + in f2fs_need_SSR()
183 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); in f2fs_need_SSR()
213 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __revoke_inmem_pages() local
256 err = f2fs_get_node_info(sbi, dn.nid, &ni); in __revoke_inmem_pages()
263 f2fs_invalidate_blocks(sbi, dn.data_blkaddr); in __revoke_inmem_pages()
266 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, in __revoke_inmem_pages()
287 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) in f2fs_drop_inmem_pages_all() argument
289 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; in f2fs_drop_inmem_pages_all()
292 unsigned int count = sbi->atomic_files; in f2fs_drop_inmem_pages_all()
295 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
297 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
304 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
327 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_drop_inmem_pages() local
335 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages()
340 sbi->atomic_files--; in f2fs_drop_inmem_pages()
342 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages()
356 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_drop_inmem_page() local
360 f2fs_bug_on(sbi, !page_private_atomic(page)); in f2fs_drop_inmem_page()
368 f2fs_bug_on(sbi, list_empty(head) || cur->page != page); in f2fs_drop_inmem_page()
372 dec_page_count(sbi, F2FS_INMEM_PAGES); in f2fs_drop_inmem_page()
387 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __f2fs_commit_inmem_pages() local
391 .sbi = sbi, in __f2fs_commit_inmem_pages()
443 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); in __f2fs_commit_inmem_pages()
470 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_commit_inmem_pages() local
474 f2fs_balance_fs(sbi, true); in f2fs_commit_inmem_pages()
478 f2fs_lock_op(sbi); in f2fs_commit_inmem_pages()
487 f2fs_unlock_op(sbi); in f2fs_commit_inmem_pages()
497 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) in f2fs_balance_fs() argument
499 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { in f2fs_balance_fs()
500 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); in f2fs_balance_fs()
501 f2fs_stop_checkpoint(sbi, false); in f2fs_balance_fs()
505 if (need && excess_cached_nats(sbi)) in f2fs_balance_fs()
506 f2fs_balance_fs_bg(sbi, false); in f2fs_balance_fs()
508 if (!f2fs_is_checkpoint_ready(sbi)) in f2fs_balance_fs()
515 if (has_not_enough_free_secs(sbi, 0, 0)) { in f2fs_balance_fs()
516 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && in f2fs_balance_fs()
517 sbi->gc_thread->f2fs_gc_task) { in f2fs_balance_fs()
520 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, in f2fs_balance_fs()
522 wake_up(&sbi->gc_thread->gc_wait_queue_head); in f2fs_balance_fs()
524 finish_wait(&sbi->gc_thread->fggc_wq, &wait); in f2fs_balance_fs()
526 down_write(&sbi->gc_lock); in f2fs_balance_fs()
527 f2fs_gc(sbi, false, false, false, NULL_SEGNO); in f2fs_balance_fs()
532 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) in f2fs_balance_fs_bg() argument
534 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) in f2fs_balance_fs_bg()
538 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE)) in f2fs_balance_fs_bg()
539 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); in f2fs_balance_fs_bg()
542 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) in f2fs_balance_fs_bg()
543 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); in f2fs_balance_fs_bg()
545 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) in f2fs_balance_fs_bg()
546 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); in f2fs_balance_fs_bg()
548 f2fs_build_free_nids(sbi, false, false); in f2fs_balance_fs_bg()
550 if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) || in f2fs_balance_fs_bg()
551 excess_prefree_segs(sbi)) in f2fs_balance_fs_bg()
555 if (is_inflight_io(sbi, REQ_TIME) || in f2fs_balance_fs_bg()
556 (!f2fs_time_over(sbi, REQ_TIME) && rwsem_is_locked(&sbi->cp_rwsem))) in f2fs_balance_fs_bg()
560 if (f2fs_time_over(sbi, CP_TIME)) in f2fs_balance_fs_bg()
564 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) || in f2fs_balance_fs_bg()
565 f2fs_available_free_memory(sbi, INO_ENTRIES)) in f2fs_balance_fs_bg()
569 if (test_opt(sbi, DATA_FLUSH) && from_bg) { in f2fs_balance_fs_bg()
572 mutex_lock(&sbi->flush_lock); in f2fs_balance_fs_bg()
575 f2fs_sync_dirty_inodes(sbi, FILE_INODE); in f2fs_balance_fs_bg()
578 mutex_unlock(&sbi->flush_lock); in f2fs_balance_fs_bg()
580 f2fs_sync_fs(sbi->sb, true); in f2fs_balance_fs_bg()
581 stat_inc_bg_cp_count(sbi->stat_info); in f2fs_balance_fs_bg()
584 static int __submit_flush_wait(struct f2fs_sb_info *sbi, in __submit_flush_wait() argument
589 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), in __submit_flush_wait()
590 test_opt(sbi, FLUSH_MERGE), ret); in __submit_flush_wait()
594 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) in submit_flush_wait() argument
599 if (!f2fs_is_multi_device(sbi)) in submit_flush_wait()
600 return __submit_flush_wait(sbi, sbi->sb->s_bdev); in submit_flush_wait()
602 for (i = 0; i < sbi->s_ndevs; i++) { in submit_flush_wait()
603 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) in submit_flush_wait()
605 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in submit_flush_wait()
614 struct f2fs_sb_info *sbi = data; in issue_flush_thread() local
615 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in issue_flush_thread()
630 ret = submit_flush_wait(sbi, cmd->ino); in issue_flush_thread()
646 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) in f2fs_issue_flush() argument
648 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in f2fs_issue_flush()
652 if (test_opt(sbi, NOBARRIER)) in f2fs_issue_flush()
655 if (!test_opt(sbi, FLUSH_MERGE)) { in f2fs_issue_flush()
657 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
664 f2fs_is_multi_device(sbi)) { in f2fs_issue_flush()
665 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
700 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
717 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) in f2fs_create_flush_cmd_control() argument
719 dev_t dev = sbi->sb->s_bdev->bd_dev; in f2fs_create_flush_cmd_control()
723 if (SM_I(sbi)->fcc_info) { in f2fs_create_flush_cmd_control()
724 fcc = SM_I(sbi)->fcc_info; in f2fs_create_flush_cmd_control()
730 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); in f2fs_create_flush_cmd_control()
737 SM_I(sbi)->fcc_info = fcc; in f2fs_create_flush_cmd_control()
738 if (!test_opt(sbi, FLUSH_MERGE)) in f2fs_create_flush_cmd_control()
742 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, in f2fs_create_flush_cmd_control()
747 SM_I(sbi)->fcc_info = NULL; in f2fs_create_flush_cmd_control()
754 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) in f2fs_destroy_flush_cmd_control() argument
756 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in f2fs_destroy_flush_cmd_control()
766 SM_I(sbi)->fcc_info = NULL; in f2fs_destroy_flush_cmd_control()
770 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) in f2fs_flush_device_cache() argument
774 if (!f2fs_is_multi_device(sbi)) in f2fs_flush_device_cache()
777 if (test_opt(sbi, NOBARRIER)) in f2fs_flush_device_cache()
780 for (i = 1; i < sbi->s_ndevs; i++) { in f2fs_flush_device_cache()
783 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) in f2fs_flush_device_cache()
787 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in f2fs_flush_device_cache()
794 f2fs_stop_checkpoint(sbi, false); in f2fs_flush_device_cache()
798 spin_lock(&sbi->dev_lock); in f2fs_flush_device_cache()
799 f2fs_clear_bit(i, (char *)&sbi->dirty_device); in f2fs_flush_device_cache()
800 spin_unlock(&sbi->dev_lock); in f2fs_flush_device_cache()
806 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __locate_dirty_segment() argument
809 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __locate_dirty_segment()
812 if (IS_CURSEG(sbi, segno)) in __locate_dirty_segment()
819 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __locate_dirty_segment()
823 f2fs_bug_on(sbi, 1); in __locate_dirty_segment()
829 if (__is_large_section(sbi)) { in __locate_dirty_segment()
830 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); in __locate_dirty_segment()
832 get_valid_blocks(sbi, segno, true); in __locate_dirty_segment()
834 f2fs_bug_on(sbi, unlikely(!valid_blocks || in __locate_dirty_segment()
835 valid_blocks == BLKS_PER_SEC(sbi))); in __locate_dirty_segment()
837 if (!IS_CURSEC(sbi, secno)) in __locate_dirty_segment()
843 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __remove_dirty_segment() argument
846 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __remove_dirty_segment()
853 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __remove_dirty_segment()
859 valid_blocks = get_valid_blocks(sbi, segno, true); in __remove_dirty_segment()
861 clear_bit(GET_SEC_FROM_SEG(sbi, segno), in __remove_dirty_segment()
864 clear_bit(segno, SIT_I(sbi)->invalid_segmap); in __remove_dirty_segment()
867 if (__is_large_section(sbi)) { in __remove_dirty_segment()
868 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); in __remove_dirty_segment()
871 valid_blocks == BLKS_PER_SEC(sbi)) { in __remove_dirty_segment()
876 if (!IS_CURSEC(sbi, secno)) in __remove_dirty_segment()
887 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) in locate_dirty_segment() argument
889 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in locate_dirty_segment()
893 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) in locate_dirty_segment()
896 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); in locate_dirty_segment()
899 valid_blocks = get_valid_blocks(sbi, segno, false); in locate_dirty_segment()
900 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); in locate_dirty_segment()
902 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || in locate_dirty_segment()
904 __locate_dirty_segment(sbi, segno, PRE); in locate_dirty_segment()
905 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
907 __locate_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
910 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
917 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) in f2fs_dirty_to_prefree() argument
919 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_dirty_to_prefree()
923 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in f2fs_dirty_to_prefree()
924 if (get_valid_blocks(sbi, segno, false)) in f2fs_dirty_to_prefree()
926 if (IS_CURSEG(sbi, segno)) in f2fs_dirty_to_prefree()
928 __locate_dirty_segment(sbi, segno, PRE); in f2fs_dirty_to_prefree()
929 __remove_dirty_segment(sbi, segno, DIRTY); in f2fs_dirty_to_prefree()
934 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) in f2fs_get_unusable_blocks() argument
937 (overprovision_segments(sbi) - reserved_segments(sbi)); in f2fs_get_unusable_blocks()
938 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg; in f2fs_get_unusable_blocks()
939 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_get_unusable_blocks()
946 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in f2fs_get_unusable_blocks()
947 se = get_seg_entry(sbi, segno); in f2fs_get_unusable_blocks()
949 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - in f2fs_get_unusable_blocks()
952 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - in f2fs_get_unusable_blocks()
963 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) in f2fs_disable_cp_again() argument
966 (overprovision_segments(sbi) - reserved_segments(sbi)); in f2fs_disable_cp_again()
967 if (unusable > F2FS_OPTION(sbi).unusable_cap) in f2fs_disable_cp_again()
969 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && in f2fs_disable_cp_again()
970 dirty_segments(sbi) > ovp_hole_segs) in f2fs_disable_cp_again()
976 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) in get_free_segment() argument
978 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in get_free_segment()
982 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in get_free_segment()
983 if (get_valid_blocks(sbi, segno, false)) in get_free_segment()
985 if (get_ckpt_valid_blocks(sbi, segno, false)) in get_free_segment()
994 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, in __create_discard_cmd() argument
998 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __create_discard_cmd()
1002 f2fs_bug_on(sbi, !len); in __create_discard_cmd()
1026 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, in __attach_discard_cmd() argument
1032 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __attach_discard_cmd()
1035 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); in __attach_discard_cmd()
1058 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, in __remove_discard_cmd() argument
1061 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __remove_discard_cmd()
1073 f2fs_bug_on(sbi, dc->ref); in __remove_discard_cmd()
1081 KERN_INFO, sbi->sb->s_id, in __remove_discard_cmd()
1103 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, in __check_sit_bitmap() argument
1110 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; in __check_sit_bitmap()
1114 segno = GET_SEGNO(sbi, blk); in __check_sit_bitmap()
1115 sentry = get_seg_entry(sbi, segno); in __check_sit_bitmap()
1116 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); in __check_sit_bitmap()
1118 if (end < START_BLOCK(sbi, segno + 1)) in __check_sit_bitmap()
1119 size = GET_BLKOFF_FROM_SEG0(sbi, end); in __check_sit_bitmap()
1124 f2fs_bug_on(sbi, offset != size); in __check_sit_bitmap()
1125 blk = START_BLOCK(sbi, segno + 1); in __check_sit_bitmap()
1130 static void __init_discard_policy(struct f2fs_sb_info *sbi, in __init_discard_policy() argument
1134 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __init_discard_policy()
1153 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) { in __init_discard_policy()
1174 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1178 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, in __submit_discard_cmd() argument
1187 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __submit_discard_cmd()
1197 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) in __submit_discard_cmd()
1225 if (time_to_inject(sbi, FAULT_DISCARD)) { in __submit_discard_cmd()
1226 f2fs_show_injection_info(sbi, FAULT_DISCARD); in __submit_discard_cmd()
1244 f2fs_bug_on(sbi, !bio); in __submit_discard_cmd()
1263 __check_sit_bitmap(sbi, lstart, lstart + len); in __submit_discard_cmd()
1272 f2fs_update_iostat(sbi, FS_DISCARD, 1); in __submit_discard_cmd()
1282 __update_discard_tree_range(sbi, bdev, lstart, start, len); in __submit_discard_cmd()
1287 static void __insert_discard_tree(struct f2fs_sb_info *sbi, in __insert_discard_tree() argument
1293 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __insert_discard_tree()
1304 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, in __insert_discard_tree()
1307 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, in __insert_discard_tree()
1317 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, in __punch_discard_cmd() argument
1320 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __punch_discard_cmd()
1325 __remove_discard_cmd(sbi, dc); in __punch_discard_cmd()
1340 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, in __punch_discard_cmd()
1354 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, in __update_discard_tree_range() argument
1358 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __update_discard_tree_range()
1427 __remove_discard_cmd(sbi, tdc); in __update_discard_tree_range()
1432 __insert_discard_tree(sbi, bdev, di.lstart, di.start, in __update_discard_tree_range()
1445 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, in __queue_discard_cmd() argument
1455 if (f2fs_is_multi_device(sbi)) { in __queue_discard_cmd()
1456 int devi = f2fs_target_device_index(sbi, blkstart); in __queue_discard_cmd()
1460 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); in __queue_discard_cmd()
1461 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); in __queue_discard_cmd()
1462 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); in __queue_discard_cmd()
1466 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, in __issue_discard_cmd_orderly() argument
1469 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd_orderly()
1496 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { in __issue_discard_cmd_orderly()
1502 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd_orderly()
1509 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_orderly()
1525 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1528 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, in __issue_discard_cmd() argument
1531 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd()
1539 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); in __issue_discard_cmd()
1545 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) in __issue_discard_cmd()
1552 return __issue_discard_cmd_orderly(sbi, dpolicy); in __issue_discard_cmd()
1560 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, in __issue_discard_cmd()
1564 f2fs_bug_on(sbi, dc->state != D_PREP); in __issue_discard_cmd()
1567 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) in __issue_discard_cmd()
1571 !is_idle(sbi, DISCARD_TIME)) { in __issue_discard_cmd()
1576 __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd()
1590 __wait_all_discard_cmd(sbi, dpolicy); in __issue_discard_cmd()
1600 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) in __drop_discard_cmd() argument
1602 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __drop_discard_cmd()
1612 f2fs_bug_on(sbi, dc->state != D_PREP); in __drop_discard_cmd()
1613 __remove_discard_cmd(sbi, dc); in __drop_discard_cmd()
1622 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) in f2fs_drop_discard_cmd() argument
1624 __drop_discard_cmd(sbi); in f2fs_drop_discard_cmd()
1627 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, in __wait_one_discard_bio() argument
1630 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_one_discard_bio()
1635 f2fs_bug_on(sbi, dc->state != D_DONE); in __wait_one_discard_bio()
1640 __remove_discard_cmd(sbi, dc); in __wait_one_discard_bio()
1647 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, in __wait_discard_cmd_range() argument
1651 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_discard_cmd_range()
1671 __remove_discard_cmd(sbi, dc); in __wait_discard_cmd_range()
1681 trimmed += __wait_one_discard_bio(sbi, dc); in __wait_discard_cmd_range()
1688 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, in __wait_all_discard_cmd() argument
1695 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); in __wait_all_discard_cmd()
1698 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); in __wait_all_discard_cmd()
1699 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); in __wait_all_discard_cmd()
1700 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); in __wait_all_discard_cmd()
1701 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); in __wait_all_discard_cmd()
1707 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_wait_discard_bio() argument
1709 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_wait_discard_bio()
1718 __punch_discard_cmd(sbi, dc, blkaddr); in f2fs_wait_discard_bio()
1727 __wait_one_discard_bio(sbi, dc); in f2fs_wait_discard_bio()
1730 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) in f2fs_stop_discard_thread() argument
1732 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_stop_discard_thread()
1743 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) in f2fs_issue_discard_timeout() argument
1745 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_issue_discard_timeout()
1749 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, in f2fs_issue_discard_timeout()
1751 __issue_discard_cmd(sbi, &dpolicy); in f2fs_issue_discard_timeout()
1752 dropped = __drop_discard_cmd(sbi); in f2fs_issue_discard_timeout()
1755 __wait_all_discard_cmd(sbi, NULL); in f2fs_issue_discard_timeout()
1757 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); in f2fs_issue_discard_timeout()
1763 struct f2fs_sb_info *sbi = data; in issue_discard_thread() local
1764 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in issue_discard_thread()
1773 if (sbi->gc_mode == GC_URGENT_HIGH || in issue_discard_thread()
1774 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) in issue_discard_thread()
1775 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); in issue_discard_thread()
1777 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, in issue_discard_thread()
1793 __wait_all_discard_cmd(sbi, NULL); in issue_discard_thread()
1797 if (f2fs_readonly(sbi->sb)) in issue_discard_thread()
1801 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { in issue_discard_thread()
1808 sb_start_intwrite(sbi->sb); in issue_discard_thread()
1810 issued = __issue_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1812 __wait_all_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1815 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); in issue_discard_thread()
1822 sb_end_intwrite(sbi->sb); in issue_discard_thread()
1829 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, in __f2fs_issue_discard_zone() argument
1836 if (f2fs_is_multi_device(sbi)) { in __f2fs_issue_discard_zone()
1837 devi = f2fs_target_device_index(sbi, blkstart); in __f2fs_issue_discard_zone()
1840 f2fs_err(sbi, "Invalid block %x", blkstart); in __f2fs_issue_discard_zone()
1847 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { in __f2fs_issue_discard_zone()
1853 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", in __f2fs_issue_discard_zone()
1854 devi, sbi->s_ndevs ? FDEV(devi).path : "", in __f2fs_issue_discard_zone()
1864 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); in __f2fs_issue_discard_zone()
1868 static int __issue_discard_async(struct f2fs_sb_info *sbi, in __issue_discard_async() argument
1872 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) in __issue_discard_async()
1873 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1875 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1878 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, in f2fs_issue_discard() argument
1888 bdev = f2fs_target_device(sbi, blkstart, NULL); in f2fs_issue_discard()
1893 f2fs_target_device(sbi, i, NULL); in f2fs_issue_discard()
1896 err = __issue_discard_async(sbi, bdev, in f2fs_issue_discard()
1906 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); in f2fs_issue_discard()
1907 offset = GET_BLKOFF_FROM_SEG0(sbi, i); in f2fs_issue_discard()
1909 if (f2fs_block_unit_discard(sbi) && in f2fs_issue_discard()
1911 sbi->discard_blks--; in f2fs_issue_discard()
1915 err = __issue_discard_async(sbi, bdev, start, len); in f2fs_issue_discard()
1919 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, in add_discard_addrs() argument
1923 int max_blocks = sbi->blocks_per_seg; in add_discard_addrs()
1924 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); in add_discard_addrs()
1928 unsigned long *dmap = SIT_I(sbi)->tmp_map; in add_discard_addrs()
1932 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; in add_discard_addrs()
1935 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) || in add_discard_addrs()
1936 !f2fs_block_unit_discard(sbi)) in add_discard_addrs()
1940 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || in add_discard_addrs()
1941 SM_I(sbi)->dcc_info->nr_discards >= in add_discard_addrs()
1942 SM_I(sbi)->dcc_info->max_discards) in add_discard_addrs()
1951 while (force || SM_I(sbi)->dcc_info->nr_discards <= in add_discard_addrs()
1952 SM_I(sbi)->dcc_info->max_discards) { in add_discard_addrs()
1968 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); in add_discard_addrs()
1975 SM_I(sbi)->dcc_info->nr_discards += end - start; in add_discard_addrs()
1986 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) in f2fs_release_discard_addrs() argument
1988 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); in f2fs_release_discard_addrs()
1999 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) in set_prefree_as_free_segments() argument
2001 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in set_prefree_as_free_segments()
2005 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) in set_prefree_as_free_segments()
2006 __set_test_and_free(sbi, segno, false); in set_prefree_as_free_segments()
2010 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, in f2fs_clear_prefree_segments() argument
2013 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_clear_prefree_segments()
2016 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_clear_prefree_segments()
2021 bool section_alignment = F2FS_OPTION(sbi).discard_unit == in f2fs_clear_prefree_segments()
2024 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) in f2fs_clear_prefree_segments()
2034 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); in f2fs_clear_prefree_segments()
2035 if (start >= MAIN_SEGS(sbi)) in f2fs_clear_prefree_segments()
2037 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), in f2fs_clear_prefree_segments()
2041 start = rounddown(start, sbi->segs_per_sec); in f2fs_clear_prefree_segments()
2042 end = roundup(end, sbi->segs_per_sec); in f2fs_clear_prefree_segments()
2050 if (!f2fs_realtime_discard_enable(sbi)) in f2fs_clear_prefree_segments()
2057 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) { in f2fs_clear_prefree_segments()
2058 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), in f2fs_clear_prefree_segments()
2059 (end - start) << sbi->log_blocks_per_seg); in f2fs_clear_prefree_segments()
2063 secno = GET_SEC_FROM_SEG(sbi, start); in f2fs_clear_prefree_segments()
2064 start_segno = GET_SEG_FROM_SEC(sbi, secno); in f2fs_clear_prefree_segments()
2065 if (!IS_CURSEC(sbi, secno) && in f2fs_clear_prefree_segments()
2066 !get_valid_blocks(sbi, start, true)) in f2fs_clear_prefree_segments()
2067 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), in f2fs_clear_prefree_segments()
2068 sbi->segs_per_sec << sbi->log_blocks_per_seg); in f2fs_clear_prefree_segments()
2070 start = start_segno + sbi->segs_per_sec; in f2fs_clear_prefree_segments()
2078 if (!f2fs_block_unit_discard(sbi)) in f2fs_clear_prefree_segments()
2089 sbi->blocks_per_seg, cur_pos); in f2fs_clear_prefree_segments()
2092 if (f2fs_sb_has_blkzoned(sbi) || in f2fs_clear_prefree_segments()
2096 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, in f2fs_clear_prefree_segments()
2101 sbi->blocks_per_seg, cur_pos); in f2fs_clear_prefree_segments()
2107 if (cur_pos < sbi->blocks_per_seg) in f2fs_clear_prefree_segments()
2115 wake_up_discard_thread(sbi, false); in f2fs_clear_prefree_segments()
2118 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) in f2fs_start_discard_thread() argument
2120 dev_t dev = sbi->sb->s_bdev->bd_dev; in f2fs_start_discard_thread()
2121 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_start_discard_thread()
2124 if (!f2fs_realtime_discard_enable(sbi)) in f2fs_start_discard_thread()
2127 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, in f2fs_start_discard_thread()
2135 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) in create_discard_cmd_control() argument
2140 if (SM_I(sbi)->dcc_info) { in create_discard_cmd_control()
2141 dcc = SM_I(sbi)->dcc_info; in create_discard_cmd_control()
2145 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); in create_discard_cmd_control()
2150 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) in create_discard_cmd_control()
2151 dcc->discard_granularity = sbi->blocks_per_seg; in create_discard_cmd_control()
2152 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) in create_discard_cmd_control()
2153 dcc->discard_granularity = BLKS_PER_SEC(sbi); in create_discard_cmd_control()
2165 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; in create_discard_cmd_control()
2172 SM_I(sbi)->dcc_info = dcc; in create_discard_cmd_control()
2174 err = f2fs_start_discard_thread(sbi); in create_discard_cmd_control()
2177 SM_I(sbi)->dcc_info = NULL; in create_discard_cmd_control()
2183 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) in destroy_discard_cmd_control() argument
2185 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in destroy_discard_cmd_control()
2190 f2fs_stop_discard_thread(sbi); in destroy_discard_cmd_control()
2197 f2fs_issue_discard_timeout(sbi); in destroy_discard_cmd_control()
2200 SM_I(sbi)->dcc_info = NULL; in destroy_discard_cmd_control()
2203 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) in __mark_sit_entry_dirty() argument
2205 struct sit_info *sit_i = SIT_I(sbi); in __mark_sit_entry_dirty()
2215 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, in __set_sit_entry_type() argument
2218 struct seg_entry *se = get_seg_entry(sbi, segno); in __set_sit_entry_type()
2222 __mark_sit_entry_dirty(sbi, segno); in __set_sit_entry_type()
2225 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, in get_segment_mtime() argument
2228 unsigned int segno = GET_SEGNO(sbi, blkaddr); in get_segment_mtime()
2232 return get_seg_entry(sbi, segno)->mtime; in get_segment_mtime()
2235 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, in update_segment_mtime() argument
2239 unsigned int segno = GET_SEGNO(sbi, blkaddr); in update_segment_mtime()
2240 unsigned long long ctime = get_mtime(sbi, false); in update_segment_mtime()
2246 se = get_seg_entry(sbi, segno); in update_segment_mtime()
2254 if (ctime > SIT_I(sbi)->max_mtime) in update_segment_mtime()
2255 SIT_I(sbi)->max_mtime = ctime; in update_segment_mtime()
2258 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) in update_sit_entry() argument
2268 segno = GET_SEGNO(sbi, blkaddr); in update_sit_entry()
2270 se = get_seg_entry(sbi, segno); in update_sit_entry()
2272 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in update_sit_entry()
2274 f2fs_bug_on(sbi, (new_vblocks < 0 || in update_sit_entry()
2275 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); in update_sit_entry()
2286 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", in update_sit_entry()
2288 f2fs_bug_on(sbi, 1); in update_sit_entry()
2292 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", in update_sit_entry()
2294 f2fs_bug_on(sbi, 1); in update_sit_entry()
2299 if (f2fs_block_unit_discard(sbi) && in update_sit_entry()
2301 sbi->discard_blks--; in update_sit_entry()
2307 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { in update_sit_entry()
2317 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", in update_sit_entry()
2319 f2fs_bug_on(sbi, 1); in update_sit_entry()
2323 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", in update_sit_entry()
2325 f2fs_bug_on(sbi, 1); in update_sit_entry()
2328 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { in update_sit_entry()
2336 spin_lock(&sbi->stat_lock); in update_sit_entry()
2337 sbi->unusable_block_count++; in update_sit_entry()
2338 spin_unlock(&sbi->stat_lock); in update_sit_entry()
2342 if (f2fs_block_unit_discard(sbi) && in update_sit_entry()
2344 sbi->discard_blks++; in update_sit_entry()
2349 __mark_sit_entry_dirty(sbi, segno); in update_sit_entry()
2352 SIT_I(sbi)->written_valid_blocks += del; in update_sit_entry()
2354 if (__is_large_section(sbi)) in update_sit_entry()
2355 get_sec_entry(sbi, segno)->valid_blocks += del; in update_sit_entry()
2358 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) in f2fs_invalidate_blocks() argument
2360 unsigned int segno = GET_SEGNO(sbi, addr); in f2fs_invalidate_blocks()
2361 struct sit_info *sit_i = SIT_I(sbi); in f2fs_invalidate_blocks()
2363 f2fs_bug_on(sbi, addr == NULL_ADDR); in f2fs_invalidate_blocks()
2367 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr); in f2fs_invalidate_blocks()
2368 f2fs_invalidate_compress_page(sbi, addr); in f2fs_invalidate_blocks()
2373 update_segment_mtime(sbi, addr, 0); in f2fs_invalidate_blocks()
2374 update_sit_entry(sbi, addr, -1); in f2fs_invalidate_blocks()
2377 locate_dirty_segment(sbi, segno); in f2fs_invalidate_blocks()
2382 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_is_checkpointed_data() argument
2384 struct sit_info *sit_i = SIT_I(sbi); in f2fs_is_checkpointed_data()
2394 segno = GET_SEGNO(sbi, blkaddr); in f2fs_is_checkpointed_data()
2395 se = get_seg_entry(sbi, segno); in f2fs_is_checkpointed_data()
2396 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in f2fs_is_checkpointed_data()
2409 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, in __add_sum_entry() argument
2412 struct curseg_info *curseg = CURSEG_I(sbi, type); in __add_sum_entry()
2422 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) in f2fs_npages_for_summary_flush() argument
2428 if (sbi->ckpt->alloc_type[i] == SSR) in f2fs_npages_for_summary_flush()
2429 valid_sum_count += sbi->blocks_per_seg; in f2fs_npages_for_summary_flush()
2433 F2FS_CKPT(sbi)->cur_data_blkoff[i]); in f2fs_npages_for_summary_flush()
2435 valid_sum_count += curseg_blkoff(sbi, i); in f2fs_npages_for_summary_flush()
2452 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) in f2fs_get_sum_page() argument
2454 if (unlikely(f2fs_cp_error(sbi))) in f2fs_get_sum_page()
2456 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno)); in f2fs_get_sum_page()
2459 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, in f2fs_update_meta_page() argument
2462 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); in f2fs_update_meta_page()
2469 static void write_sum_page(struct f2fs_sb_info *sbi, in write_sum_page() argument
2472 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); in write_sum_page()
2475 static void write_current_sum_page(struct f2fs_sb_info *sbi, in write_current_sum_page() argument
2478 struct curseg_info *curseg = CURSEG_I(sbi, type); in write_current_sum_page()
2479 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); in write_current_sum_page()
2501 static int is_next_segment_free(struct f2fs_sb_info *sbi, in is_next_segment_free() argument
2505 struct free_segmap_info *free_i = FREE_I(sbi); in is_next_segment_free()
2507 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) in is_next_segment_free()
2516 static void get_new_segment(struct f2fs_sb_info *sbi, in get_new_segment() argument
2519 struct free_segmap_info *free_i = FREE_I(sbi); in get_new_segment()
2521 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; in get_new_segment()
2522 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); in get_new_segment()
2523 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); in get_new_segment()
2531 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { in get_new_segment()
2533 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); in get_new_segment()
2534 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) in get_new_segment()
2538 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); in get_new_segment()
2539 if (secno >= MAIN_SECS(sbi)) { in get_new_segment()
2542 MAIN_SECS(sbi), 0); in get_new_segment()
2543 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); in get_new_segment()
2558 MAIN_SECS(sbi), 0); in get_new_segment()
2559 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); in get_new_segment()
2564 segno = GET_SEG_FROM_SEC(sbi, secno); in get_new_segment()
2565 zoneno = GET_ZONE_FROM_SEC(sbi, secno); in get_new_segment()
2570 if (sbi->secs_per_zone == 1) in get_new_segment()
2581 if (CURSEG_I(sbi, i)->zone == zoneno) in get_new_segment()
2587 hint = zoneno * sbi->secs_per_zone - 1; in get_new_segment()
2591 hint = (zoneno + 1) * sbi->secs_per_zone; in get_new_segment()
2597 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); in get_new_segment()
2598 __set_inuse(sbi, segno); in get_new_segment()
2603 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) in reset_curseg() argument
2605 struct curseg_info *curseg = CURSEG_I(sbi, type); in reset_curseg()
2611 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); in reset_curseg()
2618 sanity_check_seg_type(sbi, seg_type); in reset_curseg()
2624 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); in reset_curseg()
2627 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) in __get_next_segno() argument
2629 struct curseg_info *curseg = CURSEG_I(sbi, type); in __get_next_segno()
2632 sanity_check_seg_type(sbi, seg_type); in __get_next_segno()
2635 if (__is_large_section(sbi)) in __get_next_segno()
2642 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in __get_next_segno()
2645 if (test_opt(sbi, NOHEAP) && in __get_next_segno()
2649 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) in __get_next_segno()
2650 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; in __get_next_segno()
2653 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) in __get_next_segno()
2663 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) in new_curseg() argument
2665 struct curseg_info *curseg = CURSEG_I(sbi, type); in new_curseg()
2671 write_sum_page(sbi, curseg->sum_blk, in new_curseg()
2672 GET_SUM_BLOCK(sbi, segno)); in new_curseg()
2676 if (test_opt(sbi, NOHEAP)) in new_curseg()
2679 segno = __get_next_segno(sbi, type); in new_curseg()
2680 get_new_segment(sbi, &segno, new_sec, dir); in new_curseg()
2682 reset_curseg(sbi, type, 1); in new_curseg()
2686 static int __next_free_blkoff(struct f2fs_sb_info *sbi, in __next_free_blkoff() argument
2689 struct seg_entry *se = get_seg_entry(sbi, segno); in __next_free_blkoff()
2691 unsigned long *target_map = SIT_I(sbi)->tmp_map; in __next_free_blkoff()
2699 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); in __next_free_blkoff()
2707 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, in __refresh_next_blkoff() argument
2712 __next_free_blkoff(sbi, seg->segno, in __refresh_next_blkoff()
2718 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) in f2fs_segment_has_free_slot() argument
2720 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg; in f2fs_segment_has_free_slot()
2727 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush) in change_curseg() argument
2729 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in change_curseg()
2730 struct curseg_info *curseg = CURSEG_I(sbi, type); in change_curseg()
2736 write_sum_page(sbi, curseg->sum_blk, in change_curseg()
2737 GET_SUM_BLOCK(sbi, curseg->segno)); in change_curseg()
2739 __set_test_and_inuse(sbi, new_segno); in change_curseg()
2742 __remove_dirty_segment(sbi, new_segno, PRE); in change_curseg()
2743 __remove_dirty_segment(sbi, new_segno, DIRTY); in change_curseg()
2746 reset_curseg(sbi, type, 1); in change_curseg()
2748 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); in change_curseg()
2750 sum_page = f2fs_get_sum_page(sbi, new_segno); in change_curseg()
2761 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2764 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type, in get_atssr_segment() argument
2768 struct curseg_info *curseg = CURSEG_I(sbi, type); in get_atssr_segment()
2772 if (get_ssr_segment(sbi, type, alloc_mode, age)) { in get_atssr_segment()
2773 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); in get_atssr_segment()
2776 change_curseg(sbi, type, true); in get_atssr_segment()
2780 new_curseg(sbi, type, true); in get_atssr_segment()
2782 stat_inc_seg_type(sbi, curseg); in get_atssr_segment()
2785 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi) in __f2fs_init_atgc_curseg() argument
2787 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); in __f2fs_init_atgc_curseg()
2789 if (!sbi->am.atgc_enabled) in __f2fs_init_atgc_curseg()
2792 down_read(&SM_I(sbi)->curseg_lock); in __f2fs_init_atgc_curseg()
2795 down_write(&SIT_I(sbi)->sentry_lock); in __f2fs_init_atgc_curseg()
2797 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0); in __f2fs_init_atgc_curseg()
2799 up_write(&SIT_I(sbi)->sentry_lock); in __f2fs_init_atgc_curseg()
2802 up_read(&SM_I(sbi)->curseg_lock); in __f2fs_init_atgc_curseg()
2805 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) in f2fs_init_inmem_curseg() argument
2807 __f2fs_init_atgc_curseg(sbi); in f2fs_init_inmem_curseg()
2810 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) in __f2fs_save_inmem_curseg() argument
2812 struct curseg_info *curseg = CURSEG_I(sbi, type); in __f2fs_save_inmem_curseg()
2818 if (get_valid_blocks(sbi, curseg->segno, false)) { in __f2fs_save_inmem_curseg()
2819 write_sum_page(sbi, curseg->sum_blk, in __f2fs_save_inmem_curseg()
2820 GET_SUM_BLOCK(sbi, curseg->segno)); in __f2fs_save_inmem_curseg()
2822 mutex_lock(&DIRTY_I(sbi)->seglist_lock); in __f2fs_save_inmem_curseg()
2823 __set_test_and_free(sbi, curseg->segno, true); in __f2fs_save_inmem_curseg()
2824 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); in __f2fs_save_inmem_curseg()
2830 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) in f2fs_save_inmem_curseg() argument
2832 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); in f2fs_save_inmem_curseg()
2834 if (sbi->am.atgc_enabled) in f2fs_save_inmem_curseg()
2835 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); in f2fs_save_inmem_curseg()
2838 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) in __f2fs_restore_inmem_curseg() argument
2840 struct curseg_info *curseg = CURSEG_I(sbi, type); in __f2fs_restore_inmem_curseg()
2845 if (get_valid_blocks(sbi, curseg->segno, false)) in __f2fs_restore_inmem_curseg()
2848 mutex_lock(&DIRTY_I(sbi)->seglist_lock); in __f2fs_restore_inmem_curseg()
2849 __set_test_and_inuse(sbi, curseg->segno); in __f2fs_restore_inmem_curseg()
2850 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); in __f2fs_restore_inmem_curseg()
2855 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) in f2fs_restore_inmem_curseg() argument
2857 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); in f2fs_restore_inmem_curseg()
2859 if (sbi->am.atgc_enabled) in f2fs_restore_inmem_curseg()
2860 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); in f2fs_restore_inmem_curseg()
2863 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, in get_ssr_segment() argument
2866 struct curseg_info *curseg = CURSEG_I(sbi, type); in get_ssr_segment()
2867 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; in get_ssr_segment()
2873 sanity_check_seg_type(sbi, seg_type); in get_ssr_segment()
2876 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) { in get_ssr_segment()
2903 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) { in get_ssr_segment()
2910 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { in get_ssr_segment()
2911 segno = get_free_segment(sbi); in get_ssr_segment()
2924 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, in allocate_segment_by_default() argument
2927 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_segment_by_default()
2930 new_curseg(sbi, type, true); in allocate_segment_by_default()
2931 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && in allocate_segment_by_default()
2933 new_curseg(sbi, type, false); in allocate_segment_by_default()
2935 is_next_segment_free(sbi, curseg, type) && in allocate_segment_by_default()
2936 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in allocate_segment_by_default()
2937 new_curseg(sbi, type, false); in allocate_segment_by_default()
2938 else if (f2fs_need_SSR(sbi) && in allocate_segment_by_default()
2939 get_ssr_segment(sbi, type, SSR, 0)) in allocate_segment_by_default()
2940 change_curseg(sbi, type, true); in allocate_segment_by_default()
2942 new_curseg(sbi, type, false); in allocate_segment_by_default()
2944 stat_inc_seg_type(sbi, curseg); in allocate_segment_by_default()
2947 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, in f2fs_allocate_segment_for_resize() argument
2950 struct curseg_info *curseg = CURSEG_I(sbi, type); in f2fs_allocate_segment_for_resize()
2953 down_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_segment_for_resize()
2955 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_segment_for_resize()
2957 segno = CURSEG_I(sbi, type)->segno; in f2fs_allocate_segment_for_resize()
2961 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) in f2fs_allocate_segment_for_resize()
2962 change_curseg(sbi, type, true); in f2fs_allocate_segment_for_resize()
2964 new_curseg(sbi, type, true); in f2fs_allocate_segment_for_resize()
2966 stat_inc_seg_type(sbi, curseg); in f2fs_allocate_segment_for_resize()
2968 locate_dirty_segment(sbi, segno); in f2fs_allocate_segment_for_resize()
2970 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_segment_for_resize()
2973 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", in f2fs_allocate_segment_for_resize()
2977 up_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_segment_for_resize()
2980 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type, in __allocate_new_segment() argument
2983 struct curseg_info *curseg = CURSEG_I(sbi, type); in __allocate_new_segment()
2990 get_valid_blocks(sbi, curseg->segno, new_sec)) in __allocate_new_segment()
2993 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) in __allocate_new_segment()
2997 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); in __allocate_new_segment()
2998 locate_dirty_segment(sbi, old_segno); in __allocate_new_segment()
3001 static void __allocate_new_section(struct f2fs_sb_info *sbi, in __allocate_new_section() argument
3004 __allocate_new_segment(sbi, type, true, force); in __allocate_new_section()
3007 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) in f2fs_allocate_new_section() argument
3009 down_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_new_section()
3010 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_section()
3011 __allocate_new_section(sbi, type, force); in f2fs_allocate_new_section()
3012 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_section()
3013 up_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_new_section()
3016 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) in f2fs_allocate_new_segments() argument
3020 down_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_new_segments()
3021 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_segments()
3023 __allocate_new_segment(sbi, i, false, false); in f2fs_allocate_new_segments()
3024 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_segments()
3025 up_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_new_segments()
3032 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, in f2fs_exist_trim_candidates() argument
3038 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_exist_trim_candidates()
3040 if (add_discard_addrs(sbi, cpc, true)) { in f2fs_exist_trim_candidates()
3045 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_exist_trim_candidates()
3051 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, in __issue_discard_cmd_range() argument
3055 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd_range()
3068 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, in __issue_discard_cmd_range()
3093 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd_range()
3099 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_range()
3103 trimmed += __wait_all_discard_cmd(sbi, NULL); in __issue_discard_cmd_range()
3110 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_range()
3123 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) in f2fs_trim_fs() argument
3133 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); in f2fs_trim_fs()
3135 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) in f2fs_trim_fs()
3138 if (end < MAIN_BLKADDR(sbi)) in f2fs_trim_fs()
3141 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { in f2fs_trim_fs()
3142 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); in f2fs_trim_fs()
3147 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); in f2fs_trim_fs()
3148 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : in f2fs_trim_fs()
3149 GET_SEGNO(sbi, end); in f2fs_trim_fs()
3151 start_segno = rounddown(start_segno, sbi->segs_per_sec); in f2fs_trim_fs()
3152 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; in f2fs_trim_fs()
3160 if (sbi->discard_blks == 0) in f2fs_trim_fs()
3163 down_write(&sbi->gc_lock); in f2fs_trim_fs()
3164 err = f2fs_write_checkpoint(sbi, &cpc); in f2fs_trim_fs()
3165 up_write(&sbi->gc_lock); in f2fs_trim_fs()
3175 if (f2fs_realtime_discard_enable(sbi)) in f2fs_trim_fs()
3178 start_block = START_BLOCK(sbi, start_segno); in f2fs_trim_fs()
3179 end_block = START_BLOCK(sbi, end_segno + 1); in f2fs_trim_fs()
3181 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); in f2fs_trim_fs()
3182 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, in f2fs_trim_fs()
3185 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, in f2fs_trim_fs()
3193 static bool __has_curseg_space(struct f2fs_sb_info *sbi, in __has_curseg_space() argument
3196 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi, in __has_curseg_space()
3273 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, in f2fs_io_type_to_rw_hint() argument
3276 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { in f2fs_io_type_to_rw_hint()
3287 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { in f2fs_io_type_to_rw_hint()
3341 if (fio->sbi->am.atgc_enabled && in __get_segment_type_6()
3343 (fio->sbi->gc_mode != GC_URGENT_HIGH)) in __get_segment_type_6()
3368 switch (F2FS_OPTION(fio->sbi).active_logs) { in __get_segment_type()
3379 f2fs_bug_on(fio->sbi, true); in __get_segment_type()
3391 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, in f2fs_allocate_data_block() argument
3396 struct sit_info *sit_i = SIT_I(sbi); in f2fs_allocate_data_block()
3397 struct curseg_info *curseg = CURSEG_I(sbi, type); in f2fs_allocate_data_block()
3402 down_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_data_block()
3408 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); in f2fs_allocate_data_block()
3409 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); in f2fs_allocate_data_block()
3410 sanity_check_seg_type(sbi, se->type); in f2fs_allocate_data_block()
3411 f2fs_bug_on(sbi, IS_NODESEG(se->type)); in f2fs_allocate_data_block()
3413 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); in f2fs_allocate_data_block()
3415 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg); in f2fs_allocate_data_block()
3417 f2fs_wait_discard_bio(sbi, *new_blkaddr); in f2fs_allocate_data_block()
3424 __add_sum_entry(sbi, type, sum); in f2fs_allocate_data_block()
3426 __refresh_next_blkoff(sbi, curseg); in f2fs_allocate_data_block()
3428 stat_inc_block_count(sbi, curseg); in f2fs_allocate_data_block()
3431 old_mtime = get_segment_mtime(sbi, old_blkaddr); in f2fs_allocate_data_block()
3433 update_segment_mtime(sbi, old_blkaddr, 0); in f2fs_allocate_data_block()
3436 update_segment_mtime(sbi, *new_blkaddr, old_mtime); in f2fs_allocate_data_block()
3442 update_sit_entry(sbi, *new_blkaddr, 1); in f2fs_allocate_data_block()
3443 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) in f2fs_allocate_data_block()
3444 update_sit_entry(sbi, old_blkaddr, -1); in f2fs_allocate_data_block()
3446 if (!__has_curseg_space(sbi, curseg)) { in f2fs_allocate_data_block()
3448 get_atssr_segment(sbi, type, se->type, in f2fs_allocate_data_block()
3451 sit_i->s_ops->allocate_segment(sbi, type, false); in f2fs_allocate_data_block()
3458 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in f2fs_allocate_data_block()
3459 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); in f2fs_allocate_data_block()
3464 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); in f2fs_allocate_data_block()
3466 f2fs_inode_chksum_set(sbi, page); in f2fs_allocate_data_block()
3472 if (F2FS_IO_ALIGNED(sbi)) in f2fs_allocate_data_block()
3477 io = sbi->write_io[fio->type] + fio->temp; in f2fs_allocate_data_block()
3485 up_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_data_block()
3490 struct f2fs_sb_info *sbi = fio->sbi; in update_device_state() local
3493 if (!f2fs_is_multi_device(sbi)) in update_device_state()
3496 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); in update_device_state()
3499 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); in update_device_state()
3502 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { in update_device_state()
3503 spin_lock(&sbi->dev_lock); in update_device_state()
3504 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); in update_device_state()
3505 spin_unlock(&sbi->dev_lock); in update_device_state()
3512 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA); in do_write_page()
3515 down_read(&fio->sbi->io_order_lock); in do_write_page()
3517 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, in do_write_page()
3519 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) { in do_write_page()
3520 invalidate_mapping_pages(META_MAPPING(fio->sbi), in do_write_page()
3522 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr); in do_write_page()
3535 up_read(&fio->sbi->io_order_lock); in do_write_page()
3538 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, in f2fs_do_write_meta_page() argument
3542 .sbi = sbi, in f2fs_do_write_meta_page()
3554 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) in f2fs_do_write_meta_page()
3561 stat_inc_meta_count(sbi, page->index); in f2fs_do_write_meta_page()
3562 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); in f2fs_do_write_meta_page()
3572 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_do_write_node_page()
3578 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_outplace_write_data() local
3581 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); in f2fs_outplace_write_data()
3586 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_outplace_write_data()
3592 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_inplace_write_data() local
3599 segno = GET_SEGNO(sbi, fio->new_blkaddr); in f2fs_inplace_write_data()
3601 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { in f2fs_inplace_write_data()
3602 set_sbi_flag(sbi, SBI_NEED_FSCK); in f2fs_inplace_write_data()
3603 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", in f2fs_inplace_write_data()
3609 if (f2fs_cp_error(sbi)) { in f2fs_inplace_write_data()
3614 stat_inc_inplace_blocks(fio->sbi); in f2fs_inplace_write_data()
3616 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE))) in f2fs_inplace_write_data()
3622 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_inplace_write_data()
3637 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, in __f2fs_get_curseg() argument
3643 if (CURSEG_I(sbi, i)->segno == segno) in __f2fs_get_curseg()
3649 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, in f2fs_do_replace_block() argument
3654 struct sit_info *sit_i = SIT_I(sbi); in f2fs_do_replace_block()
3662 segno = GET_SEGNO(sbi, new_blkaddr); in f2fs_do_replace_block()
3663 se = get_seg_entry(sbi, segno); in f2fs_do_replace_block()
3666 down_write(&SM_I(sbi)->curseg_lock); in f2fs_do_replace_block()
3670 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { in f2fs_do_replace_block()
3677 if (IS_CURSEG(sbi, segno)) { in f2fs_do_replace_block()
3679 type = __f2fs_get_curseg(sbi, segno); in f2fs_do_replace_block()
3680 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); in f2fs_do_replace_block()
3686 f2fs_bug_on(sbi, !IS_DATASEG(type)); in f2fs_do_replace_block()
3687 curseg = CURSEG_I(sbi, type); in f2fs_do_replace_block()
3699 change_curseg(sbi, type, true); in f2fs_do_replace_block()
3702 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); in f2fs_do_replace_block()
3703 __add_sum_entry(sbi, type, sum); in f2fs_do_replace_block()
3707 update_segment_mtime(sbi, new_blkaddr, 0); in f2fs_do_replace_block()
3708 update_sit_entry(sbi, new_blkaddr, 1); in f2fs_do_replace_block()
3710 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { in f2fs_do_replace_block()
3711 invalidate_mapping_pages(META_MAPPING(sbi), in f2fs_do_replace_block()
3713 f2fs_invalidate_compress_page(sbi, old_blkaddr); in f2fs_do_replace_block()
3715 update_segment_mtime(sbi, old_blkaddr, 0); in f2fs_do_replace_block()
3716 update_sit_entry(sbi, old_blkaddr, -1); in f2fs_do_replace_block()
3719 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in f2fs_do_replace_block()
3720 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); in f2fs_do_replace_block()
3722 locate_dirty_segment(sbi, old_cursegno); in f2fs_do_replace_block()
3727 change_curseg(sbi, type, true); in f2fs_do_replace_block()
3735 up_write(&SM_I(sbi)->curseg_lock); in f2fs_do_replace_block()
3738 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, in f2fs_replace_block() argument
3747 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, in f2fs_replace_block()
3757 struct f2fs_sb_info *sbi = F2FS_P_SB(page); in f2fs_wait_on_page_writeback() local
3760 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); in f2fs_wait_on_page_writeback()
3762 f2fs_submit_merged_ipu_write(sbi, NULL, page); in f2fs_wait_on_page_writeback()
3765 f2fs_bug_on(sbi, locked && PageWriteback(page)); in f2fs_wait_on_page_writeback()
3774 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_wait_on_block_writeback() local
3783 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); in f2fs_wait_on_block_writeback()
3799 static int read_compacted_summaries(struct f2fs_sb_info *sbi) in read_compacted_summaries() argument
3801 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_compacted_summaries()
3808 start = start_sum_block(sbi); in read_compacted_summaries()
3810 page = f2fs_get_meta_page(sbi, start++); in read_compacted_summaries()
3816 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in read_compacted_summaries()
3820 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in read_compacted_summaries()
3829 seg_i = CURSEG_I(sbi, i); in read_compacted_summaries()
3833 reset_curseg(sbi, i, 0); in read_compacted_summaries()
3838 blk_off = sbi->blocks_per_seg; in read_compacted_summaries()
3853 page = f2fs_get_meta_page(sbi, start++); in read_compacted_summaries()
3864 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) in read_normal_summaries() argument
3866 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_normal_summaries()
3880 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3881 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); in read_normal_summaries()
3883 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); in read_normal_summaries()
3889 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3890 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, in read_normal_summaries()
3893 blk_addr = GET_SUM_BLOCK(sbi, segno); in read_normal_summaries()
3896 new = f2fs_get_meta_page(sbi, blk_addr); in read_normal_summaries()
3902 if (__exist_node_summaries(sbi)) { in read_normal_summaries()
3906 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { in read_normal_summaries()
3911 err = f2fs_restore_node_summary(sbi, segno, sum); in read_normal_summaries()
3918 curseg = CURSEG_I(sbi, type); in read_normal_summaries()
3929 reset_curseg(sbi, type, 0); in read_normal_summaries()
3938 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) in restore_curseg_summaries() argument
3940 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; in restore_curseg_summaries()
3941 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; in restore_curseg_summaries()
3945 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { in restore_curseg_summaries()
3946 int npages = f2fs_npages_for_summary_flush(sbi, true); in restore_curseg_summaries()
3949 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, in restore_curseg_summaries()
3953 err = read_compacted_summaries(sbi); in restore_curseg_summaries()
3959 if (__exist_node_summaries(sbi)) in restore_curseg_summaries()
3960 f2fs_ra_meta_pages(sbi, in restore_curseg_summaries()
3961 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), in restore_curseg_summaries()
3965 err = read_normal_summaries(sbi, type); in restore_curseg_summaries()
3973 f2fs_err(sbi, "invalid journal entries nats %u sits %u", in restore_curseg_summaries()
3981 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) in write_compacted_summaries() argument
3990 page = f2fs_grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
3995 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in write_compacted_summaries()
4000 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in write_compacted_summaries()
4008 seg_i = CURSEG_I(sbi, i); in write_compacted_summaries()
4009 if (sbi->ckpt->alloc_type[i] == SSR) in write_compacted_summaries()
4010 blkoff = sbi->blocks_per_seg; in write_compacted_summaries()
4012 blkoff = curseg_blkoff(sbi, i); in write_compacted_summaries()
4016 page = f2fs_grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
4040 static void write_normal_summaries(struct f2fs_sb_info *sbi, in write_normal_summaries() argument
4051 write_current_sum_page(sbi, i, blkaddr + (i - type)); in write_normal_summaries()
4054 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in f2fs_write_data_summaries() argument
4056 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) in f2fs_write_data_summaries()
4057 write_compacted_summaries(sbi, start_blk); in f2fs_write_data_summaries()
4059 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); in f2fs_write_data_summaries()
4062 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in f2fs_write_node_summaries() argument
4064 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); in f2fs_write_node_summaries()
4089 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, in get_current_sit_page() argument
4092 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno)); in get_current_sit_page()
4095 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, in get_next_sit_page() argument
4098 struct sit_info *sit_i = SIT_I(sbi); in get_next_sit_page()
4102 src_off = current_sit_addr(sbi, start); in get_next_sit_page()
4103 dst_off = next_sit_addr(sbi, src_off); in get_next_sit_page()
4105 page = f2fs_grab_meta_page(sbi, dst_off); in get_next_sit_page()
4106 seg_info_to_sit_page(sbi, page, start); in get_next_sit_page()
4166 static void add_sits_in_set(struct f2fs_sb_info *sbi) in add_sits_in_set() argument
4168 struct f2fs_sm_info *sm_info = SM_I(sbi); in add_sits_in_set()
4170 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; in add_sits_in_set()
4173 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) in add_sits_in_set()
4177 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) in remove_sits_in_journal() argument
4179 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in remove_sits_in_journal()
4189 dirtied = __mark_sit_entry_dirty(sbi, segno); in remove_sits_in_journal()
4192 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); in remove_sits_in_journal()
4202 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) in f2fs_flush_sit_entries() argument
4204 struct sit_info *sit_i = SIT_I(sbi); in f2fs_flush_sit_entries()
4206 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in f2fs_flush_sit_entries()
4209 struct list_head *head = &SM_I(sbi)->sit_entry_set; in f2fs_flush_sit_entries()
4210 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); in f2fs_flush_sit_entries()
4222 add_sits_in_set(sbi); in f2fs_flush_sit_entries()
4231 remove_sits_in_journal(sbi); in f2fs_flush_sit_entries()
4243 (unsigned long)MAIN_SEGS(sbi)); in f2fs_flush_sit_entries()
4253 page = get_next_sit_page(sbi, start_segno); in f2fs_flush_sit_entries()
4261 se = get_seg_entry(sbi, segno); in f2fs_flush_sit_entries()
4265 f2fs_bug_on(sbi, 1); in f2fs_flush_sit_entries()
4271 add_discard_addrs(sbi, cpc, false); in f2fs_flush_sit_entries()
4277 f2fs_bug_on(sbi, offset < 0); in f2fs_flush_sit_entries()
4282 check_block_count(sbi, segno, in f2fs_flush_sit_entries()
4288 check_block_count(sbi, segno, in f2fs_flush_sit_entries()
4302 f2fs_bug_on(sbi, ses->entry_cnt); in f2fs_flush_sit_entries()
4306 f2fs_bug_on(sbi, !list_empty(head)); in f2fs_flush_sit_entries()
4307 f2fs_bug_on(sbi, sit_i->dirty_sentries); in f2fs_flush_sit_entries()
4313 add_discard_addrs(sbi, cpc, false); in f2fs_flush_sit_entries()
4319 set_prefree_as_free_segments(sbi); in f2fs_flush_sit_entries()
4322 static int build_sit_info(struct f2fs_sb_info *sbi) in build_sit_info() argument
4324 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_sit_info()
4329 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0; in build_sit_info()
4332 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); in build_sit_info()
4336 SM_I(sbi)->sit_info = sit_i; in build_sit_info()
4339 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), in build_sit_info()
4340 MAIN_SEGS(sbi)), in build_sit_info()
4345 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_sit_info()
4346 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, in build_sit_info()
4352 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map); in build_sit_info()
4354 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); in build_sit_info()
4356 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); in build_sit_info()
4362 for (start = 0; start < MAIN_SEGS(sbi); start++) { in build_sit_info()
4380 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
4384 if (__is_large_section(sbi)) { in build_sit_info()
4386 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), in build_sit_info()
4387 MAIN_SECS(sbi)), in build_sit_info()
4397 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); in build_sit_info()
4398 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); in build_sit_info()
4410 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, in build_sit_info()
4420 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; in build_sit_info()
4425 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); in build_sit_info()
4431 static int build_free_segmap(struct f2fs_sb_info *sbi) in build_free_segmap() argument
4437 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); in build_free_segmap()
4441 SM_I(sbi)->free_info = free_i; in build_free_segmap()
4443 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_free_segmap()
4444 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); in build_free_segmap()
4448 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in build_free_segmap()
4449 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); in build_free_segmap()
4458 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); in build_free_segmap()
4465 static int build_curseg(struct f2fs_sb_info *sbi) in build_curseg() argument
4470 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, in build_curseg()
4475 SM_I(sbi)->curseg_array = array; in build_curseg()
4479 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); in build_curseg()
4483 array[i].journal = f2fs_kzalloc(sbi, in build_curseg()
4497 return restore_curseg_summaries(sbi); in build_curseg()
4500 static int build_sit_entries(struct f2fs_sb_info *sbi) in build_sit_entries() argument
4502 struct sit_info *sit_i = SIT_I(sbi); in build_sit_entries()
4503 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in build_sit_entries()
4507 int sit_blk_cnt = SIT_BLK_CNT(sbi); in build_sit_entries()
4514 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, in build_sit_entries()
4520 for (; start < end && start < MAIN_SEGS(sbi); start++) { in build_sit_entries()
4525 page = get_current_sit_page(sbi, start); in build_sit_entries()
4532 err = check_block_count(sbi, start, &sit); in build_sit_entries()
4539 if (f2fs_block_unit_discard(sbi)) { in build_sit_entries()
4541 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
4548 sbi->discard_blks += in build_sit_entries()
4549 sbi->blocks_per_seg - in build_sit_entries()
4554 if (__is_large_section(sbi)) in build_sit_entries()
4555 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
4566 if (start >= MAIN_SEGS(sbi)) { in build_sit_entries()
4567 f2fs_err(sbi, "Wrong journal entry on segno %u", in build_sit_entries()
4580 err = check_block_count(sbi, start, &sit); in build_sit_entries()
4587 if (f2fs_block_unit_discard(sbi)) { in build_sit_entries()
4588 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
4593 sbi->discard_blks += old_valid_blocks; in build_sit_entries()
4594 sbi->discard_blks -= se->valid_blocks; in build_sit_entries()
4598 if (__is_large_section(sbi)) { in build_sit_entries()
4599 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
4601 get_sec_entry(sbi, start)->valid_blocks -= in build_sit_entries()
4607 if (!err && total_node_blocks != valid_node_count(sbi)) { in build_sit_entries()
4608 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", in build_sit_entries()
4609 total_node_blocks, valid_node_count(sbi)); in build_sit_entries()
4616 static void init_free_segmap(struct f2fs_sb_info *sbi) in init_free_segmap() argument
4622 for (start = 0; start < MAIN_SEGS(sbi); start++) { in init_free_segmap()
4623 if (f2fs_usable_blks_in_seg(sbi, start) == 0) in init_free_segmap()
4625 sentry = get_seg_entry(sbi, start); in init_free_segmap()
4627 __set_free(sbi, start); in init_free_segmap()
4629 SIT_I(sbi)->written_valid_blocks += in init_free_segmap()
4635 struct curseg_info *curseg_t = CURSEG_I(sbi, type); in init_free_segmap()
4637 __set_test_and_inuse(sbi, curseg_t->segno); in init_free_segmap()
4641 static void init_dirty_segmap(struct f2fs_sb_info *sbi) in init_dirty_segmap() argument
4643 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_dirty_segmap()
4644 struct free_segmap_info *free_i = FREE_I(sbi); in init_dirty_segmap()
4647 block_t blks_per_sec = BLKS_PER_SEC(sbi); in init_dirty_segmap()
4651 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); in init_dirty_segmap()
4652 if (segno >= MAIN_SEGS(sbi)) in init_dirty_segmap()
4655 valid_blocks = get_valid_blocks(sbi, segno, false); in init_dirty_segmap()
4656 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); in init_dirty_segmap()
4660 f2fs_bug_on(sbi, 1); in init_dirty_segmap()
4664 __locate_dirty_segment(sbi, segno, DIRTY); in init_dirty_segmap()
4668 if (!__is_large_section(sbi)) in init_dirty_segmap()
4672 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { in init_dirty_segmap()
4673 valid_blocks = get_valid_blocks(sbi, segno, true); in init_dirty_segmap()
4674 secno = GET_SEC_FROM_SEG(sbi, segno); in init_dirty_segmap()
4678 if (IS_CURSEC(sbi, secno)) in init_dirty_segmap()
4685 static int init_victim_secmap(struct f2fs_sb_info *sbi) in init_victim_secmap() argument
4687 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_victim_secmap()
4688 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in init_victim_secmap()
4690 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); in init_victim_secmap()
4696 static int build_dirty_segmap(struct f2fs_sb_info *sbi) in build_dirty_segmap() argument
4702 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), in build_dirty_segmap()
4707 SM_I(sbi)->dirty_info = dirty_i; in build_dirty_segmap()
4710 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_dirty_segmap()
4713 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, in build_dirty_segmap()
4719 if (__is_large_section(sbi)) { in build_dirty_segmap()
4720 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in build_dirty_segmap()
4721 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, in build_dirty_segmap()
4727 init_dirty_segmap(sbi); in build_dirty_segmap()
4728 return init_victim_secmap(sbi); in build_dirty_segmap()
4731 static int sanity_check_curseg(struct f2fs_sb_info *sbi) in sanity_check_curseg() argument
4740 struct curseg_info *curseg = CURSEG_I(sbi, i); in sanity_check_curseg()
4741 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); in sanity_check_curseg()
4744 if (f2fs_sb_has_readonly(sbi) && in sanity_check_curseg()
4748 sanity_check_seg_type(sbi, curseg->seg_type); in sanity_check_curseg()
4756 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) { in sanity_check_curseg()
4760 f2fs_err(sbi, in sanity_check_curseg()
4772 static int check_zone_write_pointer(struct f2fs_sb_info *sbi, in check_zone_write_pointer() argument
4778 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; in check_zone_write_pointer()
4786 wp_segno = GET_SEGNO(sbi, wp_block); in check_zone_write_pointer()
4787 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); in check_zone_write_pointer()
4789 zone_segno = GET_SEGNO(sbi, zone_block); in check_zone_write_pointer()
4790 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno); in check_zone_write_pointer()
4792 if (zone_segno >= MAIN_SEGS(sbi)) in check_zone_write_pointer()
4800 if (zone_secno == GET_SEC_FROM_SEG(sbi, in check_zone_write_pointer()
4801 CURSEG_I(sbi, i)->segno)) in check_zone_write_pointer()
4808 for (s = sbi->segs_per_sec - 1; s >= 0; s--) { in check_zone_write_pointer()
4810 se = get_seg_entry(sbi, segno); in check_zone_write_pointer()
4811 for (b = sbi->blocks_per_seg - 1; b >= 0; b--) in check_zone_write_pointer()
4813 last_valid_block = START_BLOCK(sbi, segno) + b; in check_zone_write_pointer()
4827 f2fs_notice(sbi, "Valid block beyond write pointer: " in check_zone_write_pointer()
4829 GET_SEGNO(sbi, last_valid_block), in check_zone_write_pointer()
4830 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block), in check_zone_write_pointer()
4840 f2fs_notice(sbi, in check_zone_write_pointer()
4844 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, in check_zone_write_pointer()
4847 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", in check_zone_write_pointer()
4856 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, in get_target_zoned_dev() argument
4861 for (i = 0; i < sbi->s_ndevs; i++) { in get_target_zoned_dev()
4864 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && in get_target_zoned_dev()
4879 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) in fix_curseg_write_pointer() argument
4881 struct curseg_info *cs = CURSEG_I(sbi, type); in fix_curseg_write_pointer()
4886 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; in fix_curseg_write_pointer()
4890 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); in fix_curseg_write_pointer()
4891 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); in fix_curseg_write_pointer()
4893 zbd = get_target_zoned_dev(sbi, cs_zone_block); in fix_curseg_write_pointer()
4903 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", in fix_curseg_write_pointer()
4912 wp_segno = GET_SEGNO(sbi, wp_block); in fix_curseg_write_pointer()
4913 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); in fix_curseg_write_pointer()
4920 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " in fix_curseg_write_pointer()
4924 f2fs_notice(sbi, "Assign new section to curseg[%d]: " in fix_curseg_write_pointer()
4927 f2fs_allocate_new_section(sbi, type, true); in fix_curseg_write_pointer()
4930 if (check_zone_write_pointer(sbi, zbd, &zone)) in fix_curseg_write_pointer()
4934 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); in fix_curseg_write_pointer()
4935 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); in fix_curseg_write_pointer()
4937 zbd = get_target_zoned_dev(sbi, cs_zone_block); in fix_curseg_write_pointer()
4946 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", in fix_curseg_write_pointer()
4955 f2fs_notice(sbi, in fix_curseg_write_pointer()
4959 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, in fix_curseg_write_pointer()
4963 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", in fix_curseg_write_pointer()
4972 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) in f2fs_fix_curseg_write_pointer() argument
4977 ret = fix_curseg_write_pointer(sbi, i); in f2fs_fix_curseg_write_pointer()
4986 struct f2fs_sb_info *sbi; member
4997 return check_zone_write_pointer(args->sbi, args->fdev, zone); in check_zone_write_pointer_cb()
5000 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) in f2fs_check_write_pointer() argument
5005 for (i = 0; i < sbi->s_ndevs; i++) { in f2fs_check_write_pointer()
5009 args.sbi = sbi; in f2fs_check_write_pointer()
5020 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx, in is_conv_zone() argument
5029 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno, in get_zone_idx() argument
5032 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); in get_zone_idx()
5035 sbi->log_blocks_per_blkz; in get_zone_idx()
5043 struct f2fs_sb_info *sbi, unsigned int segno) in f2fs_usable_zone_segs_in_sec() argument
5047 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); in f2fs_usable_zone_segs_in_sec()
5048 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx); in f2fs_usable_zone_segs_in_sec()
5051 if (is_conv_zone(sbi, zone_idx, dev_idx)) in f2fs_usable_zone_segs_in_sec()
5052 return sbi->segs_per_sec; in f2fs_usable_zone_segs_in_sec()
5059 return sbi->segs_per_sec; in f2fs_usable_zone_segs_in_sec()
5062 unusable_segs_in_sec = (sbi->blocks_per_blkz - in f2fs_usable_zone_segs_in_sec()
5064 sbi->log_blocks_per_seg; in f2fs_usable_zone_segs_in_sec()
5065 return sbi->segs_per_sec - unusable_segs_in_sec; in f2fs_usable_zone_segs_in_sec()
5077 struct f2fs_sb_info *sbi, unsigned int segno) in f2fs_usable_zone_blks_in_seg() argument
5082 secno = GET_SEC_FROM_SEG(sbi, segno); in f2fs_usable_zone_blks_in_seg()
5083 seg_start = START_BLOCK(sbi, segno); in f2fs_usable_zone_blks_in_seg()
5084 dev_idx = f2fs_target_device_index(sbi, seg_start); in f2fs_usable_zone_blks_in_seg()
5085 zone_idx = get_zone_idx(sbi, secno, dev_idx); in f2fs_usable_zone_blks_in_seg()
5091 if (is_conv_zone(sbi, zone_idx, dev_idx)) in f2fs_usable_zone_blks_in_seg()
5092 return sbi->blocks_per_seg; in f2fs_usable_zone_blks_in_seg()
5095 return sbi->blocks_per_seg; in f2fs_usable_zone_blks_in_seg()
5097 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); in f2fs_usable_zone_blks_in_seg()
5109 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr) in f2fs_usable_zone_blks_in_seg()
5112 return sbi->blocks_per_seg; in f2fs_usable_zone_blks_in_seg()
5115 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi) in f2fs_fix_curseg_write_pointer() argument
5120 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi) in f2fs_check_write_pointer() argument
5125 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, in f2fs_usable_zone_blks_in_seg() argument
5131 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi, in f2fs_usable_zone_segs_in_sec() argument
5137 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, in f2fs_usable_blks_in_seg() argument
5140 if (f2fs_sb_has_blkzoned(sbi)) in f2fs_usable_blks_in_seg()
5141 return f2fs_usable_zone_blks_in_seg(sbi, segno); in f2fs_usable_blks_in_seg()
5143 return sbi->blocks_per_seg; in f2fs_usable_blks_in_seg()
5146 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi, in f2fs_usable_segs_in_sec() argument
5149 if (f2fs_sb_has_blkzoned(sbi)) in f2fs_usable_segs_in_sec()
5150 return f2fs_usable_zone_segs_in_sec(sbi, segno); in f2fs_usable_segs_in_sec()
5152 return sbi->segs_per_sec; in f2fs_usable_segs_in_sec()
5158 static void init_min_max_mtime(struct f2fs_sb_info *sbi) in init_min_max_mtime() argument
5160 struct sit_info *sit_i = SIT_I(sbi); in init_min_max_mtime()
5167 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { in init_min_max_mtime()
5171 for (i = 0; i < sbi->segs_per_sec; i++) in init_min_max_mtime()
5172 mtime += get_seg_entry(sbi, segno + i)->mtime; in init_min_max_mtime()
5174 mtime = div_u64(mtime, sbi->segs_per_sec); in init_min_max_mtime()
5179 sit_i->max_mtime = get_mtime(sbi, false); in init_min_max_mtime()
5184 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) in f2fs_build_segment_manager() argument
5186 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in f2fs_build_segment_manager()
5187 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in f2fs_build_segment_manager()
5191 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); in f2fs_build_segment_manager()
5196 sbi->sm_info = sm_info; in f2fs_build_segment_manager()
5209 if (!f2fs_lfs_mode(sbi)) in f2fs_build_segment_manager()
5213 sm_info->min_seq_blocks = sbi->blocks_per_seg; in f2fs_build_segment_manager()
5215 sm_info->min_ssr_sections = reserved_sections(sbi); in f2fs_build_segment_manager()
5221 if (!f2fs_readonly(sbi->sb)) { in f2fs_build_segment_manager()
5222 err = f2fs_create_flush_cmd_control(sbi); in f2fs_build_segment_manager()
5227 err = create_discard_cmd_control(sbi); in f2fs_build_segment_manager()
5231 err = build_sit_info(sbi); in f2fs_build_segment_manager()
5234 err = build_free_segmap(sbi); in f2fs_build_segment_manager()
5237 err = build_curseg(sbi); in f2fs_build_segment_manager()
5242 err = build_sit_entries(sbi); in f2fs_build_segment_manager()
5246 init_free_segmap(sbi); in f2fs_build_segment_manager()
5247 err = build_dirty_segmap(sbi); in f2fs_build_segment_manager()
5251 err = sanity_check_curseg(sbi); in f2fs_build_segment_manager()
5255 init_min_max_mtime(sbi); in f2fs_build_segment_manager()
5259 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, in discard_dirty_segmap() argument
5262 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in discard_dirty_segmap()
5270 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) in destroy_victim_secmap() argument
5272 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_victim_secmap()
5277 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) in destroy_dirty_segmap() argument
5279 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_dirty_segmap()
5287 discard_dirty_segmap(sbi, i); in destroy_dirty_segmap()
5289 if (__is_large_section(sbi)) { in destroy_dirty_segmap()
5295 destroy_victim_secmap(sbi); in destroy_dirty_segmap()
5296 SM_I(sbi)->dirty_info = NULL; in destroy_dirty_segmap()
5300 static void destroy_curseg(struct f2fs_sb_info *sbi) in destroy_curseg() argument
5302 struct curseg_info *array = SM_I(sbi)->curseg_array; in destroy_curseg()
5307 SM_I(sbi)->curseg_array = NULL; in destroy_curseg()
5315 static void destroy_free_segmap(struct f2fs_sb_info *sbi) in destroy_free_segmap() argument
5317 struct free_segmap_info *free_i = SM_I(sbi)->free_info; in destroy_free_segmap()
5321 SM_I(sbi)->free_info = NULL; in destroy_free_segmap()
5327 static void destroy_sit_info(struct f2fs_sb_info *sbi) in destroy_sit_info() argument
5329 struct sit_info *sit_i = SIT_I(sbi); in destroy_sit_info()
5342 SM_I(sbi)->sit_info = NULL; in destroy_sit_info()
5351 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) in f2fs_destroy_segment_manager() argument
5353 struct f2fs_sm_info *sm_info = SM_I(sbi); in f2fs_destroy_segment_manager()
5357 f2fs_destroy_flush_cmd_control(sbi, true); in f2fs_destroy_segment_manager()
5358 destroy_discard_cmd_control(sbi); in f2fs_destroy_segment_manager()
5359 destroy_dirty_segmap(sbi); in f2fs_destroy_segment_manager()
5360 destroy_curseg(sbi); in f2fs_destroy_segment_manager()
5361 destroy_free_segmap(sbi); in f2fs_destroy_segment_manager()
5362 destroy_sit_info(sbi); in f2fs_destroy_segment_manager()
5363 sbi->sm_info = NULL; in f2fs_destroy_segment_manager()