Lines Matching refs:sbi
169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) in f2fs_need_SSR() argument
171 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); in f2fs_need_SSR()
172 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); in f2fs_need_SSR()
173 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); in f2fs_need_SSR()
175 if (test_opt(sbi, LFS)) in f2fs_need_SSR()
177 if (sbi->gc_mode == GC_URGENT) in f2fs_need_SSR()
179 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in f2fs_need_SSR()
182 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + in f2fs_need_SSR()
183 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); in f2fs_need_SSR()
214 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __revoke_inmem_pages() local
256 err = f2fs_get_node_info(sbi, dn.nid, &ni); in __revoke_inmem_pages()
263 f2fs_invalidate_blocks(sbi, dn.data_blkaddr); in __revoke_inmem_pages()
266 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, in __revoke_inmem_pages()
286 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) in f2fs_drop_inmem_pages_all() argument
288 struct list_head *head = &sbi->inode_list[ATOMIC_FILE]; in f2fs_drop_inmem_pages_all()
292 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
294 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
299 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages_all()
320 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_drop_inmem_pages() local
334 spin_lock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages()
337 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); in f2fs_drop_inmem_pages()
343 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_drop_inmem_page() local
347 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page)); in f2fs_drop_inmem_page()
355 f2fs_bug_on(sbi, list_empty(head) || cur->page != page); in f2fs_drop_inmem_page()
359 dec_page_count(sbi, F2FS_INMEM_PAGES); in f2fs_drop_inmem_page()
371 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in __f2fs_commit_inmem_pages() local
375 .sbi = sbi, in __f2fs_commit_inmem_pages()
426 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA); in __f2fs_commit_inmem_pages()
453 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_commit_inmem_pages() local
457 f2fs_balance_fs(sbi, true); in f2fs_commit_inmem_pages()
461 f2fs_lock_op(sbi); in f2fs_commit_inmem_pages()
470 f2fs_unlock_op(sbi); in f2fs_commit_inmem_pages()
480 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) in f2fs_balance_fs() argument
482 if (time_to_inject(sbi, FAULT_CHECKPOINT)) { in f2fs_balance_fs()
484 f2fs_stop_checkpoint(sbi, false); in f2fs_balance_fs()
488 if (need && excess_cached_nats(sbi)) in f2fs_balance_fs()
489 f2fs_balance_fs_bg(sbi); in f2fs_balance_fs()
491 if (!f2fs_is_checkpoint_ready(sbi)) in f2fs_balance_fs()
498 if (has_not_enough_free_secs(sbi, 0, 0)) { in f2fs_balance_fs()
499 mutex_lock(&sbi->gc_mutex); in f2fs_balance_fs()
500 f2fs_gc(sbi, false, false, NULL_SEGNO); in f2fs_balance_fs()
504 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) in f2fs_balance_fs_bg() argument
506 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) in f2fs_balance_fs_bg()
510 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE)) in f2fs_balance_fs_bg()
511 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); in f2fs_balance_fs_bg()
514 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) in f2fs_balance_fs_bg()
515 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); in f2fs_balance_fs_bg()
517 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) in f2fs_balance_fs_bg()
518 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); in f2fs_balance_fs_bg()
520 f2fs_build_free_nids(sbi, false, false); in f2fs_balance_fs_bg()
522 if (!is_idle(sbi, REQ_TIME) && in f2fs_balance_fs_bg()
523 (!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi))) in f2fs_balance_fs_bg()
527 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) || in f2fs_balance_fs_bg()
528 !f2fs_available_free_memory(sbi, INO_ENTRIES) || in f2fs_balance_fs_bg()
529 excess_prefree_segs(sbi) || in f2fs_balance_fs_bg()
530 excess_dirty_nats(sbi) || in f2fs_balance_fs_bg()
531 excess_dirty_nodes(sbi) || in f2fs_balance_fs_bg()
532 f2fs_time_over(sbi, CP_TIME)) { in f2fs_balance_fs_bg()
533 if (test_opt(sbi, DATA_FLUSH)) { in f2fs_balance_fs_bg()
536 mutex_lock(&sbi->flush_lock); in f2fs_balance_fs_bg()
539 f2fs_sync_dirty_inodes(sbi, FILE_INODE); in f2fs_balance_fs_bg()
542 mutex_unlock(&sbi->flush_lock); in f2fs_balance_fs_bg()
544 f2fs_sync_fs(sbi->sb, true); in f2fs_balance_fs_bg()
545 stat_inc_bg_cp_count(sbi->stat_info); in f2fs_balance_fs_bg()
549 static int __submit_flush_wait(struct f2fs_sb_info *sbi, in __submit_flush_wait() argument
555 bio = f2fs_bio_alloc(sbi, 0, false); in __submit_flush_wait()
564 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), in __submit_flush_wait()
565 test_opt(sbi, FLUSH_MERGE), ret); in __submit_flush_wait()
569 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) in submit_flush_wait() argument
574 if (!f2fs_is_multi_device(sbi)) in submit_flush_wait()
575 return __submit_flush_wait(sbi, sbi->sb->s_bdev); in submit_flush_wait()
577 for (i = 0; i < sbi->s_ndevs; i++) { in submit_flush_wait()
578 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) in submit_flush_wait()
580 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in submit_flush_wait()
589 struct f2fs_sb_info *sbi = data; in issue_flush_thread() local
590 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in issue_flush_thread()
596 sb_start_intwrite(sbi->sb); in issue_flush_thread()
607 ret = submit_flush_wait(sbi, cmd->ino); in issue_flush_thread()
618 sb_end_intwrite(sbi->sb); in issue_flush_thread()
625 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) in f2fs_issue_flush() argument
627 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in f2fs_issue_flush()
631 if (test_opt(sbi, NOBARRIER)) in f2fs_issue_flush()
634 if (!test_opt(sbi, FLUSH_MERGE)) { in f2fs_issue_flush()
636 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
643 f2fs_is_multi_device(sbi)) { in f2fs_issue_flush()
644 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
675 ret = submit_flush_wait(sbi, ino); in f2fs_issue_flush()
692 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) in f2fs_create_flush_cmd_control() argument
694 dev_t dev = sbi->sb->s_bdev->bd_dev; in f2fs_create_flush_cmd_control()
698 if (SM_I(sbi)->fcc_info) { in f2fs_create_flush_cmd_control()
699 fcc = SM_I(sbi)->fcc_info; in f2fs_create_flush_cmd_control()
705 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); in f2fs_create_flush_cmd_control()
712 SM_I(sbi)->fcc_info = fcc; in f2fs_create_flush_cmd_control()
713 if (!test_opt(sbi, FLUSH_MERGE)) in f2fs_create_flush_cmd_control()
717 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, in f2fs_create_flush_cmd_control()
722 SM_I(sbi)->fcc_info = NULL; in f2fs_create_flush_cmd_control()
729 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) in f2fs_destroy_flush_cmd_control() argument
731 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; in f2fs_destroy_flush_cmd_control()
741 SM_I(sbi)->fcc_info = NULL; in f2fs_destroy_flush_cmd_control()
745 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) in f2fs_flush_device_cache() argument
749 if (!f2fs_is_multi_device(sbi)) in f2fs_flush_device_cache()
752 for (i = 1; i < sbi->s_ndevs; i++) { in f2fs_flush_device_cache()
753 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) in f2fs_flush_device_cache()
755 ret = __submit_flush_wait(sbi, FDEV(i).bdev); in f2fs_flush_device_cache()
759 spin_lock(&sbi->dev_lock); in f2fs_flush_device_cache()
760 f2fs_clear_bit(i, (char *)&sbi->dirty_device); in f2fs_flush_device_cache()
761 spin_unlock(&sbi->dev_lock); in f2fs_flush_device_cache()
767 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __locate_dirty_segment() argument
770 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __locate_dirty_segment()
773 if (IS_CURSEG(sbi, segno)) in __locate_dirty_segment()
780 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __locate_dirty_segment()
784 f2fs_bug_on(sbi, 1); in __locate_dirty_segment()
792 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, in __remove_dirty_segment() argument
795 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in __remove_dirty_segment()
801 struct seg_entry *sentry = get_seg_entry(sbi, segno); in __remove_dirty_segment()
807 if (get_valid_blocks(sbi, segno, true) == 0) { in __remove_dirty_segment()
808 clear_bit(GET_SEC_FROM_SEG(sbi, segno), in __remove_dirty_segment()
811 clear_bit(segno, SIT_I(sbi)->invalid_segmap); in __remove_dirty_segment()
822 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) in locate_dirty_segment() argument
824 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in locate_dirty_segment()
827 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) in locate_dirty_segment()
832 valid_blocks = get_valid_blocks(sbi, segno, false); in locate_dirty_segment()
833 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno); in locate_dirty_segment()
835 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || in locate_dirty_segment()
836 ckpt_valid_blocks == sbi->blocks_per_seg)) { in locate_dirty_segment()
837 __locate_dirty_segment(sbi, segno, PRE); in locate_dirty_segment()
838 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
839 } else if (valid_blocks < sbi->blocks_per_seg) { in locate_dirty_segment()
840 __locate_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
843 __remove_dirty_segment(sbi, segno, DIRTY); in locate_dirty_segment()
850 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) in f2fs_dirty_to_prefree() argument
852 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_dirty_to_prefree()
856 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in f2fs_dirty_to_prefree()
857 if (get_valid_blocks(sbi, segno, false)) in f2fs_dirty_to_prefree()
859 if (IS_CURSEG(sbi, segno)) in f2fs_dirty_to_prefree()
861 __locate_dirty_segment(sbi, segno, PRE); in f2fs_dirty_to_prefree()
862 __remove_dirty_segment(sbi, segno, DIRTY); in f2fs_dirty_to_prefree()
867 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) in f2fs_get_unusable_blocks() argument
870 (overprovision_segments(sbi) - reserved_segments(sbi)); in f2fs_get_unusable_blocks()
871 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg; in f2fs_get_unusable_blocks()
872 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_get_unusable_blocks()
879 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in f2fs_get_unusable_blocks()
880 se = get_seg_entry(sbi, segno); in f2fs_get_unusable_blocks()
882 holes[NODE] += sbi->blocks_per_seg - se->valid_blocks; in f2fs_get_unusable_blocks()
884 holes[DATA] += sbi->blocks_per_seg - se->valid_blocks; in f2fs_get_unusable_blocks()
894 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) in f2fs_disable_cp_again() argument
897 (overprovision_segments(sbi) - reserved_segments(sbi)); in f2fs_disable_cp_again()
898 if (unusable > F2FS_OPTION(sbi).unusable_cap) in f2fs_disable_cp_again()
900 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && in f2fs_disable_cp_again()
901 dirty_segments(sbi) > ovp_hole_segs) in f2fs_disable_cp_again()
907 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) in get_free_segment() argument
909 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in get_free_segment()
913 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { in get_free_segment()
914 if (get_valid_blocks(sbi, segno, false)) in get_free_segment()
916 if (get_ckpt_valid_blocks(sbi, segno)) in get_free_segment()
925 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, in __create_discard_cmd() argument
929 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __create_discard_cmd()
933 f2fs_bug_on(sbi, !len); in __create_discard_cmd()
957 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi, in __attach_discard_cmd() argument
963 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __attach_discard_cmd()
966 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); in __attach_discard_cmd()
989 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, in __remove_discard_cmd() argument
992 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __remove_discard_cmd()
1004 f2fs_bug_on(sbi, dc->ref); in __remove_discard_cmd()
1033 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, in __check_sit_bitmap() argument
1040 unsigned long offset, size, max_blocks = sbi->blocks_per_seg; in __check_sit_bitmap()
1044 segno = GET_SEGNO(sbi, blk); in __check_sit_bitmap()
1045 sentry = get_seg_entry(sbi, segno); in __check_sit_bitmap()
1046 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); in __check_sit_bitmap()
1048 if (end < START_BLOCK(sbi, segno + 1)) in __check_sit_bitmap()
1049 size = GET_BLKOFF_FROM_SEG0(sbi, end); in __check_sit_bitmap()
1054 f2fs_bug_on(sbi, offset != size); in __check_sit_bitmap()
1055 blk = START_BLOCK(sbi, segno + 1); in __check_sit_bitmap()
1060 static void __init_discard_policy(struct f2fs_sb_info *sbi, in __init_discard_policy() argument
1081 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) { in __init_discard_policy()
1100 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1104 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, in __submit_discard_cmd() argument
1113 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __submit_discard_cmd()
1123 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) in __submit_discard_cmd()
1151 if (time_to_inject(sbi, FAULT_DISCARD)) { in __submit_discard_cmd()
1170 f2fs_bug_on(sbi, !bio); in __submit_discard_cmd()
1189 __check_sit_bitmap(sbi, lstart, lstart + len); in __submit_discard_cmd()
1198 f2fs_update_iostat(sbi, FS_DISCARD, 1); in __submit_discard_cmd()
1207 __update_discard_tree_range(sbi, bdev, lstart, start, len); in __submit_discard_cmd()
1211 static struct discard_cmd *__insert_discard_tree(struct f2fs_sb_info *sbi, in __insert_discard_tree() argument
1217 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __insert_discard_tree()
1229 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, in __insert_discard_tree()
1232 dc = __attach_discard_cmd(sbi, bdev, lstart, start, len, parent, in __insert_discard_tree()
1246 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, in __punch_discard_cmd() argument
1249 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __punch_discard_cmd()
1254 __remove_discard_cmd(sbi, dc); in __punch_discard_cmd()
1269 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1, in __punch_discard_cmd()
1283 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, in __update_discard_tree_range() argument
1287 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __update_discard_tree_range()
1356 __remove_discard_cmd(sbi, tdc); in __update_discard_tree_range()
1361 __insert_discard_tree(sbi, bdev, di.lstart, di.start, in __update_discard_tree_range()
1374 static int __queue_discard_cmd(struct f2fs_sb_info *sbi, in __queue_discard_cmd() argument
1384 if (f2fs_is_multi_device(sbi)) { in __queue_discard_cmd()
1385 int devi = f2fs_target_device_index(sbi, blkstart); in __queue_discard_cmd()
1389 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); in __queue_discard_cmd()
1390 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); in __queue_discard_cmd()
1391 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); in __queue_discard_cmd()
1395 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, in __issue_discard_cmd_orderly() argument
1398 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd_orderly()
1425 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { in __issue_discard_cmd_orderly()
1431 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd_orderly()
1438 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_orderly()
1455 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, in __issue_discard_cmd() argument
1458 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd()
1466 f2fs_update_time(sbi, dpolicy->timeout); in __issue_discard_cmd()
1470 f2fs_time_over(sbi, dpolicy->timeout)) in __issue_discard_cmd()
1477 return __issue_discard_cmd_orderly(sbi, dpolicy); in __issue_discard_cmd()
1485 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, in __issue_discard_cmd()
1489 f2fs_bug_on(sbi, dc->state != D_PREP); in __issue_discard_cmd()
1492 f2fs_time_over(sbi, dpolicy->timeout)) in __issue_discard_cmd()
1496 !is_idle(sbi, DISCARD_TIME)) { in __issue_discard_cmd()
1501 __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd()
1520 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) in __drop_discard_cmd() argument
1522 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __drop_discard_cmd()
1532 f2fs_bug_on(sbi, dc->state != D_PREP); in __drop_discard_cmd()
1533 __remove_discard_cmd(sbi, dc); in __drop_discard_cmd()
1542 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) in f2fs_drop_discard_cmd() argument
1544 __drop_discard_cmd(sbi); in f2fs_drop_discard_cmd()
1547 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, in __wait_one_discard_bio() argument
1550 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_one_discard_bio()
1555 f2fs_bug_on(sbi, dc->state != D_DONE); in __wait_one_discard_bio()
1560 __remove_discard_cmd(sbi, dc); in __wait_one_discard_bio()
1567 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, in __wait_discard_cmd_range() argument
1571 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __wait_discard_cmd_range()
1591 __remove_discard_cmd(sbi, dc); in __wait_discard_cmd_range()
1601 trimmed += __wait_one_discard_bio(sbi, dc); in __wait_discard_cmd_range()
1608 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, in __wait_all_discard_cmd() argument
1615 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); in __wait_all_discard_cmd()
1618 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1); in __wait_all_discard_cmd()
1619 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); in __wait_all_discard_cmd()
1620 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1); in __wait_all_discard_cmd()
1621 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); in __wait_all_discard_cmd()
1627 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_wait_discard_bio() argument
1629 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_wait_discard_bio()
1638 __punch_discard_cmd(sbi, dc, blkaddr); in f2fs_wait_discard_bio()
1647 __wait_one_discard_bio(sbi, dc); in f2fs_wait_discard_bio()
1650 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) in f2fs_stop_discard_thread() argument
1652 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_stop_discard_thread()
1663 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) in f2fs_issue_discard_timeout() argument
1665 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_issue_discard_timeout()
1669 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, in f2fs_issue_discard_timeout()
1672 __issue_discard_cmd(sbi, &dpolicy); in f2fs_issue_discard_timeout()
1673 dropped = __drop_discard_cmd(sbi); in f2fs_issue_discard_timeout()
1676 __wait_all_discard_cmd(sbi, NULL); in f2fs_issue_discard_timeout()
1678 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); in f2fs_issue_discard_timeout()
1684 struct f2fs_sb_info *sbi = data; in issue_discard_thread() local
1685 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in issue_discard_thread()
1694 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, in issue_discard_thread()
1707 __wait_all_discard_cmd(sbi, NULL); in issue_discard_thread()
1711 if (f2fs_readonly(sbi->sb)) in issue_discard_thread()
1715 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { in issue_discard_thread()
1720 if (sbi->gc_mode == GC_URGENT) in issue_discard_thread()
1721 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1); in issue_discard_thread()
1723 sb_start_intwrite(sbi->sb); in issue_discard_thread()
1725 issued = __issue_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1727 __wait_all_discard_cmd(sbi, &dpolicy); in issue_discard_thread()
1730 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); in issue_discard_thread()
1737 sb_end_intwrite(sbi->sb); in issue_discard_thread()
1744 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, in __f2fs_issue_discard_zone() argument
1751 if (f2fs_is_multi_device(sbi)) { in __f2fs_issue_discard_zone()
1752 devi = f2fs_target_device_index(sbi, blkstart); in __f2fs_issue_discard_zone()
1755 f2fs_err(sbi, "Invalid block %x", blkstart); in __f2fs_issue_discard_zone()
1762 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { in __f2fs_issue_discard_zone()
1768 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", in __f2fs_issue_discard_zone()
1769 devi, sbi->s_ndevs ? FDEV(devi).path : "", in __f2fs_issue_discard_zone()
1778 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen); in __f2fs_issue_discard_zone()
1782 static int __issue_discard_async(struct f2fs_sb_info *sbi, in __issue_discard_async() argument
1786 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) in __issue_discard_async()
1787 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1789 return __queue_discard_cmd(sbi, bdev, blkstart, blklen); in __issue_discard_async()
1792 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, in f2fs_issue_discard() argument
1802 bdev = f2fs_target_device(sbi, blkstart, NULL); in f2fs_issue_discard()
1807 f2fs_target_device(sbi, i, NULL); in f2fs_issue_discard()
1810 err = __issue_discard_async(sbi, bdev, in f2fs_issue_discard()
1820 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); in f2fs_issue_discard()
1821 offset = GET_BLKOFF_FROM_SEG0(sbi, i); in f2fs_issue_discard()
1824 sbi->discard_blks--; in f2fs_issue_discard()
1828 err = __issue_discard_async(sbi, bdev, start, len); in f2fs_issue_discard()
1832 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, in add_discard_addrs() argument
1836 int max_blocks = sbi->blocks_per_seg; in add_discard_addrs()
1837 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); in add_discard_addrs()
1841 unsigned long *dmap = SIT_I(sbi)->tmp_map; in add_discard_addrs()
1845 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; in add_discard_addrs()
1848 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi)) in add_discard_addrs()
1852 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks || in add_discard_addrs()
1853 SM_I(sbi)->dcc_info->nr_discards >= in add_discard_addrs()
1854 SM_I(sbi)->dcc_info->max_discards) in add_discard_addrs()
1863 while (force || SM_I(sbi)->dcc_info->nr_discards <= in add_discard_addrs()
1864 SM_I(sbi)->dcc_info->max_discards) { in add_discard_addrs()
1880 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); in add_discard_addrs()
1887 SM_I(sbi)->dcc_info->nr_discards += end - start; in add_discard_addrs()
1898 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) in f2fs_release_discard_addrs() argument
1900 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); in f2fs_release_discard_addrs()
1911 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) in set_prefree_as_free_segments() argument
1913 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in set_prefree_as_free_segments()
1917 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) in set_prefree_as_free_segments()
1918 __set_test_and_free(sbi, segno); in set_prefree_as_free_segments()
1922 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, in f2fs_clear_prefree_segments() argument
1925 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in f2fs_clear_prefree_segments()
1928 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in f2fs_clear_prefree_segments()
1933 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi); in f2fs_clear_prefree_segments()
1942 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); in f2fs_clear_prefree_segments()
1943 if (start >= MAIN_SEGS(sbi)) in f2fs_clear_prefree_segments()
1945 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), in f2fs_clear_prefree_segments()
1949 start = rounddown(start, sbi->segs_per_sec); in f2fs_clear_prefree_segments()
1950 end = roundup(end, sbi->segs_per_sec); in f2fs_clear_prefree_segments()
1958 if (!f2fs_realtime_discard_enable(sbi)) in f2fs_clear_prefree_segments()
1965 if (!test_opt(sbi, LFS) || !__is_large_section(sbi)) { in f2fs_clear_prefree_segments()
1966 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), in f2fs_clear_prefree_segments()
1967 (end - start) << sbi->log_blocks_per_seg); in f2fs_clear_prefree_segments()
1971 secno = GET_SEC_FROM_SEG(sbi, start); in f2fs_clear_prefree_segments()
1972 start_segno = GET_SEG_FROM_SEC(sbi, secno); in f2fs_clear_prefree_segments()
1973 if (!IS_CURSEC(sbi, secno) && in f2fs_clear_prefree_segments()
1974 !get_valid_blocks(sbi, start, true)) in f2fs_clear_prefree_segments()
1975 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), in f2fs_clear_prefree_segments()
1976 sbi->segs_per_sec << sbi->log_blocks_per_seg); in f2fs_clear_prefree_segments()
1978 start = start_segno + sbi->segs_per_sec; in f2fs_clear_prefree_segments()
1994 sbi->blocks_per_seg, cur_pos); in f2fs_clear_prefree_segments()
1997 if (f2fs_sb_has_blkzoned(sbi) || in f2fs_clear_prefree_segments()
2001 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, in f2fs_clear_prefree_segments()
2006 sbi->blocks_per_seg, cur_pos); in f2fs_clear_prefree_segments()
2012 if (cur_pos < sbi->blocks_per_seg) in f2fs_clear_prefree_segments()
2019 wake_up_discard_thread(sbi, false); in f2fs_clear_prefree_segments()
2022 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) in create_discard_cmd_control() argument
2024 dev_t dev = sbi->sb->s_bdev->bd_dev; in create_discard_cmd_control()
2028 if (SM_I(sbi)->dcc_info) { in create_discard_cmd_control()
2029 dcc = SM_I(sbi)->dcc_info; in create_discard_cmd_control()
2033 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); in create_discard_cmd_control()
2048 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg; in create_discard_cmd_control()
2055 SM_I(sbi)->dcc_info = dcc; in create_discard_cmd_control()
2057 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, in create_discard_cmd_control()
2062 SM_I(sbi)->dcc_info = NULL; in create_discard_cmd_control()
2069 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) in destroy_discard_cmd_control() argument
2071 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in destroy_discard_cmd_control()
2076 f2fs_stop_discard_thread(sbi); in destroy_discard_cmd_control()
2083 f2fs_issue_discard_timeout(sbi); in destroy_discard_cmd_control()
2086 SM_I(sbi)->dcc_info = NULL; in destroy_discard_cmd_control()
2089 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) in __mark_sit_entry_dirty() argument
2091 struct sit_info *sit_i = SIT_I(sbi); in __mark_sit_entry_dirty()
2101 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, in __set_sit_entry_type() argument
2104 struct seg_entry *se = get_seg_entry(sbi, segno); in __set_sit_entry_type()
2107 __mark_sit_entry_dirty(sbi, segno); in __set_sit_entry_type()
2110 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) in update_sit_entry() argument
2120 segno = GET_SEGNO(sbi, blkaddr); in update_sit_entry()
2122 se = get_seg_entry(sbi, segno); in update_sit_entry()
2124 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in update_sit_entry()
2126 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || in update_sit_entry()
2127 (new_vblocks > sbi->blocks_per_seg))); in update_sit_entry()
2130 se->mtime = get_mtime(sbi, false); in update_sit_entry()
2131 if (se->mtime > SIT_I(sbi)->max_mtime) in update_sit_entry()
2132 SIT_I(sbi)->max_mtime = se->mtime; in update_sit_entry()
2141 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", in update_sit_entry()
2143 f2fs_bug_on(sbi, 1); in update_sit_entry()
2147 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", in update_sit_entry()
2149 f2fs_bug_on(sbi, 1); in update_sit_entry()
2155 sbi->discard_blks--; in update_sit_entry()
2161 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { in update_sit_entry()
2171 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", in update_sit_entry()
2173 f2fs_bug_on(sbi, 1); in update_sit_entry()
2177 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", in update_sit_entry()
2179 f2fs_bug_on(sbi, 1); in update_sit_entry()
2182 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { in update_sit_entry()
2190 spin_lock(&sbi->stat_lock); in update_sit_entry()
2191 sbi->unusable_block_count++; in update_sit_entry()
2192 spin_unlock(&sbi->stat_lock); in update_sit_entry()
2197 sbi->discard_blks++; in update_sit_entry()
2202 __mark_sit_entry_dirty(sbi, segno); in update_sit_entry()
2205 SIT_I(sbi)->written_valid_blocks += del; in update_sit_entry()
2207 if (__is_large_section(sbi)) in update_sit_entry()
2208 get_sec_entry(sbi, segno)->valid_blocks += del; in update_sit_entry()
2211 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) in f2fs_invalidate_blocks() argument
2213 unsigned int segno = GET_SEGNO(sbi, addr); in f2fs_invalidate_blocks()
2214 struct sit_info *sit_i = SIT_I(sbi); in f2fs_invalidate_blocks()
2216 f2fs_bug_on(sbi, addr == NULL_ADDR); in f2fs_invalidate_blocks()
2220 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr); in f2fs_invalidate_blocks()
2225 update_sit_entry(sbi, addr, -1); in f2fs_invalidate_blocks()
2228 locate_dirty_segment(sbi, segno); in f2fs_invalidate_blocks()
2233 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) in f2fs_is_checkpointed_data() argument
2235 struct sit_info *sit_i = SIT_I(sbi); in f2fs_is_checkpointed_data()
2245 segno = GET_SEGNO(sbi, blkaddr); in f2fs_is_checkpointed_data()
2246 se = get_seg_entry(sbi, segno); in f2fs_is_checkpointed_data()
2247 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); in f2fs_is_checkpointed_data()
2260 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, in __add_sum_entry() argument
2263 struct curseg_info *curseg = CURSEG_I(sbi, type); in __add_sum_entry()
2272 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) in f2fs_npages_for_summary_flush() argument
2278 if (sbi->ckpt->alloc_type[i] == SSR) in f2fs_npages_for_summary_flush()
2279 valid_sum_count += sbi->blocks_per_seg; in f2fs_npages_for_summary_flush()
2283 F2FS_CKPT(sbi)->cur_data_blkoff[i]); in f2fs_npages_for_summary_flush()
2285 valid_sum_count += curseg_blkoff(sbi, i); in f2fs_npages_for_summary_flush()
2302 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) in f2fs_get_sum_page() argument
2304 return f2fs_get_meta_page_nofail(sbi, GET_SUM_BLOCK(sbi, segno)); in f2fs_get_sum_page()
2307 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, in f2fs_update_meta_page() argument
2310 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); in f2fs_update_meta_page()
2317 static void write_sum_page(struct f2fs_sb_info *sbi, in write_sum_page() argument
2320 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr); in write_sum_page()
2323 static void write_current_sum_page(struct f2fs_sb_info *sbi, in write_current_sum_page() argument
2326 struct curseg_info *curseg = CURSEG_I(sbi, type); in write_current_sum_page()
2327 struct page *page = f2fs_grab_meta_page(sbi, blk_addr); in write_current_sum_page()
2349 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) in is_next_segment_free() argument
2351 struct curseg_info *curseg = CURSEG_I(sbi, type); in is_next_segment_free()
2353 struct free_segmap_info *free_i = FREE_I(sbi); in is_next_segment_free()
2355 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) in is_next_segment_free()
2364 static void get_new_segment(struct f2fs_sb_info *sbi, in get_new_segment() argument
2367 struct free_segmap_info *free_i = FREE_I(sbi); in get_new_segment()
2369 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; in get_new_segment()
2370 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); in get_new_segment()
2371 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); in get_new_segment()
2379 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { in get_new_segment()
2381 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); in get_new_segment()
2382 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) in get_new_segment()
2386 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); in get_new_segment()
2387 if (secno >= MAIN_SECS(sbi)) { in get_new_segment()
2390 MAIN_SECS(sbi), 0); in get_new_segment()
2391 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); in get_new_segment()
2406 MAIN_SECS(sbi), 0); in get_new_segment()
2407 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); in get_new_segment()
2412 segno = GET_SEG_FROM_SEC(sbi, secno); in get_new_segment()
2413 zoneno = GET_ZONE_FROM_SEC(sbi, secno); in get_new_segment()
2418 if (sbi->secs_per_zone == 1) in get_new_segment()
2429 if (CURSEG_I(sbi, i)->zone == zoneno) in get_new_segment()
2435 hint = zoneno * sbi->secs_per_zone - 1; in get_new_segment()
2439 hint = (zoneno + 1) * sbi->secs_per_zone; in get_new_segment()
2445 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); in get_new_segment()
2446 __set_inuse(sbi, segno); in get_new_segment()
2451 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) in reset_curseg() argument
2453 struct curseg_info *curseg = CURSEG_I(sbi, type); in reset_curseg()
2457 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); in reset_curseg()
2467 __set_sit_entry_type(sbi, type, curseg->segno, modified); in reset_curseg()
2470 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) in __get_next_segno() argument
2473 if (__is_large_section(sbi)) in __get_next_segno()
2474 return CURSEG_I(sbi, type)->segno; in __get_next_segno()
2476 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in __get_next_segno()
2479 if (test_opt(sbi, NOHEAP) && in __get_next_segno()
2483 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) in __get_next_segno()
2484 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; in __get_next_segno()
2487 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) in __get_next_segno()
2490 return CURSEG_I(sbi, type)->segno; in __get_next_segno()
2497 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) in new_curseg() argument
2499 struct curseg_info *curseg = CURSEG_I(sbi, type); in new_curseg()
2503 write_sum_page(sbi, curseg->sum_blk, in new_curseg()
2504 GET_SUM_BLOCK(sbi, segno)); in new_curseg()
2508 if (test_opt(sbi, NOHEAP)) in new_curseg()
2511 segno = __get_next_segno(sbi, type); in new_curseg()
2512 get_new_segment(sbi, &segno, new_sec, dir); in new_curseg()
2514 reset_curseg(sbi, type, 1); in new_curseg()
2518 static void __next_free_blkoff(struct f2fs_sb_info *sbi, in __next_free_blkoff() argument
2521 struct seg_entry *se = get_seg_entry(sbi, seg->segno); in __next_free_blkoff()
2523 unsigned long *target_map = SIT_I(sbi)->tmp_map; in __next_free_blkoff()
2531 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); in __next_free_blkoff()
2541 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, in __refresh_next_blkoff() argument
2545 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); in __refresh_next_blkoff()
2554 static void change_curseg(struct f2fs_sb_info *sbi, int type) in change_curseg() argument
2556 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in change_curseg()
2557 struct curseg_info *curseg = CURSEG_I(sbi, type); in change_curseg()
2562 write_sum_page(sbi, curseg->sum_blk, in change_curseg()
2563 GET_SUM_BLOCK(sbi, curseg->segno)); in change_curseg()
2564 __set_test_and_inuse(sbi, new_segno); in change_curseg()
2567 __remove_dirty_segment(sbi, new_segno, PRE); in change_curseg()
2568 __remove_dirty_segment(sbi, new_segno, DIRTY); in change_curseg()
2571 reset_curseg(sbi, type, 1); in change_curseg()
2573 __next_free_blkoff(sbi, curseg, 0); in change_curseg()
2575 sum_page = f2fs_get_sum_page(sbi, new_segno); in change_curseg()
2576 f2fs_bug_on(sbi, IS_ERR(sum_page)); in change_curseg()
2582 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) in get_ssr_segment() argument
2584 struct curseg_info *curseg = CURSEG_I(sbi, type); in get_ssr_segment()
2585 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; in get_ssr_segment()
2591 if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) { in get_ssr_segment()
2618 if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) { in get_ssr_segment()
2625 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { in get_ssr_segment()
2626 segno = get_free_segment(sbi); in get_ssr_segment()
2639 static void allocate_segment_by_default(struct f2fs_sb_info *sbi, in allocate_segment_by_default() argument
2642 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_segment_by_default()
2645 new_curseg(sbi, type, true); in allocate_segment_by_default()
2646 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && in allocate_segment_by_default()
2648 new_curseg(sbi, type, false); in allocate_segment_by_default()
2649 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type) && in allocate_segment_by_default()
2650 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) in allocate_segment_by_default()
2651 new_curseg(sbi, type, false); in allocate_segment_by_default()
2652 else if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type)) in allocate_segment_by_default()
2653 change_curseg(sbi, type); in allocate_segment_by_default()
2655 new_curseg(sbi, type, false); in allocate_segment_by_default()
2657 stat_inc_seg_type(sbi, curseg); in allocate_segment_by_default()
2660 void allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, in allocate_segment_for_resize() argument
2663 struct curseg_info *curseg = CURSEG_I(sbi, type); in allocate_segment_for_resize()
2666 down_read(&SM_I(sbi)->curseg_lock); in allocate_segment_for_resize()
2668 down_write(&SIT_I(sbi)->sentry_lock); in allocate_segment_for_resize()
2670 segno = CURSEG_I(sbi, type)->segno; in allocate_segment_for_resize()
2674 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type)) in allocate_segment_for_resize()
2675 change_curseg(sbi, type); in allocate_segment_for_resize()
2677 new_curseg(sbi, type, true); in allocate_segment_for_resize()
2679 stat_inc_seg_type(sbi, curseg); in allocate_segment_for_resize()
2681 locate_dirty_segment(sbi, segno); in allocate_segment_for_resize()
2683 up_write(&SIT_I(sbi)->sentry_lock); in allocate_segment_for_resize()
2686 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", in allocate_segment_for_resize()
2690 up_read(&SM_I(sbi)->curseg_lock); in allocate_segment_for_resize()
2693 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) in f2fs_allocate_new_segments() argument
2699 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_segments()
2702 curseg = CURSEG_I(sbi, i); in f2fs_allocate_new_segments()
2704 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); in f2fs_allocate_new_segments()
2705 locate_dirty_segment(sbi, old_segno); in f2fs_allocate_new_segments()
2708 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_allocate_new_segments()
2715 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, in f2fs_exist_trim_candidates() argument
2721 down_write(&SIT_I(sbi)->sentry_lock); in f2fs_exist_trim_candidates()
2723 if (add_discard_addrs(sbi, cpc, true)) { in f2fs_exist_trim_candidates()
2728 up_write(&SIT_I(sbi)->sentry_lock); in f2fs_exist_trim_candidates()
2734 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, in __issue_discard_cmd_range() argument
2738 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; in __issue_discard_cmd_range()
2751 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi, in __issue_discard_cmd_range()
2776 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); in __issue_discard_cmd_range()
2782 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_range()
2786 trimmed += __wait_all_discard_cmd(sbi, NULL); in __issue_discard_cmd_range()
2793 __remove_discard_cmd(sbi, dc); in __issue_discard_cmd_range()
2806 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) in f2fs_trim_fs() argument
2816 bool need_align = test_opt(sbi, LFS) && __is_large_section(sbi); in f2fs_trim_fs()
2818 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) in f2fs_trim_fs()
2821 if (end < MAIN_BLKADDR(sbi)) in f2fs_trim_fs()
2824 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { in f2fs_trim_fs()
2825 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); in f2fs_trim_fs()
2830 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); in f2fs_trim_fs()
2831 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : in f2fs_trim_fs()
2832 GET_SEGNO(sbi, end); in f2fs_trim_fs()
2834 start_segno = rounddown(start_segno, sbi->segs_per_sec); in f2fs_trim_fs()
2835 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1; in f2fs_trim_fs()
2843 if (sbi->discard_blks == 0) in f2fs_trim_fs()
2846 mutex_lock(&sbi->gc_mutex); in f2fs_trim_fs()
2847 err = f2fs_write_checkpoint(sbi, &cpc); in f2fs_trim_fs()
2848 mutex_unlock(&sbi->gc_mutex); in f2fs_trim_fs()
2858 if (f2fs_realtime_discard_enable(sbi)) in f2fs_trim_fs()
2861 start_block = START_BLOCK(sbi, start_segno); in f2fs_trim_fs()
2862 end_block = START_BLOCK(sbi, end_segno + 1); in f2fs_trim_fs()
2864 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); in f2fs_trim_fs()
2865 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, in f2fs_trim_fs()
2868 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, in f2fs_trim_fs()
2876 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) in __has_curseg_space() argument
2878 struct curseg_info *curseg = CURSEG_I(sbi, type); in __has_curseg_space()
2879 if (curseg->next_blkoff < sbi->blocks_per_seg) in __has_curseg_space()
2957 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, in f2fs_io_type_to_rw_hint() argument
2960 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) { in f2fs_io_type_to_rw_hint()
2971 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) { in f2fs_io_type_to_rw_hint()
3041 switch (F2FS_OPTION(fio->sbi).active_logs) { in __get_segment_type()
3052 f2fs_bug_on(fio->sbi, true); in __get_segment_type()
3064 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, in f2fs_allocate_data_block() argument
3069 struct sit_info *sit_i = SIT_I(sbi); in f2fs_allocate_data_block()
3070 struct curseg_info *curseg = CURSEG_I(sbi, type); in f2fs_allocate_data_block()
3072 down_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_data_block()
3077 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); in f2fs_allocate_data_block()
3079 f2fs_wait_discard_bio(sbi, *new_blkaddr); in f2fs_allocate_data_block()
3086 __add_sum_entry(sbi, type, sum); in f2fs_allocate_data_block()
3088 __refresh_next_blkoff(sbi, curseg); in f2fs_allocate_data_block()
3090 stat_inc_block_count(sbi, curseg); in f2fs_allocate_data_block()
3096 update_sit_entry(sbi, *new_blkaddr, 1); in f2fs_allocate_data_block()
3097 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) in f2fs_allocate_data_block()
3098 update_sit_entry(sbi, old_blkaddr, -1); in f2fs_allocate_data_block()
3100 if (!__has_curseg_space(sbi, type)) in f2fs_allocate_data_block()
3101 sit_i->s_ops->allocate_segment(sbi, type, false); in f2fs_allocate_data_block()
3108 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in f2fs_allocate_data_block()
3109 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); in f2fs_allocate_data_block()
3114 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); in f2fs_allocate_data_block()
3116 f2fs_inode_chksum_set(sbi, page); in f2fs_allocate_data_block()
3119 if (F2FS_IO_ALIGNED(sbi)) in f2fs_allocate_data_block()
3127 io = sbi->write_io[fio->type] + fio->temp; in f2fs_allocate_data_block()
3135 up_read(&SM_I(sbi)->curseg_lock); in f2fs_allocate_data_block()
3140 struct f2fs_sb_info *sbi = fio->sbi; in update_device_state() local
3143 if (!f2fs_is_multi_device(sbi)) in update_device_state()
3146 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr); in update_device_state()
3149 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO); in update_device_state()
3152 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { in update_device_state()
3153 spin_lock(&sbi->dev_lock); in update_device_state()
3154 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); in update_device_state()
3155 spin_unlock(&sbi->dev_lock); in update_device_state()
3162 bool keep_order = (test_opt(fio->sbi, LFS) && type == CURSEG_COLD_DATA); in do_write_page()
3165 down_read(&fio->sbi->io_order_lock); in do_write_page()
3167 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, in do_write_page()
3169 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) in do_write_page()
3170 invalidate_mapping_pages(META_MAPPING(fio->sbi), in do_write_page()
3183 up_read(&fio->sbi->io_order_lock); in do_write_page()
3186 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, in f2fs_do_write_meta_page() argument
3190 .sbi = sbi, in f2fs_do_write_meta_page()
3202 if (unlikely(page->index >= MAIN_BLKADDR(sbi))) in f2fs_do_write_meta_page()
3209 stat_inc_meta_count(sbi, page->index); in f2fs_do_write_meta_page()
3210 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); in f2fs_do_write_meta_page()
3220 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_do_write_node_page()
3226 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_outplace_write_data() local
3229 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); in f2fs_outplace_write_data()
3234 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_outplace_write_data()
3240 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_inplace_write_data() local
3247 segno = GET_SEGNO(sbi, fio->new_blkaddr); in f2fs_inplace_write_data()
3249 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { in f2fs_inplace_write_data()
3250 set_sbi_flag(sbi, SBI_NEED_FSCK); in f2fs_inplace_write_data()
3251 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", in f2fs_inplace_write_data()
3256 stat_inc_inplace_blocks(fio->sbi); in f2fs_inplace_write_data()
3264 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); in f2fs_inplace_write_data()
3270 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, in __f2fs_get_curseg() argument
3276 if (CURSEG_I(sbi, i)->segno == segno) in __f2fs_get_curseg()
3282 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, in f2fs_do_replace_block() argument
3286 struct sit_info *sit_i = SIT_I(sbi); in f2fs_do_replace_block()
3293 segno = GET_SEGNO(sbi, new_blkaddr); in f2fs_do_replace_block()
3294 se = get_seg_entry(sbi, segno); in f2fs_do_replace_block()
3297 down_write(&SM_I(sbi)->curseg_lock); in f2fs_do_replace_block()
3301 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { in f2fs_do_replace_block()
3308 if (IS_CURSEG(sbi, segno)) { in f2fs_do_replace_block()
3310 type = __f2fs_get_curseg(sbi, segno); in f2fs_do_replace_block()
3311 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); in f2fs_do_replace_block()
3317 f2fs_bug_on(sbi, !IS_DATASEG(type)); in f2fs_do_replace_block()
3318 curseg = CURSEG_I(sbi, type); in f2fs_do_replace_block()
3329 change_curseg(sbi, type); in f2fs_do_replace_block()
3332 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); in f2fs_do_replace_block()
3333 __add_sum_entry(sbi, type, sum); in f2fs_do_replace_block()
3336 update_sit_entry(sbi, new_blkaddr, 1); in f2fs_do_replace_block()
3337 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { in f2fs_do_replace_block()
3338 invalidate_mapping_pages(META_MAPPING(sbi), in f2fs_do_replace_block()
3340 update_sit_entry(sbi, old_blkaddr, -1); in f2fs_do_replace_block()
3343 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); in f2fs_do_replace_block()
3344 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); in f2fs_do_replace_block()
3346 locate_dirty_segment(sbi, old_cursegno); in f2fs_do_replace_block()
3351 change_curseg(sbi, type); in f2fs_do_replace_block()
3358 up_write(&SM_I(sbi)->curseg_lock); in f2fs_do_replace_block()
3361 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, in f2fs_replace_block() argument
3370 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, in f2fs_replace_block()
3380 struct f2fs_sb_info *sbi = F2FS_P_SB(page); in f2fs_wait_on_page_writeback() local
3382 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type); in f2fs_wait_on_page_writeback()
3385 f2fs_bug_on(sbi, locked && PageWriteback(page)); in f2fs_wait_on_page_writeback()
3394 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); in f2fs_wait_on_block_writeback() local
3403 cpage = find_lock_page(META_MAPPING(sbi), blkaddr); in f2fs_wait_on_block_writeback()
3419 static int read_compacted_summaries(struct f2fs_sb_info *sbi) in read_compacted_summaries() argument
3421 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_compacted_summaries()
3428 start = start_sum_block(sbi); in read_compacted_summaries()
3430 page = f2fs_get_meta_page(sbi, start++); in read_compacted_summaries()
3436 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in read_compacted_summaries()
3440 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in read_compacted_summaries()
3449 seg_i = CURSEG_I(sbi, i); in read_compacted_summaries()
3453 reset_curseg(sbi, i, 0); in read_compacted_summaries()
3458 blk_off = sbi->blocks_per_seg; in read_compacted_summaries()
3472 page = f2fs_get_meta_page(sbi, start++); in read_compacted_summaries()
3483 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) in read_normal_summaries() argument
3485 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in read_normal_summaries()
3499 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3500 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); in read_normal_summaries()
3502 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); in read_normal_summaries()
3508 if (__exist_node_summaries(sbi)) in read_normal_summaries()
3509 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, in read_normal_summaries()
3512 blk_addr = GET_SUM_BLOCK(sbi, segno); in read_normal_summaries()
3515 new = f2fs_get_meta_page(sbi, blk_addr); in read_normal_summaries()
3521 if (__exist_node_summaries(sbi)) { in read_normal_summaries()
3524 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { in read_normal_summaries()
3529 err = f2fs_restore_node_summary(sbi, segno, sum); in read_normal_summaries()
3536 curseg = CURSEG_I(sbi, type); in read_normal_summaries()
3547 reset_curseg(sbi, type, 0); in read_normal_summaries()
3556 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) in restore_curseg_summaries() argument
3558 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; in restore_curseg_summaries()
3559 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; in restore_curseg_summaries()
3563 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { in restore_curseg_summaries()
3564 int npages = f2fs_npages_for_summary_flush(sbi, true); in restore_curseg_summaries()
3567 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, in restore_curseg_summaries()
3571 err = read_compacted_summaries(sbi); in restore_curseg_summaries()
3577 if (__exist_node_summaries(sbi)) in restore_curseg_summaries()
3578 f2fs_ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), in restore_curseg_summaries()
3582 err = read_normal_summaries(sbi, type); in restore_curseg_summaries()
3590 f2fs_err(sbi, "invalid journal entries nats %u sits %u\n", in restore_curseg_summaries()
3598 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) in write_compacted_summaries() argument
3607 page = f2fs_grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
3612 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); in write_compacted_summaries()
3617 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); in write_compacted_summaries()
3624 seg_i = CURSEG_I(sbi, i); in write_compacted_summaries()
3625 if (sbi->ckpt->alloc_type[i] == SSR) in write_compacted_summaries()
3626 blkoff = sbi->blocks_per_seg; in write_compacted_summaries()
3628 blkoff = curseg_blkoff(sbi, i); in write_compacted_summaries()
3632 page = f2fs_grab_meta_page(sbi, blkaddr++); in write_compacted_summaries()
3656 static void write_normal_summaries(struct f2fs_sb_info *sbi, in write_normal_summaries() argument
3666 write_current_sum_page(sbi, i, blkaddr + (i - type)); in write_normal_summaries()
3669 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in f2fs_write_data_summaries() argument
3671 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) in f2fs_write_data_summaries()
3672 write_compacted_summaries(sbi, start_blk); in f2fs_write_data_summaries()
3674 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); in f2fs_write_data_summaries()
3677 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) in f2fs_write_node_summaries() argument
3679 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); in f2fs_write_node_summaries()
3704 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, in get_current_sit_page() argument
3707 return f2fs_get_meta_page_nofail(sbi, current_sit_addr(sbi, segno)); in get_current_sit_page()
3710 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, in get_next_sit_page() argument
3713 struct sit_info *sit_i = SIT_I(sbi); in get_next_sit_page()
3717 src_off = current_sit_addr(sbi, start); in get_next_sit_page()
3718 dst_off = next_sit_addr(sbi, src_off); in get_next_sit_page()
3720 page = f2fs_grab_meta_page(sbi, dst_off); in get_next_sit_page()
3721 seg_info_to_sit_page(sbi, page, start); in get_next_sit_page()
3780 static void add_sits_in_set(struct f2fs_sb_info *sbi) in add_sits_in_set() argument
3782 struct f2fs_sm_info *sm_info = SM_I(sbi); in add_sits_in_set()
3784 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; in add_sits_in_set()
3787 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) in add_sits_in_set()
3791 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) in remove_sits_in_journal() argument
3793 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in remove_sits_in_journal()
3803 dirtied = __mark_sit_entry_dirty(sbi, segno); in remove_sits_in_journal()
3806 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); in remove_sits_in_journal()
3816 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) in f2fs_flush_sit_entries() argument
3818 struct sit_info *sit_i = SIT_I(sbi); in f2fs_flush_sit_entries()
3820 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in f2fs_flush_sit_entries()
3823 struct list_head *head = &SM_I(sbi)->sit_entry_set; in f2fs_flush_sit_entries()
3824 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); in f2fs_flush_sit_entries()
3836 add_sits_in_set(sbi); in f2fs_flush_sit_entries()
3845 remove_sits_in_journal(sbi); in f2fs_flush_sit_entries()
3857 (unsigned long)MAIN_SEGS(sbi)); in f2fs_flush_sit_entries()
3867 page = get_next_sit_page(sbi, start_segno); in f2fs_flush_sit_entries()
3875 se = get_seg_entry(sbi, segno); in f2fs_flush_sit_entries()
3879 f2fs_bug_on(sbi, 1); in f2fs_flush_sit_entries()
3885 add_discard_addrs(sbi, cpc, false); in f2fs_flush_sit_entries()
3891 f2fs_bug_on(sbi, offset < 0); in f2fs_flush_sit_entries()
3896 check_block_count(sbi, segno, in f2fs_flush_sit_entries()
3902 check_block_count(sbi, segno, in f2fs_flush_sit_entries()
3916 f2fs_bug_on(sbi, ses->entry_cnt); in f2fs_flush_sit_entries()
3920 f2fs_bug_on(sbi, !list_empty(head)); in f2fs_flush_sit_entries()
3921 f2fs_bug_on(sbi, sit_i->dirty_sentries); in f2fs_flush_sit_entries()
3927 add_discard_addrs(sbi, cpc, false); in f2fs_flush_sit_entries()
3933 set_prefree_as_free_segments(sbi); in f2fs_flush_sit_entries()
3936 static int build_sit_info(struct f2fs_sb_info *sbi) in build_sit_info() argument
3938 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in build_sit_info()
3945 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); in build_sit_info()
3949 SM_I(sbi)->sit_info = sit_i; in build_sit_info()
3952 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), in build_sit_info()
3953 MAIN_SEGS(sbi)), in build_sit_info()
3958 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_sit_info()
3959 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, in build_sit_info()
3965 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4; in build_sit_info()
3967 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3; in build_sit_info()
3969 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); in build_sit_info()
3975 for (start = 0; start < MAIN_SEGS(sbi); start++) { in build_sit_info()
3991 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); in build_sit_info()
3995 if (__is_large_section(sbi)) { in build_sit_info()
3997 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), in build_sit_info()
3998 MAIN_SECS(sbi)), in build_sit_info()
4008 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); in build_sit_info()
4009 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); in build_sit_info()
4021 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, in build_sit_info()
4031 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; in build_sit_info()
4036 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); in build_sit_info()
4042 static int build_free_segmap(struct f2fs_sb_info *sbi) in build_free_segmap() argument
4048 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); in build_free_segmap()
4052 SM_I(sbi)->free_info = free_i; in build_free_segmap()
4054 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_free_segmap()
4055 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); in build_free_segmap()
4059 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in build_free_segmap()
4060 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); in build_free_segmap()
4069 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); in build_free_segmap()
4076 static int build_curseg(struct f2fs_sb_info *sbi) in build_curseg() argument
4081 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, sizeof(*array)), in build_curseg()
4086 SM_I(sbi)->curseg_array = array; in build_curseg()
4090 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); in build_curseg()
4094 array[i].journal = f2fs_kzalloc(sbi, in build_curseg()
4101 return restore_curseg_summaries(sbi); in build_curseg()
4104 static int build_sit_entries(struct f2fs_sb_info *sbi) in build_sit_entries() argument
4106 struct sit_info *sit_i = SIT_I(sbi); in build_sit_entries()
4107 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); in build_sit_entries()
4111 int sit_blk_cnt = SIT_BLK_CNT(sbi); in build_sit_entries()
4118 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, in build_sit_entries()
4124 for (; start < end && start < MAIN_SEGS(sbi); start++) { in build_sit_entries()
4129 page = get_current_sit_page(sbi, start); in build_sit_entries()
4136 err = check_block_count(sbi, start, &sit); in build_sit_entries()
4144 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
4151 sbi->discard_blks += in build_sit_entries()
4152 sbi->blocks_per_seg - in build_sit_entries()
4156 if (__is_large_section(sbi)) in build_sit_entries()
4157 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
4168 if (start >= MAIN_SEGS(sbi)) { in build_sit_entries()
4169 f2fs_err(sbi, "Wrong journal entry on segno %u", in build_sit_entries()
4182 err = check_block_count(sbi, start, &sit); in build_sit_entries()
4189 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { in build_sit_entries()
4194 sbi->discard_blks += old_valid_blocks; in build_sit_entries()
4195 sbi->discard_blks -= se->valid_blocks; in build_sit_entries()
4198 if (__is_large_section(sbi)) { in build_sit_entries()
4199 get_sec_entry(sbi, start)->valid_blocks += in build_sit_entries()
4201 get_sec_entry(sbi, start)->valid_blocks -= in build_sit_entries()
4207 if (!err && total_node_blocks != valid_node_count(sbi)) { in build_sit_entries()
4208 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", in build_sit_entries()
4209 total_node_blocks, valid_node_count(sbi)); in build_sit_entries()
4216 static void init_free_segmap(struct f2fs_sb_info *sbi) in init_free_segmap() argument
4221 for (start = 0; start < MAIN_SEGS(sbi); start++) { in init_free_segmap()
4222 struct seg_entry *sentry = get_seg_entry(sbi, start); in init_free_segmap()
4224 __set_free(sbi, start); in init_free_segmap()
4226 SIT_I(sbi)->written_valid_blocks += in init_free_segmap()
4232 struct curseg_info *curseg_t = CURSEG_I(sbi, type); in init_free_segmap()
4233 __set_test_and_inuse(sbi, curseg_t->segno); in init_free_segmap()
4237 static void init_dirty_segmap(struct f2fs_sb_info *sbi) in init_dirty_segmap() argument
4239 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_dirty_segmap()
4240 struct free_segmap_info *free_i = FREE_I(sbi); in init_dirty_segmap()
4246 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); in init_dirty_segmap()
4247 if (segno >= MAIN_SEGS(sbi)) in init_dirty_segmap()
4250 valid_blocks = get_valid_blocks(sbi, segno, false); in init_dirty_segmap()
4251 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) in init_dirty_segmap()
4253 if (valid_blocks > sbi->blocks_per_seg) { in init_dirty_segmap()
4254 f2fs_bug_on(sbi, 1); in init_dirty_segmap()
4258 __locate_dirty_segment(sbi, segno, DIRTY); in init_dirty_segmap()
4263 static int init_victim_secmap(struct f2fs_sb_info *sbi) in init_victim_secmap() argument
4265 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in init_victim_secmap()
4266 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); in init_victim_secmap()
4268 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); in init_victim_secmap()
4274 static int build_dirty_segmap(struct f2fs_sb_info *sbi) in build_dirty_segmap() argument
4280 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), in build_dirty_segmap()
4285 SM_I(sbi)->dirty_info = dirty_i; in build_dirty_segmap()
4288 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); in build_dirty_segmap()
4291 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, in build_dirty_segmap()
4297 init_dirty_segmap(sbi); in build_dirty_segmap()
4298 return init_victim_secmap(sbi); in build_dirty_segmap()
4301 static int sanity_check_curseg(struct f2fs_sb_info *sbi) in sanity_check_curseg() argument
4310 struct curseg_info *curseg = CURSEG_I(sbi, i); in sanity_check_curseg()
4311 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); in sanity_check_curseg()
4320 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) { in sanity_check_curseg()
4324 f2fs_err(sbi, in sanity_check_curseg()
4337 static void init_min_max_mtime(struct f2fs_sb_info *sbi) in init_min_max_mtime() argument
4339 struct sit_info *sit_i = SIT_I(sbi); in init_min_max_mtime()
4346 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { in init_min_max_mtime()
4350 for (i = 0; i < sbi->segs_per_sec; i++) in init_min_max_mtime()
4351 mtime += get_seg_entry(sbi, segno + i)->mtime; in init_min_max_mtime()
4353 mtime = div_u64(mtime, sbi->segs_per_sec); in init_min_max_mtime()
4358 sit_i->max_mtime = get_mtime(sbi, false); in init_min_max_mtime()
4362 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) in f2fs_build_segment_manager() argument
4364 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); in f2fs_build_segment_manager()
4365 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); in f2fs_build_segment_manager()
4369 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); in f2fs_build_segment_manager()
4374 sbi->sm_info = sm_info; in f2fs_build_segment_manager()
4387 if (!test_opt(sbi, LFS)) in f2fs_build_segment_manager()
4391 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec; in f2fs_build_segment_manager()
4393 sm_info->min_ssr_sections = reserved_sections(sbi); in f2fs_build_segment_manager()
4399 if (!f2fs_readonly(sbi->sb)) { in f2fs_build_segment_manager()
4400 err = f2fs_create_flush_cmd_control(sbi); in f2fs_build_segment_manager()
4405 err = create_discard_cmd_control(sbi); in f2fs_build_segment_manager()
4409 err = build_sit_info(sbi); in f2fs_build_segment_manager()
4412 err = build_free_segmap(sbi); in f2fs_build_segment_manager()
4415 err = build_curseg(sbi); in f2fs_build_segment_manager()
4420 err = build_sit_entries(sbi); in f2fs_build_segment_manager()
4424 init_free_segmap(sbi); in f2fs_build_segment_manager()
4425 err = build_dirty_segmap(sbi); in f2fs_build_segment_manager()
4429 err = sanity_check_curseg(sbi); in f2fs_build_segment_manager()
4433 init_min_max_mtime(sbi); in f2fs_build_segment_manager()
4437 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, in discard_dirty_segmap() argument
4440 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in discard_dirty_segmap()
4448 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) in destroy_victim_secmap() argument
4450 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_victim_secmap()
4454 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) in destroy_dirty_segmap() argument
4456 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); in destroy_dirty_segmap()
4464 discard_dirty_segmap(sbi, i); in destroy_dirty_segmap()
4466 destroy_victim_secmap(sbi); in destroy_dirty_segmap()
4467 SM_I(sbi)->dirty_info = NULL; in destroy_dirty_segmap()
4471 static void destroy_curseg(struct f2fs_sb_info *sbi) in destroy_curseg() argument
4473 struct curseg_info *array = SM_I(sbi)->curseg_array; in destroy_curseg()
4478 SM_I(sbi)->curseg_array = NULL; in destroy_curseg()
4486 static void destroy_free_segmap(struct f2fs_sb_info *sbi) in destroy_free_segmap() argument
4488 struct free_segmap_info *free_i = SM_I(sbi)->free_info; in destroy_free_segmap()
4491 SM_I(sbi)->free_info = NULL; in destroy_free_segmap()
4497 static void destroy_sit_info(struct f2fs_sb_info *sbi) in destroy_sit_info() argument
4499 struct sit_info *sit_i = SIT_I(sbi); in destroy_sit_info()
4512 SM_I(sbi)->sit_info = NULL; in destroy_sit_info()
4521 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) in f2fs_destroy_segment_manager() argument
4523 struct f2fs_sm_info *sm_info = SM_I(sbi); in f2fs_destroy_segment_manager()
4527 f2fs_destroy_flush_cmd_control(sbi, true); in f2fs_destroy_segment_manager()
4528 destroy_discard_cmd_control(sbi); in f2fs_destroy_segment_manager()
4529 destroy_dirty_segmap(sbi); in f2fs_destroy_segment_manager()
4530 destroy_curseg(sbi); in f2fs_destroy_segment_manager()
4531 destroy_free_segmap(sbi); in f2fs_destroy_segment_manager()
4532 destroy_sit_info(sbi); in f2fs_destroy_segment_manager()
4533 sbi->sm_info = NULL; in f2fs_destroy_segment_manager()