169 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)  in f2fs_need_SSR()  argument
171 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);  in f2fs_need_SSR()
172 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);  in f2fs_need_SSR()
173 	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);  in f2fs_need_SSR()
175 	if (f2fs_lfs_mode(sbi))  in f2fs_need_SSR()
177 	if (sbi->gc_mode == GC_URGENT_HIGH)  in f2fs_need_SSR()
179 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in f2fs_need_SSR()
182 	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +  in f2fs_need_SSR()
183 			SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));  in f2fs_need_SSR()
214 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __revoke_inmem_pages()  local
257 			err = f2fs_get_node_info(sbi, dn.nid, &ni);  in __revoke_inmem_pages()
264 				f2fs_invalidate_blocks(sbi, dn.data_blkaddr);  in __revoke_inmem_pages()
267 				f2fs_replace_block(sbi, &dn, dn.data_blkaddr,  in __revoke_inmem_pages()
287 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)  in f2fs_drop_inmem_pages_all()  argument
289 	struct list_head *head = &sbi->inode_list[ATOMIC_FILE];  in f2fs_drop_inmem_pages_all()
292 	unsigned int count = sbi->atomic_files;  in f2fs_drop_inmem_pages_all()
295 	spin_lock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
297 		spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
304 	spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages_all()
327 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_drop_inmem_pages()  local
339 	spin_lock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages()
344 		sbi->atomic_files--;  in f2fs_drop_inmem_pages()
346 	spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);  in f2fs_drop_inmem_pages()
352 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_drop_inmem_page()  local
356 	f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page));  in f2fs_drop_inmem_page()
364 	f2fs_bug_on(sbi, list_empty(head) || cur->page != page);  in f2fs_drop_inmem_page()
368 	dec_page_count(sbi, F2FS_INMEM_PAGES);  in f2fs_drop_inmem_page()
380 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in __f2fs_commit_inmem_pages()  local
384 		.sbi = sbi,  in __f2fs_commit_inmem_pages()
436 		f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);  in __f2fs_commit_inmem_pages()
463 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_commit_inmem_pages()  local
467 	f2fs_balance_fs(sbi, true);  in f2fs_commit_inmem_pages()
471 	f2fs_lock_op(sbi);  in f2fs_commit_inmem_pages()
480 	f2fs_unlock_op(sbi);  in f2fs_commit_inmem_pages()
490 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)  in f2fs_balance_fs()  argument
492 	if (time_to_inject(sbi, FAULT_CHECKPOINT)) {  in f2fs_balance_fs()
493 		f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);  in f2fs_balance_fs()
494 		f2fs_stop_checkpoint(sbi, false);  in f2fs_balance_fs()
498 	if (need && excess_cached_nats(sbi))  in f2fs_balance_fs()
499 		f2fs_balance_fs_bg(sbi, false);  in f2fs_balance_fs()
501 	if (!f2fs_is_checkpoint_ready(sbi))  in f2fs_balance_fs()
508 	if (has_not_enough_free_secs(sbi, 0, 0)) {  in f2fs_balance_fs()
509 		down_write(&sbi->gc_lock);  in f2fs_balance_fs()
510 		f2fs_gc(sbi, false, false, NULL_SEGNO);  in f2fs_balance_fs()
514 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)  in f2fs_balance_fs_bg()  argument
516 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))  in f2fs_balance_fs_bg()
520 	if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))  in f2fs_balance_fs_bg()
521 		f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);  in f2fs_balance_fs_bg()
524 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))  in f2fs_balance_fs_bg()
525 		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);  in f2fs_balance_fs_bg()
527 	if (!f2fs_available_free_memory(sbi, FREE_NIDS))  in f2fs_balance_fs_bg()
528 		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);  in f2fs_balance_fs_bg()
530 		f2fs_build_free_nids(sbi, false, false);  in f2fs_balance_fs_bg()
532 	if (!is_idle(sbi, REQ_TIME) &&  in f2fs_balance_fs_bg()
533 		(!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi)))  in f2fs_balance_fs_bg()
537 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) ||  in f2fs_balance_fs_bg()
538 			!f2fs_available_free_memory(sbi, INO_ENTRIES) ||  in f2fs_balance_fs_bg()
539 			excess_prefree_segs(sbi) ||  in f2fs_balance_fs_bg()
540 			excess_dirty_nats(sbi) ||  in f2fs_balance_fs_bg()
541 			excess_dirty_nodes(sbi) ||  in f2fs_balance_fs_bg()
542 			f2fs_time_over(sbi, CP_TIME)) {  in f2fs_balance_fs_bg()
543 		if (test_opt(sbi, DATA_FLUSH) && from_bg) {  in f2fs_balance_fs_bg()
546 			mutex_lock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
549 			f2fs_sync_dirty_inodes(sbi, FILE_INODE);  in f2fs_balance_fs_bg()
552 			mutex_unlock(&sbi->flush_lock);  in f2fs_balance_fs_bg()
554 		f2fs_sync_fs(sbi->sb, true);  in f2fs_balance_fs_bg()
555 		stat_inc_bg_cp_count(sbi->stat_info);  in f2fs_balance_fs_bg()
559 static int __submit_flush_wait(struct f2fs_sb_info *sbi,  in __submit_flush_wait()  argument
565 	bio = f2fs_bio_alloc(sbi, 0, false);  in __submit_flush_wait()
574 	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),  in __submit_flush_wait()
575 				test_opt(sbi, FLUSH_MERGE), ret);  in __submit_flush_wait()
579 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)  in submit_flush_wait()  argument
584 	if (!f2fs_is_multi_device(sbi))  in submit_flush_wait()
585 		return __submit_flush_wait(sbi, sbi->sb->s_bdev);  in submit_flush_wait()
587 	for (i = 0; i < sbi->s_ndevs; i++) {  in submit_flush_wait()
588 		if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))  in submit_flush_wait()
590 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in submit_flush_wait()
599 	struct f2fs_sb_info *sbi = data;  in issue_flush_thread()  local
600 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in issue_flush_thread()
606 	sb_start_intwrite(sbi->sb);  in issue_flush_thread()
617 		ret = submit_flush_wait(sbi, cmd->ino);  in issue_flush_thread()
628 	sb_end_intwrite(sbi->sb);  in issue_flush_thread()
635 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)  in f2fs_issue_flush()  argument
637 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_issue_flush()
641 	if (test_opt(sbi, NOBARRIER))  in f2fs_issue_flush()
644 	if (!test_opt(sbi, FLUSH_MERGE)) {  in f2fs_issue_flush()
646 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
653 	    f2fs_is_multi_device(sbi)) {  in f2fs_issue_flush()
654 		ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
685 			ret = submit_flush_wait(sbi, ino);  in f2fs_issue_flush()
702 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)  in f2fs_create_flush_cmd_control()  argument
704 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in f2fs_create_flush_cmd_control()
708 	if (SM_I(sbi)->fcc_info) {  in f2fs_create_flush_cmd_control()
709 		fcc = SM_I(sbi)->fcc_info;  in f2fs_create_flush_cmd_control()
715 	fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);  in f2fs_create_flush_cmd_control()
722 	SM_I(sbi)->fcc_info = fcc;  in f2fs_create_flush_cmd_control()
723 	if (!test_opt(sbi, FLUSH_MERGE))  in f2fs_create_flush_cmd_control()
727 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,  in f2fs_create_flush_cmd_control()
732 		SM_I(sbi)->fcc_info = NULL;  in f2fs_create_flush_cmd_control()
739 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)  in f2fs_destroy_flush_cmd_control()  argument
741 	struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;  in f2fs_destroy_flush_cmd_control()
751 		SM_I(sbi)->fcc_info = NULL;  in f2fs_destroy_flush_cmd_control()
755 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)  in f2fs_flush_device_cache()  argument
759 	if (!f2fs_is_multi_device(sbi))  in f2fs_flush_device_cache()
762 	if (test_opt(sbi, NOBARRIER))  in f2fs_flush_device_cache()
765 	for (i = 1; i < sbi->s_ndevs; i++) {  in f2fs_flush_device_cache()
766 		if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))  in f2fs_flush_device_cache()
768 		ret = __submit_flush_wait(sbi, FDEV(i).bdev);  in f2fs_flush_device_cache()
772 		spin_lock(&sbi->dev_lock);  in f2fs_flush_device_cache()
773 		f2fs_clear_bit(i, (char *)&sbi->dirty_device);  in f2fs_flush_device_cache()
774 		spin_unlock(&sbi->dev_lock);  in f2fs_flush_device_cache()
780 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __locate_dirty_segment()  argument
783 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __locate_dirty_segment()
786 	if (IS_CURSEG(sbi, segno))  in __locate_dirty_segment()
793 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __locate_dirty_segment()
797 			f2fs_bug_on(sbi, 1);  in __locate_dirty_segment()
803 		if (__is_large_section(sbi)) {  in __locate_dirty_segment()
804 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __locate_dirty_segment()
806 				get_valid_blocks(sbi, segno, true);  in __locate_dirty_segment()
808 			f2fs_bug_on(sbi, unlikely(!valid_blocks ||  in __locate_dirty_segment()
809 					valid_blocks == BLKS_PER_SEC(sbi)));  in __locate_dirty_segment()
811 			if (!IS_CURSEC(sbi, secno))  in __locate_dirty_segment()
817 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,  in __remove_dirty_segment()  argument
820 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in __remove_dirty_segment()
827 		struct seg_entry *sentry = get_seg_entry(sbi, segno);  in __remove_dirty_segment()
833 		valid_blocks = get_valid_blocks(sbi, segno, true);  in __remove_dirty_segment()
835 			clear_bit(GET_SEC_FROM_SEG(sbi, segno),  in __remove_dirty_segment()
838 			clear_bit(segno, SIT_I(sbi)->invalid_segmap);  in __remove_dirty_segment()
841 		if (__is_large_section(sbi)) {  in __remove_dirty_segment()
842 			unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);  in __remove_dirty_segment()
845 					valid_blocks == BLKS_PER_SEC(sbi)) {  in __remove_dirty_segment()
850 			if (!IS_CURSEC(sbi, secno))  in __remove_dirty_segment()
861 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)  in locate_dirty_segment()  argument
863 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in locate_dirty_segment()
867 	if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))  in locate_dirty_segment()
870 	usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);  in locate_dirty_segment()
873 	valid_blocks = get_valid_blocks(sbi, segno, false);  in locate_dirty_segment()
874 	ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno);  in locate_dirty_segment()
876 	if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||  in locate_dirty_segment()
878 		__locate_dirty_segment(sbi, segno, PRE);  in locate_dirty_segment()
879 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
881 		__locate_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
884 		__remove_dirty_segment(sbi, segno, DIRTY);  in locate_dirty_segment()
891 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)  in f2fs_dirty_to_prefree()  argument
893 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_dirty_to_prefree()
897 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_dirty_to_prefree()
898 		if (get_valid_blocks(sbi, segno, false))  in f2fs_dirty_to_prefree()
900 		if (IS_CURSEG(sbi, segno))  in f2fs_dirty_to_prefree()
902 		__locate_dirty_segment(sbi, segno, PRE);  in f2fs_dirty_to_prefree()
903 		__remove_dirty_segment(sbi, segno, DIRTY);  in f2fs_dirty_to_prefree()
908 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)  in f2fs_get_unusable_blocks()  argument
911 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_get_unusable_blocks()
912 	block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;  in f2fs_get_unusable_blocks()
913 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_get_unusable_blocks()
920 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in f2fs_get_unusable_blocks()
921 		se = get_seg_entry(sbi, segno);  in f2fs_get_unusable_blocks()
923 			holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
926 			holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -  in f2fs_get_unusable_blocks()
937 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)  in f2fs_disable_cp_again()  argument
940 		(overprovision_segments(sbi) - reserved_segments(sbi));  in f2fs_disable_cp_again()
941 	if (unusable > F2FS_OPTION(sbi).unusable_cap)  in f2fs_disable_cp_again()
943 	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&  in f2fs_disable_cp_again()
944 		dirty_segments(sbi) > ovp_hole_segs)  in f2fs_disable_cp_again()
950 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)  in get_free_segment()  argument
952 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in get_free_segment()
956 	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {  in get_free_segment()
957 		if (get_valid_blocks(sbi, segno, false))  in get_free_segment()
959 		if (get_ckpt_valid_blocks(sbi, segno))  in get_free_segment()
968 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,  in __create_discard_cmd()  argument
972 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __create_discard_cmd()
976 	f2fs_bug_on(sbi, !len);  in __create_discard_cmd()
1000 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,  in __attach_discard_cmd()  argument
1006 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __attach_discard_cmd()
1009 	dc = __create_discard_cmd(sbi, bdev, lstart, start, len);  in __attach_discard_cmd()
1032 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,  in __remove_discard_cmd()  argument
1035 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __remove_discard_cmd()
1047 	f2fs_bug_on(sbi, dc->ref);  in __remove_discard_cmd()
1055 			KERN_INFO, sbi->sb->s_id,  in __remove_discard_cmd()
1077 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,  in __check_sit_bitmap()  argument
1084 	unsigned long offset, size, max_blocks = sbi->blocks_per_seg;  in __check_sit_bitmap()
1088 		segno = GET_SEGNO(sbi, blk);  in __check_sit_bitmap()
1089 		sentry = get_seg_entry(sbi, segno);  in __check_sit_bitmap()
1090 		offset = GET_BLKOFF_FROM_SEG0(sbi, blk);  in __check_sit_bitmap()
1092 		if (end < START_BLOCK(sbi, segno + 1))  in __check_sit_bitmap()
1093 			size = GET_BLKOFF_FROM_SEG0(sbi, end);  in __check_sit_bitmap()
1098 		f2fs_bug_on(sbi, offset != size);  in __check_sit_bitmap()
1099 		blk = START_BLOCK(sbi, segno + 1);  in __check_sit_bitmap()
1104 static void __init_discard_policy(struct f2fs_sb_info *sbi,  in __init_discard_policy()  argument
1125 		if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {  in __init_discard_policy()
1144 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1148 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,  in __submit_discard_cmd()  argument
1157 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_discard_cmd()
1167 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))  in __submit_discard_cmd()
1195 		if (time_to_inject(sbi, FAULT_DISCARD)) {  in __submit_discard_cmd()
1196 			f2fs_show_injection_info(sbi, FAULT_DISCARD);  in __submit_discard_cmd()
1214 		f2fs_bug_on(sbi, !bio);  in __submit_discard_cmd()
1233 		__check_sit_bitmap(sbi, lstart, lstart + len);  in __submit_discard_cmd()
1242 		f2fs_update_iostat(sbi, FS_DISCARD, 1);  in __submit_discard_cmd()
1252 		__update_discard_tree_range(sbi, bdev, lstart, start, len);  in __submit_discard_cmd()
1257 static void __insert_discard_tree(struct f2fs_sb_info *sbi,  in __insert_discard_tree()  argument
1263 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __insert_discard_tree()
1274 	p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,  in __insert_discard_tree()
1277 	__attach_discard_cmd(sbi, bdev, lstart, start, len, parent,  in __insert_discard_tree()
1287 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,  in __punch_discard_cmd()  argument
1290 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __punch_discard_cmd()
1295 		__remove_discard_cmd(sbi, dc);  in __punch_discard_cmd()
1310 			__insert_discard_tree(sbi, dc->bdev, blkaddr + 1,  in __punch_discard_cmd()
1324 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,  in __update_discard_tree_range()  argument
1328 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __update_discard_tree_range()
1397 				__remove_discard_cmd(sbi, tdc);  in __update_discard_tree_range()
1402 			__insert_discard_tree(sbi, bdev, di.lstart, di.start,  in __update_discard_tree_range()
1415 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,  in __queue_discard_cmd()  argument
1425 	if (f2fs_is_multi_device(sbi)) {  in __queue_discard_cmd()
1426 		int devi = f2fs_target_device_index(sbi, blkstart);  in __queue_discard_cmd()
1430 	mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1431 	__update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);  in __queue_discard_cmd()
1432 	mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);  in __queue_discard_cmd()
1436 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_orderly()  argument
1439 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_orderly()
1466 		if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd_orderly()
1472 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd_orderly()
1479 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_orderly()
1495 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1498 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,  in __issue_discard_cmd()  argument
1501 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd()
1509 		f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);  in __issue_discard_cmd()
1515 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1522 			return __issue_discard_cmd_orderly(sbi, dpolicy);  in __issue_discard_cmd()
1530 			f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,  in __issue_discard_cmd()
1534 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __issue_discard_cmd()
1537 				f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))  in __issue_discard_cmd()
1541 						!is_idle(sbi, DISCARD_TIME)) {  in __issue_discard_cmd()
1546 			__submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd()
1560 		__wait_all_discard_cmd(sbi, dpolicy);  in __issue_discard_cmd()
1570 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)  in __drop_discard_cmd()  argument
1572 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __drop_discard_cmd()
1582 			f2fs_bug_on(sbi, dc->state != D_PREP);  in __drop_discard_cmd()
1583 			__remove_discard_cmd(sbi, dc);  in __drop_discard_cmd()
1592 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)  in f2fs_drop_discard_cmd()  argument
1594 	__drop_discard_cmd(sbi);  in f2fs_drop_discard_cmd()
1597 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,  in __wait_one_discard_bio()  argument
1600 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_one_discard_bio()
1605 	f2fs_bug_on(sbi, dc->state != D_DONE);  in __wait_one_discard_bio()
1610 		__remove_discard_cmd(sbi, dc);  in __wait_one_discard_bio()
1617 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,  in __wait_discard_cmd_range()  argument
1621 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_discard_cmd_range()
1641 			__remove_discard_cmd(sbi, dc);  in __wait_discard_cmd_range()
1651 		trimmed += __wait_one_discard_bio(sbi, dc);  in __wait_discard_cmd_range()
1658 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,  in __wait_all_discard_cmd()  argument
1665 		return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);  in __wait_all_discard_cmd()
1668 	__init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);  in __wait_all_discard_cmd()
1669 	discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1670 	__init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);  in __wait_all_discard_cmd()
1671 	discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);  in __wait_all_discard_cmd()
1677 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_wait_discard_bio()  argument
1679 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bio()
1688 			__punch_discard_cmd(sbi, dc, blkaddr);  in f2fs_wait_discard_bio()
1697 		__wait_one_discard_bio(sbi, dc);  in f2fs_wait_discard_bio()
1700 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)  in f2fs_stop_discard_thread()  argument
1702 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_stop_discard_thread()
1713 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)  in f2fs_issue_discard_timeout()  argument
1715 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_issue_discard_timeout()
1719 	__init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,  in f2fs_issue_discard_timeout()
1721 	__issue_discard_cmd(sbi, &dpolicy);  in f2fs_issue_discard_timeout()
1722 	dropped = __drop_discard_cmd(sbi);  in f2fs_issue_discard_timeout()
1725 	__wait_all_discard_cmd(sbi, NULL);  in f2fs_issue_discard_timeout()
1727 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));  in f2fs_issue_discard_timeout()
1733 	struct f2fs_sb_info *sbi = data;  in issue_discard_thread()  local
1734 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in issue_discard_thread()
1743 		__init_discard_policy(sbi, &dpolicy, DPOLICY_BG,  in issue_discard_thread()
1756 			__wait_all_discard_cmd(sbi, NULL);  in issue_discard_thread()
1760 		if (f2fs_readonly(sbi->sb))  in issue_discard_thread()
1764 		if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {  in issue_discard_thread()
1769 		if (sbi->gc_mode == GC_URGENT_HIGH)  in issue_discard_thread()
1770 			__init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);  in issue_discard_thread()
1772 		sb_start_intwrite(sbi->sb);  in issue_discard_thread()
1774 		issued = __issue_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1776 			__wait_all_discard_cmd(sbi, &dpolicy);  in issue_discard_thread()
1779 			wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);  in issue_discard_thread()
1786 		sb_end_intwrite(sbi->sb);  in issue_discard_thread()
1793 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,  in __f2fs_issue_discard_zone()  argument
1800 	if (f2fs_is_multi_device(sbi)) {  in __f2fs_issue_discard_zone()
1801 		devi = f2fs_target_device_index(sbi, blkstart);  in __f2fs_issue_discard_zone()
1804 			f2fs_err(sbi, "Invalid block %x", blkstart);  in __f2fs_issue_discard_zone()
1811 	if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {  in __f2fs_issue_discard_zone()
1817 			f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",  in __f2fs_issue_discard_zone()
1818 				 devi, sbi->s_ndevs ? FDEV(devi).path : "",  in __f2fs_issue_discard_zone()
1828 	return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);  in __f2fs_issue_discard_zone()
1832 static int __issue_discard_async(struct f2fs_sb_info *sbi,  in __issue_discard_async()  argument
1836 	if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))  in __issue_discard_async()
1837 		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1839 	return __queue_discard_cmd(sbi, bdev, blkstart, blklen);  in __issue_discard_async()
1842 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,  in f2fs_issue_discard()  argument
1852 	bdev = f2fs_target_device(sbi, blkstart, NULL);  in f2fs_issue_discard()
1857 				f2fs_target_device(sbi, i, NULL);  in f2fs_issue_discard()
1860 				err = __issue_discard_async(sbi, bdev,  in f2fs_issue_discard()
1870 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));  in f2fs_issue_discard()
1871 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);  in f2fs_issue_discard()
1874 			sbi->discard_blks--;  in f2fs_issue_discard()
1878 		err = __issue_discard_async(sbi, bdev, start, len);  in f2fs_issue_discard()
1882 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,  in add_discard_addrs()  argument
1886 	int max_blocks = sbi->blocks_per_seg;  in add_discard_addrs()
1887 	struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);  in add_discard_addrs()
1891 	unsigned long *dmap = SIT_I(sbi)->tmp_map;  in add_discard_addrs()
1895 	struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;  in add_discard_addrs()
1898 	if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))  in add_discard_addrs()
1902 		if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||  in add_discard_addrs()
1903 			SM_I(sbi)->dcc_info->nr_discards >=  in add_discard_addrs()
1904 				SM_I(sbi)->dcc_info->max_discards)  in add_discard_addrs()
1913 	while (force || SM_I(sbi)->dcc_info->nr_discards <=  in add_discard_addrs()
1914 				SM_I(sbi)->dcc_info->max_discards) {  in add_discard_addrs()
1930 			de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);  in add_discard_addrs()
1937 		SM_I(sbi)->dcc_info->nr_discards += end - start;  in add_discard_addrs()
1948 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)  in f2fs_release_discard_addrs()  argument
1950 	struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);  in f2fs_release_discard_addrs()
1961 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)  in set_prefree_as_free_segments()  argument
1963 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in set_prefree_as_free_segments()
1967 	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))  in set_prefree_as_free_segments()
1968 		__set_test_and_free(sbi, segno, false);  in set_prefree_as_free_segments()
1972 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,  in f2fs_clear_prefree_segments()  argument
1975 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_clear_prefree_segments()
1978 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in f2fs_clear_prefree_segments()
1983 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);  in f2fs_clear_prefree_segments()
1992 		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);  in f2fs_clear_prefree_segments()
1993 		if (start >= MAIN_SEGS(sbi))  in f2fs_clear_prefree_segments()
1995 		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),  in f2fs_clear_prefree_segments()
1999 			start = rounddown(start, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2000 			end = roundup(end, sbi->segs_per_sec);  in f2fs_clear_prefree_segments()
2008 		if (!f2fs_realtime_discard_enable(sbi))  in f2fs_clear_prefree_segments()
2015 		if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {  in f2fs_clear_prefree_segments()
2016 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start),  in f2fs_clear_prefree_segments()
2017 				(end - start) << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2021 		secno = GET_SEC_FROM_SEG(sbi, start);  in f2fs_clear_prefree_segments()
2022 		start_segno = GET_SEG_FROM_SEC(sbi, secno);  in f2fs_clear_prefree_segments()
2023 		if (!IS_CURSEC(sbi, secno) &&  in f2fs_clear_prefree_segments()
2024 			!get_valid_blocks(sbi, start, true))  in f2fs_clear_prefree_segments()
2025 			f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),  in f2fs_clear_prefree_segments()
2026 				sbi->segs_per_sec << sbi->log_blocks_per_seg);  in f2fs_clear_prefree_segments()
2028 		start = start_segno + sbi->segs_per_sec;  in f2fs_clear_prefree_segments()
2044 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2047 			if (f2fs_sb_has_blkzoned(sbi) ||  in f2fs_clear_prefree_segments()
2051 			f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,  in f2fs_clear_prefree_segments()
2056 					sbi->blocks_per_seg, cur_pos);  in f2fs_clear_prefree_segments()
2062 		if (cur_pos < sbi->blocks_per_seg)  in f2fs_clear_prefree_segments()
2069 	wake_up_discard_thread(sbi, false);  in f2fs_clear_prefree_segments()
2072 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)  in create_discard_cmd_control()  argument
2074 	dev_t dev = sbi->sb->s_bdev->bd_dev;  in create_discard_cmd_control()
2078 	if (SM_I(sbi)->dcc_info) {  in create_discard_cmd_control()
2079 		dcc = SM_I(sbi)->dcc_info;  in create_discard_cmd_control()
2083 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);  in create_discard_cmd_control()
2098 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;  in create_discard_cmd_control()
2105 	SM_I(sbi)->dcc_info = dcc;  in create_discard_cmd_control()
2107 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,  in create_discard_cmd_control()
2112 		SM_I(sbi)->dcc_info = NULL;  in create_discard_cmd_control()
2119 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)  in destroy_discard_cmd_control()  argument
2121 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in destroy_discard_cmd_control()
2126 	f2fs_stop_discard_thread(sbi);  in destroy_discard_cmd_control()
2133 		f2fs_issue_discard_timeout(sbi);  in destroy_discard_cmd_control()
2136 	SM_I(sbi)->dcc_info = NULL;  in destroy_discard_cmd_control()
2139 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)  in __mark_sit_entry_dirty()  argument
2141 	struct sit_info *sit_i = SIT_I(sbi);  in __mark_sit_entry_dirty()
2151 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,  in __set_sit_entry_type()  argument
2154 	struct seg_entry *se = get_seg_entry(sbi, segno);  in __set_sit_entry_type()
2157 		__mark_sit_entry_dirty(sbi, segno);  in __set_sit_entry_type()
2160 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,  in get_segment_mtime()  argument
2163 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in get_segment_mtime()
2167 	return get_seg_entry(sbi, segno)->mtime;  in get_segment_mtime()
2170 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,  in update_segment_mtime()  argument
2174 	unsigned int segno = GET_SEGNO(sbi, blkaddr);  in update_segment_mtime()
2175 	unsigned long long ctime = get_mtime(sbi, false);  in update_segment_mtime()
2181 	se = get_seg_entry(sbi, segno);  in update_segment_mtime()
2189 	if (ctime > SIT_I(sbi)->max_mtime)  in update_segment_mtime()
2190 		SIT_I(sbi)->max_mtime = ctime;  in update_segment_mtime()
2193 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  in update_sit_entry()  argument
2203 	segno = GET_SEGNO(sbi, blkaddr);  in update_sit_entry()
2205 	se = get_seg_entry(sbi, segno);  in update_sit_entry()
2207 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in update_sit_entry()
2209 	f2fs_bug_on(sbi, (new_vblocks < 0 ||  in update_sit_entry()
2210 			(new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));  in update_sit_entry()
2221 			f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2223 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2227 			f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",  in update_sit_entry()
2229 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2235 			sbi->discard_blks--;  in update_sit_entry()
2241 		if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {  in update_sit_entry()
2251 			f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",  in update_sit_entry()
2253 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2257 			f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",  in update_sit_entry()
2259 			f2fs_bug_on(sbi, 1);  in update_sit_entry()
2262 		} else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in update_sit_entry()
2270 				spin_lock(&sbi->stat_lock);  in update_sit_entry()
2271 				sbi->unusable_block_count++;  in update_sit_entry()
2272 				spin_unlock(&sbi->stat_lock);  in update_sit_entry()
2277 			sbi->discard_blks++;  in update_sit_entry()
2282 	__mark_sit_entry_dirty(sbi, segno);  in update_sit_entry()
2285 	SIT_I(sbi)->written_valid_blocks += del;  in update_sit_entry()
2287 	if (__is_large_section(sbi))  in update_sit_entry()
2288 		get_sec_entry(sbi, segno)->valid_blocks += del;  in update_sit_entry()
2291 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)  in f2fs_invalidate_blocks()  argument
2293 	unsigned int segno = GET_SEGNO(sbi, addr);  in f2fs_invalidate_blocks()
2294 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_invalidate_blocks()
2296 	f2fs_bug_on(sbi, addr == NULL_ADDR);  in f2fs_invalidate_blocks()
2300 	invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);  in f2fs_invalidate_blocks()
2305 	update_segment_mtime(sbi, addr, 0);  in f2fs_invalidate_blocks()
2306 	update_sit_entry(sbi, addr, -1);  in f2fs_invalidate_blocks()
2309 	locate_dirty_segment(sbi, segno);  in f2fs_invalidate_blocks()
2314 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)  in f2fs_is_checkpointed_data()  argument
2316 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_is_checkpointed_data()
2326 	segno = GET_SEGNO(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2327 	se = get_seg_entry(sbi, segno);  in f2fs_is_checkpointed_data()
2328 	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);  in f2fs_is_checkpointed_data()
2341 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,  in __add_sum_entry()  argument
2344 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __add_sum_entry()
2353 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)  in f2fs_npages_for_summary_flush()  argument
2359 		if (sbi->ckpt->alloc_type[i] == SSR)  in f2fs_npages_for_summary_flush()
2360 			valid_sum_count += sbi->blocks_per_seg;  in f2fs_npages_for_summary_flush()
2364 					F2FS_CKPT(sbi)->cur_data_blkoff[i]);  in f2fs_npages_for_summary_flush()
2366 				valid_sum_count += curseg_blkoff(sbi, i);  in f2fs_npages_for_summary_flush()
2383 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_get_sum_page()  argument
2385 	if (unlikely(f2fs_cp_error(sbi)))  in f2fs_get_sum_page()
2387 	return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));  in f2fs_get_sum_page()
2390 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,  in f2fs_update_meta_page()  argument
2393 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in f2fs_update_meta_page()
2400 static void write_sum_page(struct f2fs_sb_info *sbi,  in write_sum_page()  argument
2403 	f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);  in write_sum_page()
2406 static void write_current_sum_page(struct f2fs_sb_info *sbi,  in write_current_sum_page()  argument
2409 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in write_current_sum_page()
2410 	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);  in write_current_sum_page()
2432 static int is_next_segment_free(struct f2fs_sb_info *sbi,  in is_next_segment_free()  argument
2436 	struct free_segmap_info *free_i = FREE_I(sbi);  in is_next_segment_free()
2438 	if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)  in is_next_segment_free()
2447 static void get_new_segment(struct f2fs_sb_info *sbi,  in get_new_segment()  argument
2450 	struct free_segmap_info *free_i = FREE_I(sbi);  in get_new_segment()
2452 	unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;  in get_new_segment()
2453 	unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);  in get_new_segment()
2454 	unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);  in get_new_segment()
2462 	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {  in get_new_segment()
2464 			GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);  in get_new_segment()
2465 		if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))  in get_new_segment()
2469 	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);  in get_new_segment()
2470 	if (secno >= MAIN_SECS(sbi)) {  in get_new_segment()
2473 							MAIN_SECS(sbi), 0);  in get_new_segment()
2474 			f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));  in get_new_segment()
2489 							MAIN_SECS(sbi), 0);  in get_new_segment()
2490 		f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));  in get_new_segment()
2495 	segno = GET_SEG_FROM_SEC(sbi, secno);  in get_new_segment()
2496 	zoneno = GET_ZONE_FROM_SEC(sbi, secno);  in get_new_segment()
2501 	if (sbi->secs_per_zone == 1)  in get_new_segment()
2512 		if (CURSEG_I(sbi, i)->zone == zoneno)  in get_new_segment()
2518 			hint = zoneno * sbi->secs_per_zone - 1;  in get_new_segment()
2522 			hint = (zoneno + 1) * sbi->secs_per_zone;  in get_new_segment()
2528 	f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));  in get_new_segment()
2529 	__set_inuse(sbi, segno);  in get_new_segment()
2534 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)  in reset_curseg()  argument
2536 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in reset_curseg()
2542 	curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);  in reset_curseg()
2549 	sanity_check_seg_type(sbi, seg_type);  in reset_curseg()
2555 	__set_sit_entry_type(sbi, seg_type, curseg->segno, modified);  in reset_curseg()
2558 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)  in __get_next_segno()  argument
2560 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __get_next_segno()
2563 	sanity_check_seg_type(sbi, seg_type);  in __get_next_segno()
2566 	if (__is_large_section(sbi))  in __get_next_segno()
2573 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in __get_next_segno()
2576 	if (test_opt(sbi, NOHEAP) &&  in __get_next_segno()
2580 	if (SIT_I(sbi)->last_victim[ALLOC_NEXT])  in __get_next_segno()
2581 		return SIT_I(sbi)->last_victim[ALLOC_NEXT];  in __get_next_segno()
2584 	if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)  in __get_next_segno()
2594 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)  in new_curseg()  argument
2596 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in new_curseg()
2602 		write_sum_page(sbi, curseg->sum_blk,  in new_curseg()
2603 				GET_SUM_BLOCK(sbi, segno));  in new_curseg()
2607 	if (test_opt(sbi, NOHEAP))  in new_curseg()
2610 	segno = __get_next_segno(sbi, type);  in new_curseg()
2611 	get_new_segment(sbi, &segno, new_sec, dir);  in new_curseg()
2613 	reset_curseg(sbi, type, 1);  in new_curseg()
2617 static void __next_free_blkoff(struct f2fs_sb_info *sbi,  in __next_free_blkoff()  argument
2620 	struct seg_entry *se = get_seg_entry(sbi, seg->segno);  in __next_free_blkoff()
2622 	unsigned long *target_map = SIT_I(sbi)->tmp_map;  in __next_free_blkoff()
2630 	pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);  in __next_free_blkoff()
2640 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,  in __refresh_next_blkoff()  argument
2644 		__next_free_blkoff(sbi, seg, seg->next_blkoff + 1);  in __refresh_next_blkoff()
2653 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)  in change_curseg()  argument
2655 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in change_curseg()
2656 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in change_curseg()
2662 		write_sum_page(sbi, curseg->sum_blk,  in change_curseg()
2663 					GET_SUM_BLOCK(sbi, curseg->segno));  in change_curseg()
2665 	__set_test_and_inuse(sbi, new_segno);  in change_curseg()
2668 	__remove_dirty_segment(sbi, new_segno, PRE);  in change_curseg()
2669 	__remove_dirty_segment(sbi, new_segno, DIRTY);  in change_curseg()
2672 	reset_curseg(sbi, type, 1);  in change_curseg()
2674 	__next_free_blkoff(sbi, curseg, 0);  in change_curseg()
2676 	sum_page = f2fs_get_sum_page(sbi, new_segno);  in change_curseg()
2687 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2690 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,  in get_atssr_segment()  argument
2694 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_atssr_segment()
2698 	if (get_ssr_segment(sbi, type, alloc_mode, age)) {  in get_atssr_segment()
2699 		struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);  in get_atssr_segment()
2702 		change_curseg(sbi, type, true);  in get_atssr_segment()
2706 		new_curseg(sbi, type, true);  in get_atssr_segment()
2708 	stat_inc_seg_type(sbi, curseg);  in get_atssr_segment()
2711 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)  in __f2fs_init_atgc_curseg()  argument
2713 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);  in __f2fs_init_atgc_curseg()
2715 	if (!sbi->am.atgc_enabled)  in __f2fs_init_atgc_curseg()
2718 	down_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2721 	down_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2723 	get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);  in __f2fs_init_atgc_curseg()
2725 	up_write(&SIT_I(sbi)->sentry_lock);  in __f2fs_init_atgc_curseg()
2728 	up_read(&SM_I(sbi)->curseg_lock);  in __f2fs_init_atgc_curseg()
2731 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_init_inmem_curseg()  argument
2733 	__f2fs_init_atgc_curseg(sbi);  in f2fs_init_inmem_curseg()
2736 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_save_inmem_curseg()  argument
2738 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_save_inmem_curseg()
2744 	if (get_valid_blocks(sbi, curseg->segno, false)) {  in __f2fs_save_inmem_curseg()
2745 		write_sum_page(sbi, curseg->sum_blk,  in __f2fs_save_inmem_curseg()
2746 				GET_SUM_BLOCK(sbi, curseg->segno));  in __f2fs_save_inmem_curseg()
2748 		mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2749 		__set_test_and_free(sbi, curseg->segno, true);  in __f2fs_save_inmem_curseg()
2750 		mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_save_inmem_curseg()
2756 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_save_inmem_curseg()  argument
2758 	__f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_save_inmem_curseg()
2760 	if (sbi->am.atgc_enabled)  in f2fs_save_inmem_curseg()
2761 		__f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_save_inmem_curseg()
2764 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)  in __f2fs_restore_inmem_curseg()  argument
2766 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __f2fs_restore_inmem_curseg()
2771 	if (get_valid_blocks(sbi, curseg->segno, false))  in __f2fs_restore_inmem_curseg()
2774 	mutex_lock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2775 	__set_test_and_inuse(sbi, curseg->segno);  in __f2fs_restore_inmem_curseg()
2776 	mutex_unlock(&DIRTY_I(sbi)->seglist_lock);  in __f2fs_restore_inmem_curseg()
2781 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)  in f2fs_restore_inmem_curseg()  argument
2783 	__f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);  in f2fs_restore_inmem_curseg()
2785 	if (sbi->am.atgc_enabled)  in f2fs_restore_inmem_curseg()
2786 		__f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);  in f2fs_restore_inmem_curseg()
2789 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,  in get_ssr_segment()  argument
2792 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in get_ssr_segment()
2793 	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;  in get_ssr_segment()
2799 	sanity_check_seg_type(sbi, seg_type);  in get_ssr_segment()
2802 	if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {  in get_ssr_segment()
2829 		if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {  in get_ssr_segment()
2836 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {  in get_ssr_segment()
2837 		segno = get_free_segment(sbi);  in get_ssr_segment()
2850 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,  in allocate_segment_by_default()  argument
2853 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in allocate_segment_by_default()
2856 		new_curseg(sbi, type, true);  in allocate_segment_by_default()
2857 	else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&  in allocate_segment_by_default()
2859 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2861 			is_next_segment_free(sbi, curseg, type) &&  in allocate_segment_by_default()
2862 			likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))  in allocate_segment_by_default()
2863 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2864 	else if (f2fs_need_SSR(sbi) &&  in allocate_segment_by_default()
2865 			get_ssr_segment(sbi, type, SSR, 0))  in allocate_segment_by_default()
2866 		change_curseg(sbi, type, true);  in allocate_segment_by_default()
2868 		new_curseg(sbi, type, false);  in allocate_segment_by_default()
2870 	stat_inc_seg_type(sbi, curseg);  in allocate_segment_by_default()
2873 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,  in f2fs_allocate_segment_for_resize()  argument
2876 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_segment_for_resize()
2879 	down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
2881 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
2883 	segno = CURSEG_I(sbi, type)->segno;  in f2fs_allocate_segment_for_resize()
2887 	if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))  in f2fs_allocate_segment_for_resize()
2888 		change_curseg(sbi, type, true);  in f2fs_allocate_segment_for_resize()
2890 		new_curseg(sbi, type, true);  in f2fs_allocate_segment_for_resize()
2892 	stat_inc_seg_type(sbi, curseg);  in f2fs_allocate_segment_for_resize()
2894 	locate_dirty_segment(sbi, segno);  in f2fs_allocate_segment_for_resize()
2896 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_segment_for_resize()
2899 		f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",  in f2fs_allocate_segment_for_resize()
2903 	up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_segment_for_resize()
2906 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type)  in __allocate_new_segment()  argument
2908 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in __allocate_new_segment()
2915 		!get_valid_blocks(sbi, curseg->segno, false) &&  in __allocate_new_segment()
2916 		!get_ckpt_valid_blocks(sbi, curseg->segno))  in __allocate_new_segment()
2921 	SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);  in __allocate_new_segment()
2922 	locate_dirty_segment(sbi, old_segno);  in __allocate_new_segment()
2925 void f2fs_allocate_new_segment(struct f2fs_sb_info *sbi, int type)  in f2fs_allocate_new_segment()  argument
2927 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segment()
2928 	__allocate_new_segment(sbi, type);  in f2fs_allocate_new_segment()
2929 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segment()
2932 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)  in f2fs_allocate_new_segments()  argument
2936 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
2938 		__allocate_new_segment(sbi, i);  in f2fs_allocate_new_segments()
2939 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_allocate_new_segments()
2946 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,  in f2fs_exist_trim_candidates()  argument
2952 	down_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
2954 		if (add_discard_addrs(sbi, cpc, true)) {  in f2fs_exist_trim_candidates()
2959 	up_write(&SIT_I(sbi)->sentry_lock);  in f2fs_exist_trim_candidates()
2965 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,  in __issue_discard_cmd_range()  argument
2969 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_range()
2982 		f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,  in __issue_discard_cmd_range()
3007 		err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);  in __issue_discard_cmd_range()
3013 				__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3017 			trimmed += __wait_all_discard_cmd(sbi, NULL);  in __issue_discard_cmd_range()
3024 			__remove_discard_cmd(sbi, dc);  in __issue_discard_cmd_range()
3037 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)  in f2fs_trim_fs()  argument
3047 	bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);  in f2fs_trim_fs()
3049 	if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)  in f2fs_trim_fs()
3052 	if (end < MAIN_BLKADDR(sbi))  in f2fs_trim_fs()
3055 	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {  in f2fs_trim_fs()
3056 		f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");  in f2fs_trim_fs()
3061 	start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);  in f2fs_trim_fs()
3062 	end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :  in f2fs_trim_fs()
3063 						GET_SEGNO(sbi, end);  in f2fs_trim_fs()
3065 		start_segno = rounddown(start_segno, sbi->segs_per_sec);  in f2fs_trim_fs()
3066 		end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;  in f2fs_trim_fs()
3074 	if (sbi->discard_blks == 0)  in f2fs_trim_fs()
3077 	down_write(&sbi->gc_lock);  in f2fs_trim_fs()
3078 	err = f2fs_write_checkpoint(sbi, &cpc);  in f2fs_trim_fs()
3079 	up_write(&sbi->gc_lock);  in f2fs_trim_fs()
3089 	if (f2fs_realtime_discard_enable(sbi))  in f2fs_trim_fs()
3092 	start_block = START_BLOCK(sbi, start_segno);  in f2fs_trim_fs()
3093 	end_block = START_BLOCK(sbi, end_segno + 1);  in f2fs_trim_fs()
3095 	__init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);  in f2fs_trim_fs()
3096 	trimmed = __issue_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3099 	trimmed += __wait_discard_cmd_range(sbi, &dpolicy,  in f2fs_trim_fs()
3107 static bool __has_curseg_space(struct f2fs_sb_info *sbi,  in __has_curseg_space()  argument
3110 	return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,  in __has_curseg_space()
3187 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,  in f2fs_io_type_to_rw_hint()  argument
3190 	if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {  in f2fs_io_type_to_rw_hint()
3201 	} else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {  in f2fs_io_type_to_rw_hint()
3252 			if (fio->sbi->am.atgc_enabled)  in __get_segment_type_6()
3277 	switch (F2FS_OPTION(fio->sbi).active_logs) {  in __get_segment_type()
3288 		f2fs_bug_on(fio->sbi, true);  in __get_segment_type()
3300 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_allocate_data_block()  argument
3305 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_allocate_data_block()
3306 	struct curseg_info *curseg = CURSEG_I(sbi, type);  in f2fs_allocate_data_block()
3311 	down_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3317 		f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);  in f2fs_allocate_data_block()
3318 		se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3319 		sanity_check_seg_type(sbi, se->type);  in f2fs_allocate_data_block()
3320 		f2fs_bug_on(sbi, IS_NODESEG(se->type));  in f2fs_allocate_data_block()
3322 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);  in f2fs_allocate_data_block()
3324 	f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);  in f2fs_allocate_data_block()
3326 	f2fs_wait_discard_bio(sbi, *new_blkaddr);  in f2fs_allocate_data_block()
3333 	__add_sum_entry(sbi, type, sum);  in f2fs_allocate_data_block()
3335 	__refresh_next_blkoff(sbi, curseg);  in f2fs_allocate_data_block()
3337 	stat_inc_block_count(sbi, curseg);  in f2fs_allocate_data_block()
3340 		old_mtime = get_segment_mtime(sbi, old_blkaddr);  in f2fs_allocate_data_block()
3342 		update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_allocate_data_block()
3345 	update_segment_mtime(sbi, *new_blkaddr, old_mtime);  in f2fs_allocate_data_block()
3351 	update_sit_entry(sbi, *new_blkaddr, 1);  in f2fs_allocate_data_block()
3352 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)  in f2fs_allocate_data_block()
3353 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_allocate_data_block()
3355 	if (!__has_curseg_space(sbi, curseg)) {  in f2fs_allocate_data_block()
3357 			get_atssr_segment(sbi, type, se->type,  in f2fs_allocate_data_block()
3360 			sit_i->s_ops->allocate_segment(sbi, type, false);  in f2fs_allocate_data_block()
3367 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_allocate_data_block()
3368 	locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));  in f2fs_allocate_data_block()
3373 		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));  in f2fs_allocate_data_block()
3375 		f2fs_inode_chksum_set(sbi, page);  in f2fs_allocate_data_block()
3378 	if (F2FS_IO_ALIGNED(sbi))  in f2fs_allocate_data_block()
3386 		io = sbi->write_io[fio->type] + fio->temp;  in f2fs_allocate_data_block()
3394 	up_read(&SM_I(sbi)->curseg_lock);  in f2fs_allocate_data_block()
3399 	struct f2fs_sb_info *sbi = fio->sbi;  in update_device_state()  local
3402 	if (!f2fs_is_multi_device(sbi))  in update_device_state()
3405 	devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);  in update_device_state()
3408 	f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);  in update_device_state()
3411 	if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {  in update_device_state()
3412 		spin_lock(&sbi->dev_lock);  in update_device_state()
3413 		f2fs_set_bit(devidx, (char *)&sbi->dirty_device);  in update_device_state()
3414 		spin_unlock(&sbi->dev_lock);  in update_device_state()
3421 	bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);  in do_write_page()
3424 		down_read(&fio->sbi->io_order_lock);  in do_write_page()
3426 	f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,  in do_write_page()
3428 	if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)  in do_write_page()
3429 		invalidate_mapping_pages(META_MAPPING(fio->sbi),  in do_write_page()
3442 		up_read(&fio->sbi->io_order_lock);  in do_write_page()
3445 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,  in f2fs_do_write_meta_page()  argument
3449 		.sbi = sbi,  in f2fs_do_write_meta_page()
3461 	if (unlikely(page->index >= MAIN_BLKADDR(sbi)))  in f2fs_do_write_meta_page()
3468 	stat_inc_meta_count(sbi, page->index);  in f2fs_do_write_meta_page()
3469 	f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);  in f2fs_do_write_meta_page()
3479 	f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_do_write_node_page()
3485 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_outplace_write_data()  local
3488 	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);  in f2fs_outplace_write_data()
3493 	f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_outplace_write_data()
3499 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_inplace_write_data()  local
3506 	segno = GET_SEGNO(sbi, fio->new_blkaddr);  in f2fs_inplace_write_data()
3508 	if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {  in f2fs_inplace_write_data()
3509 		set_sbi_flag(sbi, SBI_NEED_FSCK);  in f2fs_inplace_write_data()
3510 		f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",  in f2fs_inplace_write_data()
3515 	stat_inc_inplace_blocks(fio->sbi);  in f2fs_inplace_write_data()
3517 	if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))  in f2fs_inplace_write_data()
3523 		f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);  in f2fs_inplace_write_data()
3529 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,  in __f2fs_get_curseg()  argument
3535 		if (CURSEG_I(sbi, i)->segno == segno)  in __f2fs_get_curseg()
3541 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,  in f2fs_do_replace_block()  argument
3546 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_do_replace_block()
3553 	segno = GET_SEGNO(sbi, new_blkaddr);  in f2fs_do_replace_block()
3554 	se = get_seg_entry(sbi, segno);  in f2fs_do_replace_block()
3557 	down_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3561 		if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3568 		if (IS_CURSEG(sbi, segno)) {  in f2fs_do_replace_block()
3570 			type = __f2fs_get_curseg(sbi, segno);  in f2fs_do_replace_block()
3571 			f2fs_bug_on(sbi, type == NO_CHECK_TYPE);  in f2fs_do_replace_block()
3577 	f2fs_bug_on(sbi, !IS_DATASEG(type));  in f2fs_do_replace_block()
3578 	curseg = CURSEG_I(sbi, type);  in f2fs_do_replace_block()
3589 		change_curseg(sbi, type, true);  in f2fs_do_replace_block()
3592 	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);  in f2fs_do_replace_block()
3593 	__add_sum_entry(sbi, type, sum);  in f2fs_do_replace_block()
3597 			update_segment_mtime(sbi, new_blkaddr, 0);  in f2fs_do_replace_block()
3598 		update_sit_entry(sbi, new_blkaddr, 1);  in f2fs_do_replace_block()
3600 	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {  in f2fs_do_replace_block()
3601 		invalidate_mapping_pages(META_MAPPING(sbi),  in f2fs_do_replace_block()
3604 			update_segment_mtime(sbi, old_blkaddr, 0);  in f2fs_do_replace_block()
3605 		update_sit_entry(sbi, old_blkaddr, -1);  in f2fs_do_replace_block()
3608 	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));  in f2fs_do_replace_block()
3609 	locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));  in f2fs_do_replace_block()
3611 	locate_dirty_segment(sbi, old_cursegno);  in f2fs_do_replace_block()
3616 			change_curseg(sbi, type, true);  in f2fs_do_replace_block()
3623 	up_write(&SM_I(sbi)->curseg_lock);  in f2fs_do_replace_block()
3626 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,  in f2fs_replace_block()  argument
3635 	f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,  in f2fs_replace_block()
3645 		struct f2fs_sb_info *sbi = F2FS_P_SB(page);  in f2fs_wait_on_page_writeback()  local
3648 		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);  in f2fs_wait_on_page_writeback()
3650 		f2fs_submit_merged_ipu_write(sbi, NULL, page);  in f2fs_wait_on_page_writeback()
3653 			f2fs_bug_on(sbi, locked && PageWriteback(page));  in f2fs_wait_on_page_writeback()
3662 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);  in f2fs_wait_on_block_writeback()  local
3671 	cpage = find_lock_page(META_MAPPING(sbi), blkaddr);  in f2fs_wait_on_block_writeback()
3687 static int read_compacted_summaries(struct f2fs_sb_info *sbi)  in read_compacted_summaries()  argument
3689 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_compacted_summaries()
3696 	start = start_sum_block(sbi);  in read_compacted_summaries()
3698 	page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3704 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in read_compacted_summaries()
3708 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in read_compacted_summaries()
3717 		seg_i = CURSEG_I(sbi, i);  in read_compacted_summaries()
3721 		reset_curseg(sbi, i, 0);  in read_compacted_summaries()
3726 			blk_off = sbi->blocks_per_seg;  in read_compacted_summaries()
3740 			page = f2fs_get_meta_page(sbi, start++);  in read_compacted_summaries()
3751 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)  in read_normal_summaries()  argument
3753 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in read_normal_summaries()
3767 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3768 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);  in read_normal_summaries()
3770 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);  in read_normal_summaries()
3776 		if (__exist_node_summaries(sbi))  in read_normal_summaries()
3777 			blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,  in read_normal_summaries()
3780 			blk_addr = GET_SUM_BLOCK(sbi, segno);  in read_normal_summaries()
3783 	new = f2fs_get_meta_page(sbi, blk_addr);  in read_normal_summaries()
3789 		if (__exist_node_summaries(sbi)) {  in read_normal_summaries()
3792 			for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {  in read_normal_summaries()
3797 			err = f2fs_restore_node_summary(sbi, segno, sum);  in read_normal_summaries()
3804 	curseg = CURSEG_I(sbi, type);  in read_normal_summaries()
3815 	reset_curseg(sbi, type, 0);  in read_normal_summaries()
3824 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)  in restore_curseg_summaries()  argument
3826 	struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;  in restore_curseg_summaries()
3827 	struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;  in restore_curseg_summaries()
3831 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {  in restore_curseg_summaries()
3832 		int npages = f2fs_npages_for_summary_flush(sbi, true);  in restore_curseg_summaries()
3835 			f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,  in restore_curseg_summaries()
3839 		err = read_compacted_summaries(sbi);  in restore_curseg_summaries()
3845 	if (__exist_node_summaries(sbi))  in restore_curseg_summaries()
3846 		f2fs_ra_meta_pages(sbi,  in restore_curseg_summaries()
3847 				sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),  in restore_curseg_summaries()
3851 		err = read_normal_summaries(sbi, type);  in restore_curseg_summaries()
3859 		f2fs_err(sbi, "invalid journal entries nats %u sits %u\n",  in restore_curseg_summaries()
3867 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)  in write_compacted_summaries()  argument
3876 	page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
3881 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);  in write_compacted_summaries()
3886 	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);  in write_compacted_summaries()
3893 		seg_i = CURSEG_I(sbi, i);  in write_compacted_summaries()
3894 		if (sbi->ckpt->alloc_type[i] == SSR)  in write_compacted_summaries()
3895 			blkoff = sbi->blocks_per_seg;  in write_compacted_summaries()
3897 			blkoff = curseg_blkoff(sbi, i);  in write_compacted_summaries()
3901 				page = f2fs_grab_meta_page(sbi, blkaddr++);  in write_compacted_summaries()
3925 static void write_normal_summaries(struct f2fs_sb_info *sbi,  in write_normal_summaries()  argument
3935 		write_current_sum_page(sbi, i, blkaddr + (i - type));  in write_normal_summaries()
3938 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_data_summaries()  argument
3940 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))  in f2fs_write_data_summaries()
3941 		write_compacted_summaries(sbi, start_blk);  in f2fs_write_data_summaries()
3943 		write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);  in f2fs_write_data_summaries()
3946 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)  in f2fs_write_node_summaries()  argument
3948 	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);  in f2fs_write_node_summaries()
3973 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,  in get_current_sit_page()  argument
3976 	return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));  in get_current_sit_page()
3979 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,  in get_next_sit_page()  argument
3982 	struct sit_info *sit_i = SIT_I(sbi);  in get_next_sit_page()
3986 	src_off = current_sit_addr(sbi, start);  in get_next_sit_page()
3987 	dst_off = next_sit_addr(sbi, src_off);  in get_next_sit_page()
3989 	page = f2fs_grab_meta_page(sbi, dst_off);  in get_next_sit_page()
3990 	seg_info_to_sit_page(sbi, page, start);  in get_next_sit_page()
4049 static void add_sits_in_set(struct f2fs_sb_info *sbi)  in add_sits_in_set()  argument
4051 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in add_sits_in_set()
4053 	unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;  in add_sits_in_set()
4056 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))  in add_sits_in_set()
4060 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)  in remove_sits_in_journal()  argument
4062 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in remove_sits_in_journal()
4072 		dirtied = __mark_sit_entry_dirty(sbi, segno);  in remove_sits_in_journal()
4075 			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);  in remove_sits_in_journal()
4085 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)  in f2fs_flush_sit_entries()  argument
4087 	struct sit_info *sit_i = SIT_I(sbi);  in f2fs_flush_sit_entries()
4089 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in f2fs_flush_sit_entries()
4092 	struct list_head *head = &SM_I(sbi)->sit_entry_set;  in f2fs_flush_sit_entries()
4093 	bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);  in f2fs_flush_sit_entries()
4105 	add_sits_in_set(sbi);  in f2fs_flush_sit_entries()
4114 		remove_sits_in_journal(sbi);  in f2fs_flush_sit_entries()
4126 						(unsigned long)MAIN_SEGS(sbi));  in f2fs_flush_sit_entries()
4136 			page = get_next_sit_page(sbi, start_segno);  in f2fs_flush_sit_entries()
4144 			se = get_seg_entry(sbi, segno);  in f2fs_flush_sit_entries()
4148 				f2fs_bug_on(sbi, 1);  in f2fs_flush_sit_entries()
4154 				add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4160 				f2fs_bug_on(sbi, offset < 0);  in f2fs_flush_sit_entries()
4165 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4171 				check_block_count(sbi, segno,  in f2fs_flush_sit_entries()
4185 		f2fs_bug_on(sbi, ses->entry_cnt);  in f2fs_flush_sit_entries()
4189 	f2fs_bug_on(sbi, !list_empty(head));  in f2fs_flush_sit_entries()
4190 	f2fs_bug_on(sbi, sit_i->dirty_sentries);  in f2fs_flush_sit_entries()
4196 			add_discard_addrs(sbi, cpc, false);  in f2fs_flush_sit_entries()
4202 	set_prefree_as_free_segments(sbi);  in f2fs_flush_sit_entries()
4205 static int build_sit_info(struct f2fs_sb_info *sbi)  in build_sit_info()  argument
4207 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in build_sit_info()
4214 	sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);  in build_sit_info()
4218 	SM_I(sbi)->sit_info = sit_i;  in build_sit_info()
4221 		f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),  in build_sit_info()
4222 					      MAIN_SEGS(sbi)),  in build_sit_info()
4227 	main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_sit_info()
4228 	sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,  in build_sit_info()
4234 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;  in build_sit_info()
4236 	bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;  in build_sit_info()
4238 	sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in build_sit_info()
4244 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in build_sit_info()
4260 	sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);  in build_sit_info()
4264 	if (__is_large_section(sbi)) {  in build_sit_info()
4266 			f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),  in build_sit_info()
4267 						      MAIN_SECS(sbi)),  in build_sit_info()
4277 	sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);  in build_sit_info()
4278 	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);  in build_sit_info()
4290 	sit_i->invalid_segmap = f2fs_kvzalloc(sbi,  in build_sit_info()
4300 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;  in build_sit_info()
4305 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);  in build_sit_info()
4311 static int build_free_segmap(struct f2fs_sb_info *sbi)  in build_free_segmap()  argument
4317 	free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);  in build_free_segmap()
4321 	SM_I(sbi)->free_info = free_i;  in build_free_segmap()
4323 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_free_segmap()
4324 	free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);  in build_free_segmap()
4328 	sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_free_segmap()
4329 	free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);  in build_free_segmap()
4338 	free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));  in build_free_segmap()
4345 static int build_curseg(struct f2fs_sb_info *sbi)  in build_curseg()  argument
4350 	array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,  in build_curseg()
4355 	SM_I(sbi)->curseg_array = array;  in build_curseg()
4359 		array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);  in build_curseg()
4363 		array[i].journal = f2fs_kzalloc(sbi,  in build_curseg()
4377 	return restore_curseg_summaries(sbi);  in build_curseg()
4380 static int build_sit_entries(struct f2fs_sb_info *sbi)  in build_sit_entries()  argument
4382 	struct sit_info *sit_i = SIT_I(sbi);  in build_sit_entries()
4383 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);  in build_sit_entries()
4387 	int sit_blk_cnt = SIT_BLK_CNT(sbi);  in build_sit_entries()
4394 		readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,  in build_sit_entries()
4400 		for (; start < end && start < MAIN_SEGS(sbi); start++) {  in build_sit_entries()
4405 			page = get_current_sit_page(sbi, start);  in build_sit_entries()
4412 			err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4420 			if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4427 				sbi->discard_blks +=  in build_sit_entries()
4428 					sbi->blocks_per_seg -  in build_sit_entries()
4432 			if (__is_large_section(sbi))  in build_sit_entries()
4433 				get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4444 		if (start >= MAIN_SEGS(sbi)) {  in build_sit_entries()
4445 			f2fs_err(sbi, "Wrong journal entry on segno %u",  in build_sit_entries()
4458 		err = check_block_count(sbi, start, &sit);  in build_sit_entries()
4465 		if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {  in build_sit_entries()
4470 			sbi->discard_blks += old_valid_blocks;  in build_sit_entries()
4471 			sbi->discard_blks -= se->valid_blocks;  in build_sit_entries()
4474 		if (__is_large_section(sbi)) {  in build_sit_entries()
4475 			get_sec_entry(sbi, start)->valid_blocks +=  in build_sit_entries()
4477 			get_sec_entry(sbi, start)->valid_blocks -=  in build_sit_entries()
4483 	if (!err && total_node_blocks != valid_node_count(sbi)) {  in build_sit_entries()
4484 		f2fs_err(sbi, "SIT is corrupted node# %u vs %u",  in build_sit_entries()
4485 			 total_node_blocks, valid_node_count(sbi));  in build_sit_entries()
4492 static void init_free_segmap(struct f2fs_sb_info *sbi)  in init_free_segmap()  argument
4498 	for (start = 0; start < MAIN_SEGS(sbi); start++) {  in init_free_segmap()
4499 		if (f2fs_usable_blks_in_seg(sbi, start) == 0)  in init_free_segmap()
4501 		sentry = get_seg_entry(sbi, start);  in init_free_segmap()
4503 			__set_free(sbi, start);  in init_free_segmap()
4505 			SIT_I(sbi)->written_valid_blocks +=  in init_free_segmap()
4511 		struct curseg_info *curseg_t = CURSEG_I(sbi, type);  in init_free_segmap()
4512 		__set_test_and_inuse(sbi, curseg_t->segno);  in init_free_segmap()
4516 static void init_dirty_segmap(struct f2fs_sb_info *sbi)  in init_dirty_segmap()  argument
4518 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_dirty_segmap()
4519 	struct free_segmap_info *free_i = FREE_I(sbi);  in init_dirty_segmap()
4522 	block_t blks_per_sec = BLKS_PER_SEC(sbi);  in init_dirty_segmap()
4526 		segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);  in init_dirty_segmap()
4527 		if (segno >= MAIN_SEGS(sbi))  in init_dirty_segmap()
4530 		valid_blocks = get_valid_blocks(sbi, segno, false);  in init_dirty_segmap()
4531 		usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);  in init_dirty_segmap()
4535 			f2fs_bug_on(sbi, 1);  in init_dirty_segmap()
4539 		__locate_dirty_segment(sbi, segno, DIRTY);  in init_dirty_segmap()
4543 	if (!__is_large_section(sbi))  in init_dirty_segmap()
4547 	for (segno = 0; segno < MAIN_SECS(sbi); segno += blks_per_sec) {  in init_dirty_segmap()
4548 		valid_blocks = get_valid_blocks(sbi, segno, true);  in init_dirty_segmap()
4549 		secno = GET_SEC_FROM_SEG(sbi, segno);  in init_dirty_segmap()
4553 		if (IS_CURSEC(sbi, secno))  in init_dirty_segmap()
4560 static int init_victim_secmap(struct f2fs_sb_info *sbi)  in init_victim_secmap()  argument
4562 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in init_victim_secmap()
4563 	unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in init_victim_secmap()
4565 	dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);  in init_victim_secmap()
4571 static int build_dirty_segmap(struct f2fs_sb_info *sbi)  in build_dirty_segmap()  argument
4577 	dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),  in build_dirty_segmap()
4582 	SM_I(sbi)->dirty_info = dirty_i;  in build_dirty_segmap()
4585 	bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));  in build_dirty_segmap()
4588 		dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,  in build_dirty_segmap()
4594 	if (__is_large_section(sbi)) {  in build_dirty_segmap()
4595 		bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));  in build_dirty_segmap()
4596 		dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,  in build_dirty_segmap()
4602 	init_dirty_segmap(sbi);  in build_dirty_segmap()
4603 	return init_victim_secmap(sbi);  in build_dirty_segmap()
4606 static int sanity_check_curseg(struct f2fs_sb_info *sbi)  in sanity_check_curseg()  argument
4615 		struct curseg_info *curseg = CURSEG_I(sbi, i);  in sanity_check_curseg()
4616 		struct seg_entry *se = get_seg_entry(sbi, curseg->segno);  in sanity_check_curseg()
4619 		sanity_check_seg_type(sbi, curseg->seg_type);  in sanity_check_curseg()
4627 		for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {  in sanity_check_curseg()
4631 			f2fs_err(sbi,  in sanity_check_curseg()
4643 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,  in check_zone_write_pointer()  argument
4649 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in check_zone_write_pointer()
4657 	wp_segno = GET_SEGNO(sbi, wp_block);  in check_zone_write_pointer()
4658 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in check_zone_write_pointer()
4660 	zone_segno = GET_SEGNO(sbi, zone_block);  in check_zone_write_pointer()
4661 	zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);  in check_zone_write_pointer()
4663 	if (zone_segno >= MAIN_SEGS(sbi))  in check_zone_write_pointer()
4671 		if (zone_secno == GET_SEC_FROM_SEG(sbi,  in check_zone_write_pointer()
4672 						   CURSEG_I(sbi, i)->segno))  in check_zone_write_pointer()
4679 	for (s = sbi->segs_per_sec - 1; s >= 0; s--) {  in check_zone_write_pointer()
4681 		se = get_seg_entry(sbi, segno);  in check_zone_write_pointer()
4682 		for (b = sbi->blocks_per_seg - 1; b >= 0; b--)  in check_zone_write_pointer()
4684 				last_valid_block = START_BLOCK(sbi, segno) + b;  in check_zone_write_pointer()
4698 		f2fs_notice(sbi, "Valid block beyond write pointer: "  in check_zone_write_pointer()
4700 			    GET_SEGNO(sbi, last_valid_block),  in check_zone_write_pointer()
4701 			    GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),  in check_zone_write_pointer()
4711 		f2fs_notice(sbi,  in check_zone_write_pointer()
4715 		ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,  in check_zone_write_pointer()
4718 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in check_zone_write_pointer()
4727 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,  in get_target_zoned_dev()  argument
4732 	for (i = 0; i < sbi->s_ndevs; i++) {  in get_target_zoned_dev()
4735 		if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&  in get_target_zoned_dev()
4749 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)  in fix_curseg_write_pointer()  argument
4751 	struct curseg_info *cs = CURSEG_I(sbi, type);  in fix_curseg_write_pointer()
4756 	unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;  in fix_curseg_write_pointer()
4760 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
4761 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
4763 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
4773 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
4782 	wp_segno = GET_SEGNO(sbi, wp_block);  in fix_curseg_write_pointer()
4783 	wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);  in fix_curseg_write_pointer()
4790 	f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "  in fix_curseg_write_pointer()
4794 	f2fs_notice(sbi, "Assign new section to curseg[%d]: "  in fix_curseg_write_pointer()
4796 	allocate_segment_by_default(sbi, type, true);  in fix_curseg_write_pointer()
4799 	if (check_zone_write_pointer(sbi, zbd, &zone))  in fix_curseg_write_pointer()
4803 	cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);  in fix_curseg_write_pointer()
4804 	cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));  in fix_curseg_write_pointer()
4806 	zbd = get_target_zoned_dev(sbi, cs_zone_block);  in fix_curseg_write_pointer()
4815 		f2fs_err(sbi, "Report zone failed: %s errno=(%d)",  in fix_curseg_write_pointer()
4824 		f2fs_notice(sbi,  in fix_curseg_write_pointer()
4828 		err = __f2fs_issue_discard_zone(sbi, zbd->bdev,  in fix_curseg_write_pointer()
4832 			f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",  in fix_curseg_write_pointer()
4841 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
4846 		ret = fix_curseg_write_pointer(sbi, i);  in f2fs_fix_curseg_write_pointer()
4855 	struct f2fs_sb_info *sbi;  member
4864 	return check_zone_write_pointer(args->sbi, args->fdev, zone);  in check_zone_write_pointer_cb()
4867 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
4872 	for (i = 0; i < sbi->s_ndevs; i++) {  in f2fs_check_write_pointer()
4876 		args.sbi = sbi;  in f2fs_check_write_pointer()
4887 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,  in is_conv_zone()  argument
4896 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,  in get_zone_idx()  argument
4899 	block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));  in get_zone_idx()
4902 						sbi->log_blocks_per_blkz;  in get_zone_idx()
4910 		struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_usable_zone_segs_in_sec()  argument
4914 	dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));  in f2fs_usable_zone_segs_in_sec()
4915 	zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);  in f2fs_usable_zone_segs_in_sec()
4918 	if (is_conv_zone(sbi, zone_idx, dev_idx))  in f2fs_usable_zone_segs_in_sec()
4919 		return sbi->segs_per_sec;  in f2fs_usable_zone_segs_in_sec()
4926 		return sbi->segs_per_sec;  in f2fs_usable_zone_segs_in_sec()
4929 	unusable_segs_in_sec = (sbi->blocks_per_blkz -  in f2fs_usable_zone_segs_in_sec()
4931 				sbi->log_blocks_per_seg;  in f2fs_usable_zone_segs_in_sec()
4932 	return sbi->segs_per_sec - unusable_segs_in_sec;  in f2fs_usable_zone_segs_in_sec()
4944 			struct f2fs_sb_info *sbi, unsigned int segno)  in f2fs_usable_zone_blks_in_seg()  argument
4949 	secno = GET_SEC_FROM_SEG(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
4950 	seg_start = START_BLOCK(sbi, segno);  in f2fs_usable_zone_blks_in_seg()
4951 	dev_idx = f2fs_target_device_index(sbi, seg_start);  in f2fs_usable_zone_blks_in_seg()
4952 	zone_idx = get_zone_idx(sbi, secno, dev_idx);  in f2fs_usable_zone_blks_in_seg()
4958 	if (is_conv_zone(sbi, zone_idx, dev_idx))  in f2fs_usable_zone_blks_in_seg()
4959 		return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
4962 		return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
4964 	sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));  in f2fs_usable_zone_blks_in_seg()
4976 	if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)  in f2fs_usable_zone_blks_in_seg()
4979 	return sbi->blocks_per_seg;  in f2fs_usable_zone_blks_in_seg()
4982 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_fix_curseg_write_pointer()  argument
4987 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)  in f2fs_check_write_pointer()  argument
4992 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_zone_blks_in_seg()  argument
4998 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,  in f2fs_usable_zone_segs_in_sec()  argument
5004 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,  in f2fs_usable_blks_in_seg()  argument
5007 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_blks_in_seg()
5008 		return f2fs_usable_zone_blks_in_seg(sbi, segno);  in f2fs_usable_blks_in_seg()
5010 	return sbi->blocks_per_seg;  in f2fs_usable_blks_in_seg()
5013 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,  in f2fs_usable_segs_in_sec()  argument
5016 	if (f2fs_sb_has_blkzoned(sbi))  in f2fs_usable_segs_in_sec()
5017 		return f2fs_usable_zone_segs_in_sec(sbi, segno);  in f2fs_usable_segs_in_sec()
5019 	return sbi->segs_per_sec;  in f2fs_usable_segs_in_sec()
5025 static void init_min_max_mtime(struct f2fs_sb_info *sbi)  in init_min_max_mtime()  argument
5027 	struct sit_info *sit_i = SIT_I(sbi);  in init_min_max_mtime()
5034 	for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {  in init_min_max_mtime()
5038 		for (i = 0; i < sbi->segs_per_sec; i++)  in init_min_max_mtime()
5039 			mtime += get_seg_entry(sbi, segno + i)->mtime;  in init_min_max_mtime()
5041 		mtime = div_u64(mtime, sbi->segs_per_sec);  in init_min_max_mtime()
5046 	sit_i->max_mtime = get_mtime(sbi, false);  in init_min_max_mtime()
5051 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_build_segment_manager()  argument
5053 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);  in f2fs_build_segment_manager()
5054 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);  in f2fs_build_segment_manager()
5058 	sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);  in f2fs_build_segment_manager()
5063 	sbi->sm_info = sm_info;  in f2fs_build_segment_manager()
5076 	if (!f2fs_lfs_mode(sbi))  in f2fs_build_segment_manager()
5080 	sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;  in f2fs_build_segment_manager()
5082 	sm_info->min_ssr_sections = reserved_sections(sbi);  in f2fs_build_segment_manager()
5088 	if (!f2fs_readonly(sbi->sb)) {  in f2fs_build_segment_manager()
5089 		err = f2fs_create_flush_cmd_control(sbi);  in f2fs_build_segment_manager()
5094 	err = create_discard_cmd_control(sbi);  in f2fs_build_segment_manager()
5098 	err = build_sit_info(sbi);  in f2fs_build_segment_manager()
5101 	err = build_free_segmap(sbi);  in f2fs_build_segment_manager()
5104 	err = build_curseg(sbi);  in f2fs_build_segment_manager()
5109 	err = build_sit_entries(sbi);  in f2fs_build_segment_manager()
5113 	init_free_segmap(sbi);  in f2fs_build_segment_manager()
5114 	err = build_dirty_segmap(sbi);  in f2fs_build_segment_manager()
5118 	err = sanity_check_curseg(sbi);  in f2fs_build_segment_manager()
5122 	init_min_max_mtime(sbi);  in f2fs_build_segment_manager()
5126 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,  in discard_dirty_segmap()  argument
5129 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in discard_dirty_segmap()
5137 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)  in destroy_victim_secmap()  argument
5139 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_victim_secmap()
5143 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)  in destroy_dirty_segmap()  argument
5145 	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  in destroy_dirty_segmap()
5153 		discard_dirty_segmap(sbi, i);  in destroy_dirty_segmap()
5155 	if (__is_large_section(sbi)) {  in destroy_dirty_segmap()
5161 	destroy_victim_secmap(sbi);  in destroy_dirty_segmap()
5162 	SM_I(sbi)->dirty_info = NULL;  in destroy_dirty_segmap()
5166 static void destroy_curseg(struct f2fs_sb_info *sbi)  in destroy_curseg()  argument
5168 	struct curseg_info *array = SM_I(sbi)->curseg_array;  in destroy_curseg()
5173 	SM_I(sbi)->curseg_array = NULL;  in destroy_curseg()
5181 static void destroy_free_segmap(struct f2fs_sb_info *sbi)  in destroy_free_segmap()  argument
5183 	struct free_segmap_info *free_i = SM_I(sbi)->free_info;  in destroy_free_segmap()
5186 	SM_I(sbi)->free_info = NULL;  in destroy_free_segmap()
5192 static void destroy_sit_info(struct f2fs_sb_info *sbi)  in destroy_sit_info()  argument
5194 	struct sit_info *sit_i = SIT_I(sbi);  in destroy_sit_info()
5207 	SM_I(sbi)->sit_info = NULL;  in destroy_sit_info()
5216 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)  in f2fs_destroy_segment_manager()  argument
5218 	struct f2fs_sm_info *sm_info = SM_I(sbi);  in f2fs_destroy_segment_manager()
5222 	f2fs_destroy_flush_cmd_control(sbi, true);  in f2fs_destroy_segment_manager()
5223 	destroy_discard_cmd_control(sbi);  in f2fs_destroy_segment_manager()
5224 	destroy_dirty_segmap(sbi);  in f2fs_destroy_segment_manager()
5225 	destroy_curseg(sbi);  in f2fs_destroy_segment_manager()
5226 	destroy_free_segmap(sbi);  in f2fs_destroy_segment_manager()
5227 	destroy_sit_info(sbi);  in f2fs_destroy_segment_manager()
5228 	sbi->sm_info = NULL;  in f2fs_destroy_segment_manager()