828 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __create_discard_cmd()  local
834 	pend_list = &dcc->pend_list[plist_idx(len)];  in __create_discard_cmd()
850 	atomic_inc(&dcc->discard_cmd_cnt);  in __create_discard_cmd()
851 	dcc->undiscard_blks += len;  in __create_discard_cmd()
861 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __attach_discard_cmd()  local
867 	rb_insert_color(&dc->rb_node, &dcc->root);  in __attach_discard_cmd()
872 static void __detach_discard_cmd(struct discard_cmd_control *dcc,  in __detach_discard_cmd()  argument
876 		atomic_sub(dc->issuing, &dcc->issing_discard);  in __detach_discard_cmd()
879 	rb_erase(&dc->rb_node, &dcc->root);  in __detach_discard_cmd()
880 	dcc->undiscard_blks -= dc->len;  in __detach_discard_cmd()
884 	atomic_dec(&dcc->discard_cmd_cnt);  in __detach_discard_cmd()
890 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __remove_discard_cmd()  local
911 	__detach_discard_cmd(dcc, dc);  in __remove_discard_cmd()
1008 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_discard_cmd()  local
1010 					&(dcc->fstrim_list) : &(dcc->wait_list);  in __submit_discard_cmd()
1079 		atomic_inc(&dcc->issing_discard);  in __submit_discard_cmd()
1091 		atomic_inc(&dcc->issued_discard);  in __submit_discard_cmd()
1112 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __insert_discard_tree()  local
1123 	p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent, lstart);  in __insert_discard_tree()
1132 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,  in __relocate_discard_cmd()  argument
1135 	list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);  in __relocate_discard_cmd()
1141 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __punch_discard_cmd()  local
1150 	dcc->undiscard_blks -= di.len;  in __punch_discard_cmd()
1154 		dcc->undiscard_blks += dc->len;  in __punch_discard_cmd()
1155 		__relocate_discard_cmd(dcc, dc);  in __punch_discard_cmd()
1169 			dcc->undiscard_blks += dc->len;  in __punch_discard_cmd()
1170 			__relocate_discard_cmd(dcc, dc);  in __punch_discard_cmd()
1179 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __update_discard_tree_range()  local
1189 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __update_discard_tree_range()
1231 			dcc->undiscard_blks += di.len;  in __update_discard_tree_range()
1232 			__relocate_discard_cmd(dcc, prev_dc);  in __update_discard_tree_range()
1245 			dcc->undiscard_blks += di.len;  in __update_discard_tree_range()
1246 			__relocate_discard_cmd(dcc, next_dc);  in __update_discard_tree_range()
1287 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_orderly()  local
1292 	unsigned int pos = dcc->next_pos;  in __issue_discard_cmd_orderly()
1296 	mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd_orderly()
1297 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __issue_discard_cmd_orderly()
1319 		dcc->next_pos = dc->lstart + dc->len;  in __issue_discard_cmd_orderly()
1334 		dcc->next_pos = 0;  in __issue_discard_cmd_orderly()
1336 	mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_orderly()
1347 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd()  local
1361 		pend_list = &dcc->pend_list[i];  in __issue_discard_cmd()
1363 		mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd()
1366 		if (unlikely(dcc->rbtree_check))  in __issue_discard_cmd()
1368 								&dcc->root));  in __issue_discard_cmd()
1386 		mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd()
1400 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __drop_discard_cmd()  local
1406 	mutex_lock(&dcc->cmd_lock);  in __drop_discard_cmd()
1408 		pend_list = &dcc->pend_list[i];  in __drop_discard_cmd()
1415 	mutex_unlock(&dcc->cmd_lock);  in __drop_discard_cmd()
1428 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_one_discard_bio()  local
1432 	mutex_lock(&dcc->cmd_lock);  in __wait_one_discard_bio()
1440 	mutex_unlock(&dcc->cmd_lock);  in __wait_one_discard_bio()
1449 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_discard_cmd_range()  local
1451 					&(dcc->fstrim_list) : &(dcc->wait_list);  in __wait_discard_cmd_range()
1459 	mutex_lock(&dcc->cmd_lock);  in __wait_discard_cmd_range()
1476 	mutex_unlock(&dcc->cmd_lock);  in __wait_discard_cmd_range()
1507 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bio()  local
1511 	mutex_lock(&dcc->cmd_lock);  in f2fs_wait_discard_bio()
1512 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,  in f2fs_wait_discard_bio()
1522 	mutex_unlock(&dcc->cmd_lock);  in f2fs_wait_discard_bio()
1530 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_stop_discard_thread()  local
1532 	if (dcc && dcc->f2fs_issue_discard) {  in f2fs_stop_discard_thread()
1533 		struct task_struct *discard_thread = dcc->f2fs_issue_discard;  in f2fs_stop_discard_thread()
1535 		dcc->f2fs_issue_discard = NULL;  in f2fs_stop_discard_thread()
1543 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bios()  local
1548 					dcc->discard_granularity);  in f2fs_wait_discard_bios()
1555 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));  in f2fs_wait_discard_bios()
1562 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in issue_discard_thread()  local
1563 	wait_queue_head_t *q = &dcc->discard_wait_queue;  in issue_discard_thread()
1572 					dcc->discard_granularity);  in issue_discard_thread()
1576 				dcc->discard_wake,  in issue_discard_thread()
1579 		if (dcc->discard_wake)  in issue_discard_thread()
1580 			dcc->discard_wake = 0;  in issue_discard_thread()
1805 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_clear_prefree_segments()  local
1806 	struct list_head *head = &dcc->entry_list;  in f2fs_clear_prefree_segments()
1896 		dcc->nr_discards -= total_len;  in f2fs_clear_prefree_segments()
1905 	struct discard_cmd_control *dcc;  in create_discard_cmd_control()  local
1909 		dcc = SM_I(sbi)->dcc_info;  in create_discard_cmd_control()
1913 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);  in create_discard_cmd_control()
1914 	if (!dcc)  in create_discard_cmd_control()
1917 	dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;  in create_discard_cmd_control()
1918 	INIT_LIST_HEAD(&dcc->entry_list);  in create_discard_cmd_control()
1920 		INIT_LIST_HEAD(&dcc->pend_list[i]);  in create_discard_cmd_control()
1921 	INIT_LIST_HEAD(&dcc->wait_list);  in create_discard_cmd_control()
1922 	INIT_LIST_HEAD(&dcc->fstrim_list);  in create_discard_cmd_control()
1923 	mutex_init(&dcc->cmd_lock);  in create_discard_cmd_control()
1924 	atomic_set(&dcc->issued_discard, 0);  in create_discard_cmd_control()
1925 	atomic_set(&dcc->issing_discard, 0);  in create_discard_cmd_control()
1926 	atomic_set(&dcc->discard_cmd_cnt, 0);  in create_discard_cmd_control()
1927 	dcc->nr_discards = 0;  in create_discard_cmd_control()
1928 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;  in create_discard_cmd_control()
1929 	dcc->undiscard_blks = 0;  in create_discard_cmd_control()
1930 	dcc->next_pos = 0;  in create_discard_cmd_control()
1931 	dcc->root = RB_ROOT;  in create_discard_cmd_control()
1932 	dcc->rbtree_check = false;  in create_discard_cmd_control()
1934 	init_waitqueue_head(&dcc->discard_wait_queue);  in create_discard_cmd_control()
1935 	SM_I(sbi)->dcc_info = dcc;  in create_discard_cmd_control()
1937 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,  in create_discard_cmd_control()
1939 	if (IS_ERR(dcc->f2fs_issue_discard)) {  in create_discard_cmd_control()
1940 		err = PTR_ERR(dcc->f2fs_issue_discard);  in create_discard_cmd_control()
1941 		kfree(dcc);  in create_discard_cmd_control()
1951 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in destroy_discard_cmd_control()  local
1953 	if (!dcc)  in destroy_discard_cmd_control()
1958 	kfree(dcc);  in destroy_discard_cmd_control()
2553 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_range()  local
2564 	mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd_range()
2565 	if (unlikely(dcc->rbtree_check))  in __issue_discard_cmd_range()
2567 								&dcc->root));  in __issue_discard_cmd_range()
2569 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __issue_discard_cmd_range()
2587 			list_move_tail(&dc->list, &dcc->fstrim_list);  in __issue_discard_cmd_range()
2600 			mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_range()
2616 	mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_range()