900 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __create_discard_cmd()  local
906 	pend_list = &dcc->pend_list[plist_idx(len)];  in __create_discard_cmd()
922 	atomic_inc(&dcc->discard_cmd_cnt);  in __create_discard_cmd()
923 	dcc->undiscard_blks += len;  in __create_discard_cmd()
934 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __attach_discard_cmd()  local
940 	rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);  in __attach_discard_cmd()
945 static void __detach_discard_cmd(struct discard_cmd_control *dcc,  in __detach_discard_cmd()  argument
949 		atomic_sub(dc->queued, &dcc->queued_discard);  in __detach_discard_cmd()
952 	rb_erase_cached(&dc->rb_node, &dcc->root);  in __detach_discard_cmd()
953 	dcc->undiscard_blks -= dc->len;  in __detach_discard_cmd()
957 	atomic_dec(&dcc->discard_cmd_cnt);  in __detach_discard_cmd()
963 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __remove_discard_cmd()  local
985 	__detach_discard_cmd(dcc, dc);  in __remove_discard_cmd()
1036 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __init_discard_policy()  local
1044 	dpolicy->max_requests = dcc->max_discard_request;  in __init_discard_policy()
1049 		dpolicy->min_interval = dcc->min_discard_issue_time;  in __init_discard_policy()
1050 		dpolicy->mid_interval = dcc->mid_discard_issue_time;  in __init_discard_policy()
1051 		dpolicy->max_interval = dcc->max_discard_issue_time;  in __init_discard_policy()
1057 			if (atomic_read(&dcc->discard_cmd_cnt))  in __init_discard_policy()
1059 					dcc->min_discard_issue_time;  in __init_discard_policy()
1062 		dpolicy->min_interval = dcc->min_discard_issue_time;  in __init_discard_policy()
1063 		dpolicy->mid_interval = dcc->mid_discard_issue_time;  in __init_discard_policy()
1064 		dpolicy->max_interval = dcc->max_discard_issue_time;  in __init_discard_policy()
1088 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __submit_discard_cmd()  local
1090 					&(dcc->fstrim_list) : &(dcc->wait_list);  in __submit_discard_cmd()
1159 		atomic_inc(&dcc->queued_discard);  in __submit_discard_cmd()
1171 		atomic_inc(&dcc->issued_discard);  in __submit_discard_cmd()
1182 		dcc->undiscard_blks -= len;  in __submit_discard_cmd()
1194 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __insert_discard_tree()  local
1205 	p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,  in __insert_discard_tree()
1212 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,  in __relocate_discard_cmd()  argument
1215 	list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);  in __relocate_discard_cmd()
1221 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __punch_discard_cmd()  local
1230 	dcc->undiscard_blks -= di.len;  in __punch_discard_cmd()
1234 		dcc->undiscard_blks += dc->len;  in __punch_discard_cmd()
1235 		__relocate_discard_cmd(dcc, dc);  in __punch_discard_cmd()
1249 			dcc->undiscard_blks += dc->len;  in __punch_discard_cmd()
1250 			__relocate_discard_cmd(dcc, dc);  in __punch_discard_cmd()
1259 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __update_discard_tree_range()  local
1268 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __update_discard_tree_range()
1310 			dcc->undiscard_blks += di.len;  in __update_discard_tree_range()
1311 			__relocate_discard_cmd(dcc, prev_dc);  in __update_discard_tree_range()
1324 			dcc->undiscard_blks += di.len;  in __update_discard_tree_range()
1325 			__relocate_discard_cmd(dcc, next_dc);  in __update_discard_tree_range()
1369 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_orderly()  local
1374 	unsigned int pos = dcc->next_pos;  in __issue_discard_cmd_orderly()
1378 	mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd_orderly()
1379 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __issue_discard_cmd_orderly()
1401 		dcc->next_pos = dc->lstart + dc->len;  in __issue_discard_cmd_orderly()
1416 		dcc->next_pos = 0;  in __issue_discard_cmd_orderly()
1418 	mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_orderly()
1431 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd()  local
1454 		pend_list = &dcc->pend_list[i];  in __issue_discard_cmd()
1456 		mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd()
1459 		if (unlikely(dcc->rbtree_check))  in __issue_discard_cmd()
1461 							&dcc->root, false));  in __issue_discard_cmd()
1483 		mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd()
1502 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __drop_discard_cmd()  local
1508 	mutex_lock(&dcc->cmd_lock);  in __drop_discard_cmd()
1510 		pend_list = &dcc->pend_list[i];  in __drop_discard_cmd()
1517 	mutex_unlock(&dcc->cmd_lock);  in __drop_discard_cmd()
1530 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_one_discard_bio()  local
1534 	mutex_lock(&dcc->cmd_lock);  in __wait_one_discard_bio()
1542 	mutex_unlock(&dcc->cmd_lock);  in __wait_one_discard_bio()
1551 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __wait_discard_cmd_range()  local
1553 					&(dcc->fstrim_list) : &(dcc->wait_list);  in __wait_discard_cmd_range()
1560 	mutex_lock(&dcc->cmd_lock);  in __wait_discard_cmd_range()
1577 	mutex_unlock(&dcc->cmd_lock);  in __wait_discard_cmd_range()
1608 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_wait_discard_bio()  local
1612 	mutex_lock(&dcc->cmd_lock);  in f2fs_wait_discard_bio()
1613 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,  in f2fs_wait_discard_bio()
1623 	mutex_unlock(&dcc->cmd_lock);  in f2fs_wait_discard_bio()
1631 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_stop_discard_thread()  local
1633 	if (dcc && dcc->f2fs_issue_discard) {  in f2fs_stop_discard_thread()
1634 		struct task_struct *discard_thread = dcc->f2fs_issue_discard;  in f2fs_stop_discard_thread()
1636 		dcc->f2fs_issue_discard = NULL;  in f2fs_stop_discard_thread()
1644 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_issue_discard_timeout()  local
1649 					dcc->discard_granularity);  in f2fs_issue_discard_timeout()
1656 	f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));  in f2fs_issue_discard_timeout()
1663 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in issue_discard_thread()  local
1664 	wait_queue_head_t *q = &dcc->discard_wait_queue;  in issue_discard_thread()
1666 	unsigned int wait_ms = dcc->min_discard_issue_time;  in issue_discard_thread()
1677 						dcc->discard_granularity);  in issue_discard_thread()
1679 		if (!atomic_read(&dcc->discard_cmd_cnt))  in issue_discard_thread()
1684 				dcc->discard_wake,  in issue_discard_thread()
1687 		if (dcc->discard_wake)  in issue_discard_thread()
1688 			dcc->discard_wake = 0;  in issue_discard_thread()
1691 		if (atomic_read(&dcc->queued_discard))  in issue_discard_thread()
1704 		if (!atomic_read(&dcc->discard_cmd_cnt))  in issue_discard_thread()
1912 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_clear_prefree_segments()  local
1913 	struct list_head *head = &dcc->entry_list;  in f2fs_clear_prefree_segments()
2010 		dcc->nr_discards -= total_len;  in f2fs_clear_prefree_segments()
2020 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in f2fs_start_discard_thread()  local
2026 	dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,  in f2fs_start_discard_thread()
2028 	if (IS_ERR(dcc->f2fs_issue_discard))  in f2fs_start_discard_thread()
2029 		err = PTR_ERR(dcc->f2fs_issue_discard);  in f2fs_start_discard_thread()
2036 	struct discard_cmd_control *dcc;  in create_discard_cmd_control()  local
2040 		dcc = SM_I(sbi)->dcc_info;  in create_discard_cmd_control()
2044 	dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);  in create_discard_cmd_control()
2045 	if (!dcc)  in create_discard_cmd_control()
2048 	dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;  in create_discard_cmd_control()
2050 		dcc->discard_granularity = sbi->blocks_per_seg;  in create_discard_cmd_control()
2052 		dcc->discard_granularity = BLKS_PER_SEC(sbi);  in create_discard_cmd_control()
2054 	INIT_LIST_HEAD(&dcc->entry_list);  in create_discard_cmd_control()
2056 		INIT_LIST_HEAD(&dcc->pend_list[i]);  in create_discard_cmd_control()
2057 	INIT_LIST_HEAD(&dcc->wait_list);  in create_discard_cmd_control()
2058 	INIT_LIST_HEAD(&dcc->fstrim_list);  in create_discard_cmd_control()
2059 	mutex_init(&dcc->cmd_lock);  in create_discard_cmd_control()
2060 	atomic_set(&dcc->issued_discard, 0);  in create_discard_cmd_control()
2061 	atomic_set(&dcc->queued_discard, 0);  in create_discard_cmd_control()
2062 	atomic_set(&dcc->discard_cmd_cnt, 0);  in create_discard_cmd_control()
2063 	dcc->nr_discards = 0;  in create_discard_cmd_control()
2064 	dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;  in create_discard_cmd_control()
2065 	dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;  in create_discard_cmd_control()
2066 	dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;  in create_discard_cmd_control()
2067 	dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;  in create_discard_cmd_control()
2068 	dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;  in create_discard_cmd_control()
2069 	dcc->undiscard_blks = 0;  in create_discard_cmd_control()
2070 	dcc->next_pos = 0;  in create_discard_cmd_control()
2071 	dcc->root = RB_ROOT_CACHED;  in create_discard_cmd_control()
2072 	dcc->rbtree_check = false;  in create_discard_cmd_control()
2074 	init_waitqueue_head(&dcc->discard_wait_queue);  in create_discard_cmd_control()
2075 	SM_I(sbi)->dcc_info = dcc;  in create_discard_cmd_control()
2079 		kfree(dcc);  in create_discard_cmd_control()
2088 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in destroy_discard_cmd_control()  local
2090 	if (!dcc)  in destroy_discard_cmd_control()
2099 	if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))  in destroy_discard_cmd_control()
2102 	kfree(dcc);  in destroy_discard_cmd_control()
2973 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  in __issue_discard_cmd_range()  local
2984 	mutex_lock(&dcc->cmd_lock);  in __issue_discard_cmd_range()
2985 	if (unlikely(dcc->rbtree_check))  in __issue_discard_cmd_range()
2987 							&dcc->root, false));  in __issue_discard_cmd_range()
2989 	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,  in __issue_discard_cmd_range()
3007 			list_move_tail(&dc->list, &dcc->fstrim_list);  in __issue_discard_cmd_range()
3020 			mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_range()
3036 	mutex_unlock(&dcc->cmd_lock);  in __issue_discard_cmd_range()