138 void bfq_mark_bfqq_##name(struct bfq_queue *bfqq)			\
140 	__set_bit(BFQQF_##name, &(bfqq)->flags);			\
142 void bfq_clear_bfqq_##name(struct bfq_queue *bfqq)			\
144 	__clear_bit(BFQQF_##name, &(bfqq)->flags);		\
146 int bfq_bfqq_##name(const struct bfq_queue *bfqq)			\
148 	return test_bit(BFQQF_##name, &(bfqq)->flags);		\
238 #define BFQQ_SEEKY(bfqq)	(hweight32(bfqq->seek_history) > 19)  argument
246 #define BFQQ_TOTALLY_SEEKY(bfqq)	(bfqq->seek_history == -1)  argument
382 	return bic->bfqq[is_sync];  in bic_to_bfqq()
385 static void bfq_put_stable_ref(struct bfq_queue *bfqq);
387 void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync)  in bic_set_bfqq()  argument
402 	bic->bfqq[is_sync] = bfqq;  in bic_set_bfqq()
404 	if (bfqq && bic->stable_merge_bfqq == bfqq) {  in bic_set_bfqq()
467 #define bfq_class_idle(bfqq)	((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE)  argument
568 static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)  in bfqq_request_over_limit()  argument
570 	struct bfq_data *bfqd = bfqq->bfqd;  in bfqq_request_over_limit()
571 	struct bfq_entity *entity = &bfqq->entity;  in bfqq_request_over_limit()
575 	int class_idx = bfqq->ioprio_class - 1;  in bfqq_request_over_limit()
586 	depth = bfqg_to_blkg(bfqq_group(bfqq))->blkcg->css.cgroup->level + 1;  in bfqq_request_over_limit()
636 			bfq_log_bfqq(bfqq->bfqd, bfqq,  in bfqq_request_over_limit()
650 static bool bfqq_request_over_limit(struct bfq_queue *bfqq, int limit)  in bfqq_request_over_limit()  argument
675 	struct bfq_queue *bfqq = bic ? bic_to_bfqq(bic, op_is_sync(opf)) : NULL;  in bfq_limit_depth()  local
692 	if (bfqq && bfqq_request_over_limit(bfqq, limit))  in bfq_limit_depth()
707 	struct bfq_queue *bfqq = NULL;  in bfq_rq_pos_tree_lookup()  local
715 		bfqq = rb_entry(parent, struct bfq_queue, pos_node);  in bfq_rq_pos_tree_lookup()
721 		if (sector > blk_rq_pos(bfqq->next_rq))  in bfq_rq_pos_tree_lookup()
723 		else if (sector < blk_rq_pos(bfqq->next_rq))  in bfq_rq_pos_tree_lookup()
728 		bfqq = NULL;  in bfq_rq_pos_tree_lookup()
737 		bfqq ? bfqq->pid : 0);  in bfq_rq_pos_tree_lookup()
739 	return bfqq;  in bfq_rq_pos_tree_lookup()
742 static bool bfq_too_late_for_merging(struct bfq_queue *bfqq)  in bfq_too_late_for_merging()  argument
744 	return bfqq->service_from_backlogged > 0 &&  in bfq_too_late_for_merging()
745 		time_is_before_jiffies(bfqq->first_IO_time +  in bfq_too_late_for_merging()
758 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_pos_tree_add_move()  argument
763 	if (bfqq->pos_root) {  in bfq_pos_tree_add_move()
764 		rb_erase(&bfqq->pos_node, bfqq->pos_root);  in bfq_pos_tree_add_move()
765 		bfqq->pos_root = NULL;  in bfq_pos_tree_add_move()
769 	if (bfqq == &bfqd->oom_bfqq)  in bfq_pos_tree_add_move()
777 	if (bfq_too_late_for_merging(bfqq))  in bfq_pos_tree_add_move()
780 	if (bfq_class_idle(bfqq))  in bfq_pos_tree_add_move()
782 	if (!bfqq->next_rq)  in bfq_pos_tree_add_move()
785 	bfqq->pos_root = &bfqq_group(bfqq)->rq_pos_tree;  in bfq_pos_tree_add_move()
786 	__bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root,  in bfq_pos_tree_add_move()
787 			blk_rq_pos(bfqq->next_rq), &parent, &p);  in bfq_pos_tree_add_move()
789 		rb_link_node(&bfqq->pos_node, parent, p);  in bfq_pos_tree_add_move()
790 		rb_insert_color(&bfqq->pos_node, bfqq->pos_root);  in bfq_pos_tree_add_move()
792 		bfqq->pos_root = NULL;  in bfq_pos_tree_add_move()
829 				   struct bfq_queue *bfqq)  in bfq_asymmetric_scenario()  argument
831 	bool smallest_weight = bfqq &&  in bfq_asymmetric_scenario()
832 		bfqq->weight_counter &&  in bfq_asymmetric_scenario()
833 		bfqq->weight_counter ==  in bfq_asymmetric_scenario()
873 void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_weights_tree_add()  argument
876 	struct bfq_entity *entity = &bfqq->entity;  in bfq_weights_tree_add()
892 	if (bfqq->weight_counter)  in bfq_weights_tree_add()
902 			bfqq->weight_counter = __counter;  in bfq_weights_tree_add()
913 	bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter),  in bfq_weights_tree_add()
928 	if (unlikely(!bfqq->weight_counter))  in bfq_weights_tree_add()
931 	bfqq->weight_counter->weight = entity->weight;  in bfq_weights_tree_add()
932 	rb_link_node(&bfqq->weight_counter->weights_node, parent, new);  in bfq_weights_tree_add()
933 	rb_insert_color_cached(&bfqq->weight_counter->weights_node, root,  in bfq_weights_tree_add()
937 	bfqq->weight_counter->num_active++;  in bfq_weights_tree_add()
938 	bfqq->ref++;  in bfq_weights_tree_add()
948 			       struct bfq_queue *bfqq,  in __bfq_weights_tree_remove()  argument
951 	if (!bfqq->weight_counter)  in __bfq_weights_tree_remove()
954 	bfqq->weight_counter->num_active--;  in __bfq_weights_tree_remove()
955 	if (bfqq->weight_counter->num_active > 0)  in __bfq_weights_tree_remove()
958 	rb_erase_cached(&bfqq->weight_counter->weights_node, root);  in __bfq_weights_tree_remove()
959 	kfree(bfqq->weight_counter);  in __bfq_weights_tree_remove()
962 	bfqq->weight_counter = NULL;  in __bfq_weights_tree_remove()
963 	bfq_put_queue(bfqq);  in __bfq_weights_tree_remove()
971 			     struct bfq_queue *bfqq)  in bfq_weights_tree_remove()  argument
973 	struct bfq_entity *entity = bfqq->entity.parent;  in bfq_weights_tree_remove()
1015 	__bfq_weights_tree_remove(bfqd, bfqq,  in bfq_weights_tree_remove()
1022 static struct request *bfq_check_fifo(struct bfq_queue *bfqq,  in bfq_check_fifo()  argument
1027 	if (bfq_bfqq_fifo_expire(bfqq))  in bfq_check_fifo()
1030 	bfq_mark_bfqq_fifo_expire(bfqq);  in bfq_check_fifo()
1032 	rq = rq_entry_fifo(bfqq->fifo.next);  in bfq_check_fifo()
1037 	bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq);  in bfq_check_fifo()
1042 					struct bfq_queue *bfqq,  in bfq_find_next_rq()  argument
1050 	next = bfq_check_fifo(bfqq, last);  in bfq_find_next_rq()
1060 		rbnext = rb_first(&bfqq->sort_list);  in bfq_find_next_rq()
1070 					struct bfq_queue *bfqq)  in bfq_serv_to_charge()  argument
1072 	if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 ||  in bfq_serv_to_charge()
1073 	    bfq_asymmetric_scenario(bfqq->bfqd, bfqq))  in bfq_serv_to_charge()
1091 				 struct bfq_queue *bfqq)  in bfq_updated_next_req()  argument
1093 	struct bfq_entity *entity = &bfqq->entity;  in bfq_updated_next_req()
1094 	struct request *next_rq = bfqq->next_rq;  in bfq_updated_next_req()
1100 	if (bfqq == bfqd->in_service_queue)  in bfq_updated_next_req()
1108 			   max_t(unsigned long, bfqq->max_budget,  in bfq_updated_next_req()
1109 				 bfq_serv_to_charge(next_rq, bfqq)),  in bfq_updated_next_req()
1113 		bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu",  in bfq_updated_next_req()
1115 		bfq_requeue_bfqq(bfqd, bfqq, false);  in bfq_updated_next_req()
1153 static void switch_back_to_interactive_wr(struct bfq_queue *bfqq,  in switch_back_to_interactive_wr()  argument
1156 	bfqq->wr_coeff = bfqd->bfq_wr_coeff;  in switch_back_to_interactive_wr()
1157 	bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);  in switch_back_to_interactive_wr()
1158 	bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt;  in switch_back_to_interactive_wr()
1162 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd,  in bfq_bfqq_resume_state()  argument
1166 	bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq);  in bfq_bfqq_resume_state()
1169 		bfq_mark_bfqq_has_short_ttime(bfqq);  in bfq_bfqq_resume_state()
1171 		bfq_clear_bfqq_has_short_ttime(bfqq);  in bfq_bfqq_resume_state()
1174 		bfq_mark_bfqq_IO_bound(bfqq);  in bfq_bfqq_resume_state()
1176 		bfq_clear_bfqq_IO_bound(bfqq);  in bfq_bfqq_resume_state()
1178 	bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns;  in bfq_bfqq_resume_state()
1179 	bfqq->inject_limit = bic->saved_inject_limit;  in bfq_bfqq_resume_state()
1180 	bfqq->decrease_time_jif = bic->saved_decrease_time_jif;  in bfq_bfqq_resume_state()
1182 	bfqq->entity.new_weight = bic->saved_weight;  in bfq_bfqq_resume_state()
1183 	bfqq->ttime = bic->saved_ttime;  in bfq_bfqq_resume_state()
1184 	bfqq->io_start_time = bic->saved_io_start_time;  in bfq_bfqq_resume_state()
1185 	bfqq->tot_idle_time = bic->saved_tot_idle_time;  in bfq_bfqq_resume_state()
1190 		old_wr_coeff = bfqq->wr_coeff;  in bfq_bfqq_resume_state()
1191 		bfqq->wr_coeff = bic->saved_wr_coeff;  in bfq_bfqq_resume_state()
1193 	bfqq->service_from_wr = bic->saved_service_from_wr;  in bfq_bfqq_resume_state()
1194 	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;  in bfq_bfqq_resume_state()
1195 	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;  in bfq_bfqq_resume_state()
1196 	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;  in bfq_bfqq_resume_state()
1198 	if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) ||  in bfq_bfqq_resume_state()
1199 	    time_is_before_jiffies(bfqq->last_wr_start_finish +  in bfq_bfqq_resume_state()
1200 				   bfqq->wr_cur_max_time))) {  in bfq_bfqq_resume_state()
1201 		if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&  in bfq_bfqq_resume_state()
1202 		    !bfq_bfqq_in_large_burst(bfqq) &&  in bfq_bfqq_resume_state()
1203 		    time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt +  in bfq_bfqq_resume_state()
1205 			switch_back_to_interactive_wr(bfqq, bfqd);  in bfq_bfqq_resume_state()
1207 			bfqq->wr_coeff = 1;  in bfq_bfqq_resume_state()
1208 			bfq_log_bfqq(bfqq->bfqd, bfqq,  in bfq_bfqq_resume_state()
1214 	bfqq->entity.prio_changed = 1;  in bfq_bfqq_resume_state()
1219 	if (old_wr_coeff == 1 && bfqq->wr_coeff > 1)  in bfq_bfqq_resume_state()
1221 	else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1)  in bfq_bfqq_resume_state()
1225 static int bfqq_process_refs(struct bfq_queue *bfqq)  in bfqq_process_refs()  argument
1227 	return bfqq->ref - bfqq->entity.allocated -  in bfqq_process_refs()
1228 		bfqq->entity.on_st_or_in_serv -  in bfqq_process_refs()
1229 		(bfqq->weight_counter != NULL) - bfqq->stable_ref;  in bfqq_process_refs()
1233 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_reset_burst_list()  argument
1247 		hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);  in bfq_reset_burst_list()
1252 	bfqd->burst_parent_entity = bfqq->entity.parent;  in bfq_reset_burst_list()
1256 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_add_to_burst()  argument
1278 		bfq_mark_bfqq_in_large_burst(bfqq);  in bfq_add_to_burst()
1296 		hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list);  in bfq_add_to_burst()
1408 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_handle_burst()  argument
1415 	if (!hlist_unhashed(&bfqq->burst_list_node) ||  in bfq_handle_burst()
1416 	    bfq_bfqq_in_large_burst(bfqq) ||  in bfq_handle_burst()
1417 	    time_is_after_eq_jiffies(bfqq->split_time +  in bfq_handle_burst()
1440 	    bfqq->entity.parent != bfqd->burst_parent_entity) {  in bfq_handle_burst()
1442 		bfq_reset_burst_list(bfqd, bfqq);  in bfq_handle_burst()
1452 		bfq_mark_bfqq_in_large_burst(bfqq);  in bfq_handle_burst()
1461 	bfq_add_to_burst(bfqd, bfqq);  in bfq_handle_burst()
1474 static int bfq_bfqq_budget_left(struct bfq_queue *bfqq)  in bfq_bfqq_budget_left()  argument
1476 	struct bfq_entity *entity = &bfqq->entity;  in bfq_bfqq_budget_left()
1611 						struct bfq_queue *bfqq,  in bfq_bfqq_update_budg_for_activation()  argument
1614 	struct bfq_entity *entity = &bfqq->entity;  in bfq_bfqq_update_budg_for_activation()
1623 	if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time &&  in bfq_bfqq_update_budg_for_activation()
1624 	    bfq_bfqq_budget_left(bfqq) > 0) {  in bfq_bfqq_update_budg_for_activation()
1643 				       bfq_bfqq_budget_left(bfqq),  in bfq_bfqq_update_budg_for_activation()
1644 				       bfqq->max_budget);  in bfq_bfqq_update_budg_for_activation()
1664 	entity->budget = max_t(unsigned long, bfqq->max_budget,  in bfq_bfqq_update_budg_for_activation()
1665 			       bfq_serv_to_charge(bfqq->next_rq, bfqq));  in bfq_bfqq_update_budg_for_activation()
1666 	bfq_clear_bfqq_non_blocking_wait_rq(bfqq);  in bfq_bfqq_update_budg_for_activation()
1680 					     struct bfq_queue *bfqq,  in bfq_update_bfqq_wr_on_rq_arrival()  argument
1690 			bfqq->service_from_wr = 0;  in bfq_update_bfqq_wr_on_rq_arrival()
1691 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;  in bfq_update_bfqq_wr_on_rq_arrival()
1692 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);  in bfq_update_bfqq_wr_on_rq_arrival()
1705 			bfqq->wr_start_at_switch_to_srt =  in bfq_update_bfqq_wr_on_rq_arrival()
1707 			bfqq->wr_coeff = bfqd->bfq_wr_coeff *  in bfq_update_bfqq_wr_on_rq_arrival()
1709 			bfqq->wr_cur_max_time =  in bfq_update_bfqq_wr_on_rq_arrival()
1722 		bfqq->entity.budget = min_t(unsigned long,  in bfq_update_bfqq_wr_on_rq_arrival()
1723 					    bfqq->entity.budget,  in bfq_update_bfqq_wr_on_rq_arrival()
1727 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;  in bfq_update_bfqq_wr_on_rq_arrival()
1728 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);  in bfq_update_bfqq_wr_on_rq_arrival()
1730 			bfqq->wr_coeff = 1;  in bfq_update_bfqq_wr_on_rq_arrival()
1761 			if (bfqq->wr_cur_max_time !=  in bfq_update_bfqq_wr_on_rq_arrival()
1763 				bfqq->wr_start_at_switch_to_srt =  in bfq_update_bfqq_wr_on_rq_arrival()
1764 					bfqq->last_wr_start_finish;  in bfq_update_bfqq_wr_on_rq_arrival()
1766 				bfqq->wr_cur_max_time =  in bfq_update_bfqq_wr_on_rq_arrival()
1768 				bfqq->wr_coeff = bfqd->bfq_wr_coeff *  in bfq_update_bfqq_wr_on_rq_arrival()
1771 			bfqq->last_wr_start_finish = jiffies;  in bfq_update_bfqq_wr_on_rq_arrival()
1777 					struct bfq_queue *bfqq)  in bfq_bfqq_idle_for_long_time()  argument
1779 	return bfqq->dispatched == 0 &&  in bfq_bfqq_idle_for_long_time()
1781 			bfqq->budget_timeout +  in bfq_bfqq_idle_for_long_time()
1790 static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq,  in bfq_bfqq_higher_class_or_weight()  argument
1795 	if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class)  in bfq_bfqq_higher_class_or_weight()
1798 	if (in_serv_bfqq->entity.parent == bfqq->entity.parent) {  in bfq_bfqq_higher_class_or_weight()
1799 		bfqq_weight = bfqq->entity.weight;  in bfq_bfqq_higher_class_or_weight()
1802 		if (bfqq->entity.parent)  in bfq_bfqq_higher_class_or_weight()
1803 			bfqq_weight = bfqq->entity.parent->weight;  in bfq_bfqq_higher_class_or_weight()
1805 			bfqq_weight = bfqq->entity.weight;  in bfq_bfqq_higher_class_or_weight()
1815 static bool bfq_better_to_idle(struct bfq_queue *bfqq);
1818 					     struct bfq_queue *bfqq,  in bfq_bfqq_handle_idle_busy_switch()  argument
1825 		idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq),  in bfq_bfqq_handle_idle_busy_switch()
1832 			bfqq->ttime.last_end_request +  in bfq_bfqq_handle_idle_busy_switch()
1845 	in_burst = bfq_bfqq_in_large_burst(bfqq);  in bfq_bfqq_handle_idle_busy_switch()
1847 		!BFQQ_TOTALLY_SEEKY(bfqq) &&  in bfq_bfqq_handle_idle_busy_switch()
1849 		time_is_before_jiffies(bfqq->soft_rt_next_start) &&  in bfq_bfqq_handle_idle_busy_switch()
1850 		bfqq->dispatched == 0 &&  in bfq_bfqq_handle_idle_busy_switch()
1851 		bfqq->entity.new_weight == 40;  in bfq_bfqq_handle_idle_busy_switch()
1853 		bfqq->entity.new_weight == 40;  in bfq_bfqq_handle_idle_busy_switch()
1867 		(bfqq->wr_coeff > 1 ||  in bfq_bfqq_handle_idle_busy_switch()
1868 		 (bfq_bfqq_sync(bfqq) &&  in bfq_bfqq_handle_idle_busy_switch()
1869 		  (bfqq->bic || RQ_BIC(rq)->stably_merged) &&  in bfq_bfqq_handle_idle_busy_switch()
1877 		bfq_bfqq_update_budg_for_activation(bfqd, bfqq,  in bfq_bfqq_handle_idle_busy_switch()
1893 	if (likely(!bfq_bfqq_just_created(bfqq)) &&  in bfq_bfqq_handle_idle_busy_switch()
1896 		    bfqq->budget_timeout +  in bfq_bfqq_handle_idle_busy_switch()
1898 		hlist_del_init(&bfqq->burst_list_node);  in bfq_bfqq_handle_idle_busy_switch()
1899 		bfq_clear_bfqq_in_large_burst(bfqq);  in bfq_bfqq_handle_idle_busy_switch()
1902 	bfq_clear_bfqq_just_created(bfqq);  in bfq_bfqq_handle_idle_busy_switch()
1905 		if (unlikely(time_is_after_jiffies(bfqq->split_time)))  in bfq_bfqq_handle_idle_busy_switch()
1907 			bfqq->split_time =  in bfq_bfqq_handle_idle_busy_switch()
1910 		if (time_is_before_jiffies(bfqq->split_time +  in bfq_bfqq_handle_idle_busy_switch()
1912 			bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq,  in bfq_bfqq_handle_idle_busy_switch()
1919 			if (old_wr_coeff != bfqq->wr_coeff)  in bfq_bfqq_handle_idle_busy_switch()
1920 				bfqq->entity.prio_changed = 1;  in bfq_bfqq_handle_idle_busy_switch()
1924 	bfqq->last_idle_bklogged = jiffies;  in bfq_bfqq_handle_idle_busy_switch()
1925 	bfqq->service_from_backlogged = 0;  in bfq_bfqq_handle_idle_busy_switch()
1926 	bfq_clear_bfqq_softrt_update(bfqq);  in bfq_bfqq_handle_idle_busy_switch()
1928 	bfq_add_bfqq_busy(bfqq);  in bfq_bfqq_handle_idle_busy_switch()
1975 	      bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) ||  in bfq_bfqq_handle_idle_busy_switch()
1976 	     bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue) ||  in bfq_bfqq_handle_idle_busy_switch()
1984 				   struct bfq_queue *bfqq)  in bfq_reset_inject_limit()  argument
1987 	bfqq->last_serv_time_ns = 0;  in bfq_reset_inject_limit()
2041 	if (bfq_bfqq_has_short_ttime(bfqq))  in bfq_reset_inject_limit()
2042 		bfqq->inject_limit = 0;  in bfq_reset_inject_limit()
2044 		bfqq->inject_limit = 1;  in bfq_reset_inject_limit()
2046 	bfqq->decrease_time_jif = jiffies;  in bfq_reset_inject_limit()
2049 static void bfq_update_io_intensity(struct bfq_queue *bfqq, u64 now_ns)  in bfq_update_io_intensity()  argument
2051 	u64 tot_io_time = now_ns - bfqq->io_start_time;  in bfq_update_io_intensity()
2053 	if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq->dispatched == 0)  in bfq_update_io_intensity()
2054 		bfqq->tot_idle_time +=  in bfq_update_io_intensity()
2055 			now_ns - bfqq->ttime.last_end_request;  in bfq_update_io_intensity()
2057 	if (unlikely(bfq_bfqq_just_created(bfqq)))  in bfq_update_io_intensity()
2064 	if (bfqq->tot_idle_time * 5 > tot_io_time)  in bfq_update_io_intensity()
2065 		bfq_clear_bfqq_IO_bound(bfqq);  in bfq_update_io_intensity()
2067 		bfq_mark_bfqq_IO_bound(bfqq);  in bfq_update_io_intensity()
2074 		bfqq->io_start_time = now_ns - (tot_io_time>>1);  in bfq_update_io_intensity()
2075 		bfqq->tot_idle_time >>= 1;  in bfq_update_io_intensity()
2130 static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_check_waker()  argument
2136 	    bfqd->last_completed_rq_bfqq == bfqq ||  in bfq_check_waker()
2137 	    bfq_bfqq_has_short_ttime(bfqq) ||  in bfq_check_waker()
2148 	    bfqq->tentative_waker_bfqq ||  in bfq_check_waker()
2149 	    now_ns > bfqq->waker_detection_started +  in bfq_check_waker()
2156 		bfqq->tentative_waker_bfqq =  in bfq_check_waker()
2158 		bfqq->num_waker_detections = 1;  in bfq_check_waker()
2159 		bfqq->waker_detection_started = now_ns;  in bfq_check_waker()
2160 		bfq_bfqq_name(bfqq->tentative_waker_bfqq, waker_name,  in bfq_check_waker()
2162 		bfq_log_bfqq(bfqd, bfqq, "set tentative waker %s", waker_name);  in bfq_check_waker()
2164 		bfqq->num_waker_detections++;  in bfq_check_waker()
2166 	if (bfqq->num_waker_detections == 3) {  in bfq_check_waker()
2167 		bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;  in bfq_check_waker()
2168 		bfqq->tentative_waker_bfqq = NULL;  in bfq_check_waker()
2169 		bfq_bfqq_name(bfqq->waker_bfqq, waker_name,  in bfq_check_waker()
2171 		bfq_log_bfqq(bfqd, bfqq, "set waker %s", waker_name);  in bfq_check_waker()
2193 		if (!hlist_unhashed(&bfqq->woken_list_node))  in bfq_check_waker()
2194 			hlist_del_init(&bfqq->woken_list_node);  in bfq_check_waker()
2195 		hlist_add_head(&bfqq->woken_list_node,  in bfq_check_waker()
2202 	struct bfq_queue *bfqq = RQ_BFQQ(rq);  in bfq_add_request()  local
2203 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_add_request()
2205 	unsigned int old_wr_coeff = bfqq->wr_coeff;  in bfq_add_request()
2209 	bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));  in bfq_add_request()
2210 	bfqq->queued[rq_is_sync(rq)]++;  in bfq_add_request()
2217 	if (bfq_bfqq_sync(bfqq) && RQ_BIC(rq)->requests <= 1) {  in bfq_add_request()
2218 		bfq_check_waker(bfqd, bfqq, now_ns);  in bfq_add_request()
2226 		if (time_is_before_eq_jiffies(bfqq->decrease_time_jif +  in bfq_add_request()
2228 			bfq_reset_inject_limit(bfqd, bfqq);  in bfq_add_request()
2255 		if (bfqq == bfqd->in_service_queue &&  in bfq_add_request()
2257 		     (bfqq->last_serv_time_ns > 0 &&  in bfq_add_request()
2259 		    time_is_before_eq_jiffies(bfqq->decrease_time_jif +  in bfq_add_request()
2287 	if (bfq_bfqq_sync(bfqq))  in bfq_add_request()
2288 		bfq_update_io_intensity(bfqq, now_ns);  in bfq_add_request()
2290 	elv_rb_add(&bfqq->sort_list, rq);  in bfq_add_request()
2295 	prev = bfqq->next_rq;  in bfq_add_request()
2296 	next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position);  in bfq_add_request()
2297 	bfqq->next_rq = next_rq;  in bfq_add_request()
2303 	if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq))  in bfq_add_request()
2304 		bfq_pos_tree_add_move(bfqd, bfqq);  in bfq_add_request()
2306 	if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */  in bfq_add_request()
2307 		bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff,  in bfq_add_request()
2312 				bfqq->last_wr_start_finish +  in bfq_add_request()
2314 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;  in bfq_add_request()
2315 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);  in bfq_add_request()
2318 			bfqq->entity.prio_changed = 1;  in bfq_add_request()
2320 		if (prev != bfqq->next_rq)  in bfq_add_request()
2321 			bfq_updated_next_req(bfqd, bfqq);  in bfq_add_request()
2351 		(old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive))  in bfq_add_request()
2352 		bfqq->last_wr_start_finish = jiffies;  in bfq_add_request()
2359 	struct bfq_queue *bfqq = bfqd->bio_bfqq;  in bfq_find_rq_fmerge()  local
2362 	if (bfqq)  in bfq_find_rq_fmerge()
2363 		return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio));  in bfq_find_rq_fmerge()
2395 	struct bfq_queue *bfqq = RQ_BFQQ(rq);  in bfq_remove_request()  local
2396 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_remove_request()
2399 	if (bfqq->next_rq == rq) {  in bfq_remove_request()
2400 		bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq);  in bfq_remove_request()
2401 		bfq_updated_next_req(bfqd, bfqq);  in bfq_remove_request()
2406 	bfqq->queued[sync]--;  in bfq_remove_request()
2412 	elv_rb_del(&bfqq->sort_list, rq);  in bfq_remove_request()
2418 	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {  in bfq_remove_request()
2419 		bfqq->next_rq = NULL;  in bfq_remove_request()
2421 		if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) {  in bfq_remove_request()
2422 			bfq_del_bfqq_busy(bfqq, false);  in bfq_remove_request()
2436 			bfqq->entity.budget = bfqq->entity.service = 0;  in bfq_remove_request()
2442 		if (bfqq->pos_root) {  in bfq_remove_request()
2443 			rb_erase(&bfqq->pos_node, bfqq->pos_root);  in bfq_remove_request()
2444 			bfqq->pos_root = NULL;  in bfq_remove_request()
2449 			bfq_pos_tree_add_move(bfqd, bfqq);  in bfq_remove_request()
2453 		bfqq->meta_pending--;  in bfq_remove_request()
2522 		struct bfq_queue *bfqq = RQ_BFQQ(req);  in bfq_request_merged()  local
2526 		if (!bfqq)  in bfq_request_merged()
2529 		bfqd = bfqq->bfqd;  in bfq_request_merged()
2532 		elv_rb_del(&bfqq->sort_list, req);  in bfq_request_merged()
2533 		elv_rb_add(&bfqq->sort_list, req);  in bfq_request_merged()
2536 		prev = bfqq->next_rq;  in bfq_request_merged()
2537 		next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req,  in bfq_request_merged()
2539 		bfqq->next_rq = next_rq;  in bfq_request_merged()
2545 		if (prev != bfqq->next_rq) {  in bfq_request_merged()
2546 			bfq_updated_next_req(bfqd, bfqq);  in bfq_request_merged()
2552 				bfq_pos_tree_add_move(bfqd, bfqq);  in bfq_request_merged()
2574 	struct bfq_queue *bfqq = RQ_BFQQ(rq),  in bfq_requests_merged()  local
2577 	if (!bfqq)  in bfq_requests_merged()
2589 	if (bfqq == next_bfqq &&  in bfq_requests_merged()
2597 	if (bfqq->next_rq == next)  in bfq_requests_merged()
2598 		bfqq->next_rq = rq;  in bfq_requests_merged()
2600 	bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags);  in bfq_requests_merged()
2612 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq)  in bfq_bfqq_end_wr()  argument
2628 	if (bfqq->wr_cur_max_time !=  in bfq_bfqq_end_wr()
2629 	    bfqq->bfqd->bfq_wr_rt_max_time)  in bfq_bfqq_end_wr()
2630 		bfqq->soft_rt_next_start = jiffies;  in bfq_bfqq_end_wr()
2632 	if (bfq_bfqq_busy(bfqq))  in bfq_bfqq_end_wr()
2633 		bfqq->bfqd->wr_busy_queues--;  in bfq_bfqq_end_wr()
2634 	bfqq->wr_coeff = 1;  in bfq_bfqq_end_wr()
2635 	bfqq->wr_cur_max_time = 0;  in bfq_bfqq_end_wr()
2636 	bfqq->last_wr_start_finish = jiffies;  in bfq_bfqq_end_wr()
2641 	bfqq->entity.prio_changed = 1;  in bfq_bfqq_end_wr()
2659 	struct bfq_queue *bfqq;  in bfq_end_wr()  local
2663 	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list)  in bfq_end_wr()
2664 		bfq_bfqq_end_wr(bfqq);  in bfq_end_wr()
2665 	list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list)  in bfq_end_wr()
2666 		bfq_bfqq_end_wr(bfqq);  in bfq_end_wr()
2688 					 struct bfq_queue *bfqq,  in bfqq_find_close()
2691 	struct rb_root *root = &bfqq_group(bfqq)->rq_pos_tree;  in bfqq_find_close()
2733 	struct bfq_queue *bfqq;  in bfq_find_close_cooperator()  local
2742 	bfqq = bfqq_find_close(bfqd, cur_bfqq, sector);  in bfq_find_close_cooperator()
2743 	if (!bfqq || bfqq == cur_bfqq)  in bfq_find_close_cooperator()
2746 	return bfqq;  in bfq_find_close_cooperator()
2750 bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)  in bfq_setup_merge()  argument
2766 		if (__bfqq == bfqq)  in bfq_setup_merge()
2771 	process_refs = bfqq_process_refs(bfqq);  in bfq_setup_merge()
2785 	if (new_bfqq->entity.parent != bfqq->entity.parent)  in bfq_setup_merge()
2788 	bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d",  in bfq_setup_merge()
2811 	bfqq->new_bfqq = new_bfqq;  in bfq_setup_merge()
2825 static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,  in bfq_may_be_close_cooperator()  argument
2831 	if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) ||  in bfq_may_be_close_cooperator()
2832 	    (bfqq->ioprio_class != new_bfqq->ioprio_class))  in bfq_may_be_close_cooperator()
2840 	if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq))  in bfq_may_be_close_cooperator()
2848 	if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq))  in bfq_may_be_close_cooperator()
2855 					     struct bfq_queue *bfqq);
2878 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_setup_cooperator()  argument
2884 	if (bfqq->new_bfqq)  in bfq_setup_cooperator()
2885 		return bfqq->new_bfqq;  in bfq_setup_cooperator()
2903 		if (bfq_bfqq_sync(bfqq) && bic->stable_merge_bfqq &&  in bfq_setup_cooperator()
2904 		    !bfq_bfqq_just_created(bfqq) &&  in bfq_setup_cooperator()
2905 		    time_is_before_jiffies(bfqq->split_time +  in bfq_setup_cooperator()
2907 		    time_is_before_jiffies(bfqq->creation_time +  in bfq_setup_cooperator()
2911 			int proc_ref = min(bfqq_process_refs(bfqq),  in bfq_setup_cooperator()
2919 			if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&  in bfq_setup_cooperator()
2923 					bfq_setup_merge(bfqq, stable_merge_bfqq);  in bfq_setup_cooperator()
2988 	if (bfq_too_late_for_merging(bfqq))  in bfq_setup_cooperator()
2991 	if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq))  in bfq_setup_cooperator()
3000 	if (in_service_bfqq && in_service_bfqq != bfqq &&  in bfq_setup_cooperator()
3004 	    bfqq->entity.parent == in_service_bfqq->entity.parent &&  in bfq_setup_cooperator()
3005 	    bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) {  in bfq_setup_cooperator()
3006 		new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq);  in bfq_setup_cooperator()
3015 	new_bfqq = bfq_find_close_cooperator(bfqd, bfqq,  in bfq_setup_cooperator()
3019 	    bfq_may_be_close_cooperator(bfqq, new_bfqq))  in bfq_setup_cooperator()
3020 		return bfq_setup_merge(bfqq, new_bfqq);  in bfq_setup_cooperator()
3025 static void bfq_bfqq_save_state(struct bfq_queue *bfqq)  in bfq_bfqq_save_state()  argument
3027 	struct bfq_io_cq *bic = bfqq->bic;  in bfq_bfqq_save_state()
3037 	bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns;  in bfq_bfqq_save_state()
3038 	bic->saved_inject_limit = bfqq->inject_limit;  in bfq_bfqq_save_state()
3039 	bic->saved_decrease_time_jif = bfqq->decrease_time_jif;  in bfq_bfqq_save_state()
3041 	bic->saved_weight = bfqq->entity.orig_weight;  in bfq_bfqq_save_state()
3042 	bic->saved_ttime = bfqq->ttime;  in bfq_bfqq_save_state()
3043 	bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);  in bfq_bfqq_save_state()
3044 	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);  in bfq_bfqq_save_state()
3045 	bic->saved_io_start_time = bfqq->io_start_time;  in bfq_bfqq_save_state()
3046 	bic->saved_tot_idle_time = bfqq->tot_idle_time;  in bfq_bfqq_save_state()
3047 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);  in bfq_bfqq_save_state()
3048 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);  in bfq_bfqq_save_state()
3049 	if (unlikely(bfq_bfqq_just_created(bfqq) &&  in bfq_bfqq_save_state()
3050 		     !bfq_bfqq_in_large_burst(bfqq) &&  in bfq_bfqq_save_state()
3051 		     bfqq->bfqd->low_latency)) {  in bfq_bfqq_save_state()
3061 		bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff;  in bfq_bfqq_save_state()
3063 		bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd);  in bfq_bfqq_save_state()
3066 		bic->saved_wr_coeff = bfqq->wr_coeff;  in bfq_bfqq_save_state()
3068 			bfqq->wr_start_at_switch_to_srt;  in bfq_bfqq_save_state()
3069 		bic->saved_service_from_wr = bfqq->service_from_wr;  in bfq_bfqq_save_state()
3070 		bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;  in bfq_bfqq_save_state()
3071 		bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;  in bfq_bfqq_save_state()
3086 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_release_process_ref()  argument
3099 	if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) &&  in bfq_release_process_ref()
3100 	    bfqq != bfqd->in_service_queue)  in bfq_release_process_ref()
3101 		bfq_del_bfqq_busy(bfqq, false);  in bfq_release_process_ref()
3103 	bfq_reassign_last_bfqq(bfqq, NULL);  in bfq_release_process_ref()
3105 	bfq_put_queue(bfqq);  in bfq_release_process_ref()
3110 		struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)  in bfq_merge_bfqqs()  argument
3112 	bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu",  in bfq_merge_bfqqs()
3115 	bfq_bfqq_save_state(bfqq);  in bfq_merge_bfqqs()
3117 	if (bfq_bfqq_IO_bound(bfqq))  in bfq_merge_bfqqs()
3119 	bfq_clear_bfqq_IO_bound(bfqq);  in bfq_merge_bfqqs()
3128 	if (bfqq->waker_bfqq && !new_bfqq->waker_bfqq &&  in bfq_merge_bfqqs()
3129 	    bfqq->waker_bfqq != new_bfqq) {  in bfq_merge_bfqqs()
3130 		new_bfqq->waker_bfqq = bfqq->waker_bfqq;  in bfq_merge_bfqqs()
3153 	if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) {  in bfq_merge_bfqqs()
3154 		new_bfqq->wr_coeff = bfqq->wr_coeff;  in bfq_merge_bfqqs()
3155 		new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time;  in bfq_merge_bfqqs()
3156 		new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish;  in bfq_merge_bfqqs()
3158 			bfqq->wr_start_at_switch_to_srt;  in bfq_merge_bfqqs()
3164 	if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */  in bfq_merge_bfqqs()
3165 		bfqq->wr_coeff = 1;  in bfq_merge_bfqqs()
3166 		bfqq->entity.prio_changed = 1;  in bfq_merge_bfqqs()
3167 		if (bfq_bfqq_busy(bfqq))  in bfq_merge_bfqqs()
3200 	bfqq->bic = NULL;  in bfq_merge_bfqqs()
3202 	bfq_reassign_last_bfqq(bfqq, new_bfqq);  in bfq_merge_bfqqs()
3204 	bfq_release_process_ref(bfqd, bfqq);  in bfq_merge_bfqqs()
3212 	struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq;  in bfq_allow_bio_merge()  local
3224 	if (!bfqq)  in bfq_allow_bio_merge()
3231 	new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false, bfqd->bio_bic);  in bfq_allow_bio_merge()
3240 		bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq,  in bfq_allow_bio_merge()
3247 		bfqq = new_bfqq;  in bfq_allow_bio_merge()
3255 		bfqd->bio_bfqq = bfqq;  in bfq_allow_bio_merge()
3258 	return bfqq == RQ_BFQQ(rq);  in bfq_allow_bio_merge()
3268 				   struct bfq_queue *bfqq)  in bfq_set_budget_timeout()  argument
3272 	if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time)  in bfq_set_budget_timeout()
3275 		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;  in bfq_set_budget_timeout()
3279 	bfqq->budget_timeout = jiffies +  in bfq_set_budget_timeout()
3284 				       struct bfq_queue *bfqq)  in __bfq_set_in_service_queue()  argument
3286 	if (bfqq) {  in __bfq_set_in_service_queue()
3287 		bfq_clear_bfqq_fifo_expire(bfqq);  in __bfq_set_in_service_queue()
3291 		if (time_is_before_jiffies(bfqq->last_wr_start_finish) &&  in __bfq_set_in_service_queue()
3292 		    bfqq->wr_coeff > 1 &&  in __bfq_set_in_service_queue()
3293 		    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&  in __bfq_set_in_service_queue()
3294 		    time_is_before_jiffies(bfqq->budget_timeout)) {  in __bfq_set_in_service_queue()
3319 			if (time_after(bfqq->budget_timeout,  in __bfq_set_in_service_queue()
3320 				       bfqq->last_wr_start_finish))  in __bfq_set_in_service_queue()
3321 				bfqq->last_wr_start_finish +=  in __bfq_set_in_service_queue()
3322 					jiffies - bfqq->budget_timeout;  in __bfq_set_in_service_queue()
3324 				bfqq->last_wr_start_finish = jiffies;  in __bfq_set_in_service_queue()
3327 		bfq_set_budget_timeout(bfqd, bfqq);  in __bfq_set_in_service_queue()
3328 		bfq_log_bfqq(bfqd, bfqq,  in __bfq_set_in_service_queue()
3330 			     bfqq->entity.budget);  in __bfq_set_in_service_queue()
3333 	bfqd->in_service_queue = bfqq;  in __bfq_set_in_service_queue()
3342 	struct bfq_queue *bfqq = bfq_get_next_queue(bfqd);  in bfq_set_in_service_queue()  local
3344 	__bfq_set_in_service_queue(bfqd, bfqq);  in bfq_set_in_service_queue()
3345 	return bfqq;  in bfq_set_in_service_queue()
3350 	struct bfq_queue *bfqq = bfqd->in_service_queue;  in bfq_arm_slice_timer()  local
3353 	bfq_mark_bfqq_wait_request(bfqq);  in bfq_arm_slice_timer()
3371 	if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 &&  in bfq_arm_slice_timer()
3372 	    !bfq_asymmetric_scenario(bfqd, bfqq))  in bfq_arm_slice_timer()
3374 	else if (bfqq->wr_coeff > 1)  in bfq_arm_slice_timer()
3382 	bfqg_stats_set_start_idle_time(bfqq_group(bfqq));  in bfq_arm_slice_timer()
3635 	struct bfq_queue *bfqq = RQ_BFQQ(rq);  in bfq_dispatch_remove()  local
3649 	bfqq->dispatched++;  in bfq_dispatch_remove()
3857 						 struct bfq_queue *bfqq)  in idling_needed_for_service_guarantees()  argument
3862 	if (unlikely(!bfqq_process_refs(bfqq)))  in idling_needed_for_service_guarantees()
3865 	return (bfqq->wr_coeff > 1 &&  in idling_needed_for_service_guarantees()
3869 		 bfqq->dispatched + 4)) ||  in idling_needed_for_service_guarantees()
3870 		bfq_asymmetric_scenario(bfqd, bfqq) ||  in idling_needed_for_service_guarantees()
3874 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in __bfq_bfqq_expire()  argument
3883 	if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq))  in __bfq_bfqq_expire()
3884 		bfq_mark_bfqq_split_coop(bfqq);  in __bfq_bfqq_expire()
3899 	if (RB_EMPTY_ROOT(&bfqq->sort_list) &&  in __bfq_bfqq_expire()
3901 	      idling_needed_for_service_guarantees(bfqd, bfqq))) {  in __bfq_bfqq_expire()
3902 		if (bfqq->dispatched == 0)  in __bfq_bfqq_expire()
3909 			bfqq->budget_timeout = jiffies;  in __bfq_bfqq_expire()
3911 		bfq_del_bfqq_busy(bfqq, true);  in __bfq_bfqq_expire()
3913 		bfq_requeue_bfqq(bfqd, bfqq, true);  in __bfq_bfqq_expire()
3919 			     !RB_EMPTY_ROOT(&bfqq->sort_list)))  in __bfq_bfqq_expire()
3920 			bfq_pos_tree_add_move(bfqd, bfqq);  in __bfq_bfqq_expire()
3943 				     struct bfq_queue *bfqq,  in __bfq_bfqq_recalc_budget()  argument
3951 	if (bfqq->wr_coeff == 1)  in __bfq_bfqq_recalc_budget()
3952 		budget = bfqq->max_budget;  in __bfq_bfqq_recalc_budget()
3961 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d",  in __bfq_bfqq_recalc_budget()
3962 		bfqq->entity.budget, bfq_bfqq_budget_left(bfqq));  in __bfq_bfqq_recalc_budget()
3963 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d",  in __bfq_bfqq_recalc_budget()
3965 	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d",  in __bfq_bfqq_recalc_budget()
3966 		bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue));  in __bfq_bfqq_recalc_budget()
3968 	if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) {  in __bfq_bfqq_recalc_budget()
3999 			if (bfqq->dispatched > 0) /* still outstanding reqs */  in __bfq_bfqq_recalc_budget()
4062 			budget = max_t(int, bfqq->entity.service, min_budget);  in __bfq_bfqq_recalc_budget()
4067 	} else if (!bfq_bfqq_sync(bfqq)) {  in __bfq_bfqq_recalc_budget()
4077 	bfqq->max_budget = budget;  in __bfq_bfqq_recalc_budget()
4081 		bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget);  in __bfq_bfqq_recalc_budget()
4093 	next_rq = bfqq->next_rq;  in __bfq_bfqq_recalc_budget()
4095 		bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget,  in __bfq_bfqq_recalc_budget()
4096 					    bfq_serv_to_charge(next_rq, bfqq));  in __bfq_bfqq_recalc_budget()
4098 	bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d",  in __bfq_bfqq_recalc_budget()
4100 			bfqq->entity.budget);  in __bfq_bfqq_recalc_budget()
4134 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_is_slow()  argument
4140 	bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */  in bfq_bfqq_is_slow()
4142 	if (!bfq_bfqq_sync(bfqq))  in bfq_bfqq_is_slow()
4183 		slow = bfqq->entity.service < bfqd->bfq_max_budget / 2;  in bfq_bfqq_is_slow()
4186 	bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow);  in bfq_bfqq_is_slow()
4285 						struct bfq_queue *bfqq)  in bfq_bfqq_softrt_next_start()  argument
4287 	return max3(bfqq->soft_rt_next_start,  in bfq_bfqq_softrt_next_start()
4288 		    bfqq->last_idle_bklogged +  in bfq_bfqq_softrt_next_start()
4289 		    HZ * bfqq->service_from_backlogged /  in bfq_bfqq_softrt_next_start()
4291 		    jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4);  in bfq_bfqq_softrt_next_start()
4321 		     struct bfq_queue *bfqq,  in bfq_bfqq_expire()  argument
4327 	struct bfq_entity *entity = &bfqq->entity;  in bfq_bfqq_expire()
4332 	slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta);  in bfq_bfqq_expire()
4349 	if (bfqq->wr_coeff == 1 &&  in bfq_bfqq_expire()
4352 	      bfq_bfqq_budget_left(bfqq) >=  entity->budget / 3)))  in bfq_bfqq_expire()
4353 		bfq_bfqq_charge_time(bfqd, bfqq, delta);  in bfq_bfqq_expire()
4355 	if (bfqd->low_latency && bfqq->wr_coeff == 1)  in bfq_bfqq_expire()
4356 		bfqq->last_wr_start_finish = jiffies;  in bfq_bfqq_expire()
4359 	    RB_EMPTY_ROOT(&bfqq->sort_list)) {  in bfq_bfqq_expire()
4372 		if (bfqq->dispatched == 0)  in bfq_bfqq_expire()
4373 			bfqq->soft_rt_next_start =  in bfq_bfqq_expire()
4374 				bfq_bfqq_softrt_next_start(bfqd, bfqq);  in bfq_bfqq_expire()
4375 		else if (bfqq->dispatched > 0) {  in bfq_bfqq_expire()
4380 			bfq_mark_bfqq_softrt_update(bfqq);  in bfq_bfqq_expire()
4384 	bfq_log_bfqq(bfqd, bfqq,  in bfq_bfqq_expire()
4386 		slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq));  in bfq_bfqq_expire()
4400 	__bfq_bfqq_recalc_budget(bfqd, bfqq, reason);  in bfq_bfqq_expire()
4401 	if (__bfq_bfqq_expire(bfqd, bfqq, reason))  in bfq_bfqq_expire()
4406 	if (!bfq_bfqq_busy(bfqq) &&  in bfq_bfqq_expire()
4409 		bfq_mark_bfqq_non_blocking_wait_rq(bfqq);  in bfq_bfqq_expire()
4445 static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq)  in bfq_bfqq_budget_timeout()  argument
4447 	return time_is_before_eq_jiffies(bfqq->budget_timeout);  in bfq_bfqq_budget_timeout()
4458 static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)  in bfq_may_expire_for_budg_timeout()  argument
4460 	bfq_log_bfqq(bfqq->bfqd, bfqq,  in bfq_may_expire_for_budg_timeout()
4462 		bfq_bfqq_wait_request(bfqq),  in bfq_may_expire_for_budg_timeout()
4463 			bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3,  in bfq_may_expire_for_budg_timeout()
4464 		bfq_bfqq_budget_timeout(bfqq));  in bfq_may_expire_for_budg_timeout()
4466 	return (!bfq_bfqq_wait_request(bfqq) ||  in bfq_may_expire_for_budg_timeout()
4467 		bfq_bfqq_budget_left(bfqq) >=  bfqq->entity.budget / 3)  in bfq_may_expire_for_budg_timeout()
4469 		bfq_bfqq_budget_timeout(bfqq);  in bfq_may_expire_for_budg_timeout()
4473 					     struct bfq_queue *bfqq)  in idling_boosts_thr_without_issues()  argument
4481 	if (unlikely(!bfqq_process_refs(bfqq)))  in idling_boosts_thr_without_issues()
4484 	bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) &&  in idling_boosts_thr_without_issues()
4485 		bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq);  in idling_boosts_thr_without_issues()
4572 static bool bfq_better_to_idle(struct bfq_queue *bfqq)  in bfq_better_to_idle()  argument
4574 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_better_to_idle()
4578 	if (unlikely(!bfqq_process_refs(bfqq)))  in bfq_better_to_idle()
4592 	if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) ||  in bfq_better_to_idle()
4593 	   bfq_class_idle(bfqq))  in bfq_better_to_idle()
4597 		idling_boosts_thr_without_issues(bfqd, bfqq);  in bfq_better_to_idle()
4600 		idling_needed_for_service_guarantees(bfqd, bfqq);  in bfq_better_to_idle()
4623 static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq)  in bfq_bfqq_must_idle()  argument
4625 	return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq);  in bfq_bfqq_must_idle()
4638 	struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue;  in bfq_choose_bfqq_for_injection()  local
4686 	list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list)  in bfq_choose_bfqq_for_injection()
4687 		if (!RB_EMPTY_ROOT(&bfqq->sort_list) &&  in bfq_choose_bfqq_for_injection()
4688 		    (in_serv_always_inject || bfqq->wr_coeff > 1) &&  in bfq_choose_bfqq_for_injection()
4689 		    bfq_serv_to_charge(bfqq->next_rq, bfqq) <=  in bfq_choose_bfqq_for_injection()
4690 		    bfq_bfqq_budget_left(bfqq)) {  in bfq_choose_bfqq_for_injection()
4709 			    blk_rq_sectors(bfqq->next_rq) >=  in bfq_choose_bfqq_for_injection()
4717 				return bfqq;  in bfq_choose_bfqq_for_injection()
4730 	struct bfq_queue *bfqq;  in bfq_select_queue()  local
4734 	bfqq = bfqd->in_service_queue;  in bfq_select_queue()
4735 	if (!bfqq)  in bfq_select_queue()
4738 	bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue");  in bfq_select_queue()
4747 	if (bfq_may_expire_for_budg_timeout(bfqq) &&  in bfq_select_queue()
4748 	    !bfq_bfqq_must_idle(bfqq))  in bfq_select_queue()
4758 	next_rq = bfqq->next_rq;  in bfq_select_queue()
4764 		if (bfq_serv_to_charge(next_rq, bfqq) >  in bfq_select_queue()
4765 			bfq_bfqq_budget_left(bfqq)) {  in bfq_select_queue()
4780 			if (bfq_bfqq_wait_request(bfqq)) {  in bfq_select_queue()
4794 				bfq_clear_bfqq_wait_request(bfqq);  in bfq_select_queue()
4809 	if (bfq_bfqq_wait_request(bfqq) ||  in bfq_select_queue()
4810 	    (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) {  in bfq_select_queue()
4812 			bfqq->bic && bfqq->bic->bfqq[0] &&  in bfq_select_queue()
4813 			bfq_bfqq_busy(bfqq->bic->bfqq[0]) &&  in bfq_select_queue()
4814 			bfqq->bic->bfqq[0]->next_rq ?  in bfq_select_queue()
4815 			bfqq->bic->bfqq[0] : NULL;  in bfq_select_queue()
4817 			!hlist_empty(&bfqq->woken_list) ?  in bfq_select_queue()
4818 			container_of(bfqq->woken_list.first,  in bfq_select_queue()
4904 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&  in bfq_select_queue()
4907 			bfqq = bfqq->bic->bfqq[0];  in bfq_select_queue()
4908 		else if (bfqq->waker_bfqq &&  in bfq_select_queue()
4909 			   bfq_bfqq_busy(bfqq->waker_bfqq) &&  in bfq_select_queue()
4910 			   bfqq->waker_bfqq->next_rq &&  in bfq_select_queue()
4911 			   bfq_serv_to_charge(bfqq->waker_bfqq->next_rq,  in bfq_select_queue()
4912 					      bfqq->waker_bfqq) <=  in bfq_select_queue()
4913 			   bfq_bfqq_budget_left(bfqq->waker_bfqq)  in bfq_select_queue()
4915 			bfqq = bfqq->waker_bfqq;  in bfq_select_queue()
4923 			bfqq = blocked_bfqq;  in bfq_select_queue()
4924 		else if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&  in bfq_select_queue()
4925 			 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 ||  in bfq_select_queue()
4926 			  !bfq_bfqq_has_short_ttime(bfqq)))  in bfq_select_queue()
4927 			bfqq = bfq_choose_bfqq_for_injection(bfqd);  in bfq_select_queue()
4929 			bfqq = NULL;  in bfq_select_queue()
4936 	bfq_bfqq_expire(bfqd, bfqq, false, reason);  in bfq_select_queue()
4938 	bfqq = bfq_set_in_service_queue(bfqd);  in bfq_select_queue()
4939 	if (bfqq) {  in bfq_select_queue()
4940 		bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue");  in bfq_select_queue()
4944 	if (bfqq)  in bfq_select_queue()
4945 		bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue");  in bfq_select_queue()
4949 	return bfqq;  in bfq_select_queue()
4952 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_update_wr_data()  argument
4954 	struct bfq_entity *entity = &bfqq->entity;  in bfq_update_wr_data()
4956 	if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */  in bfq_update_wr_data()
4957 		bfq_log_bfqq(bfqd, bfqq,  in bfq_update_wr_data()
4959 			jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish),  in bfq_update_wr_data()
4960 			jiffies_to_msecs(bfqq->wr_cur_max_time),  in bfq_update_wr_data()
4961 			bfqq->wr_coeff,  in bfq_update_wr_data()
4962 			bfqq->entity.weight, bfqq->entity.orig_weight);  in bfq_update_wr_data()
4965 			bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change");  in bfq_update_wr_data()
4972 		if (bfq_bfqq_in_large_burst(bfqq))  in bfq_update_wr_data()
4973 			bfq_bfqq_end_wr(bfqq);  in bfq_update_wr_data()
4974 		else if (time_is_before_jiffies(bfqq->last_wr_start_finish +  in bfq_update_wr_data()
4975 						bfqq->wr_cur_max_time)) {  in bfq_update_wr_data()
4976 			if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time ||  in bfq_update_wr_data()
4977 			time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +  in bfq_update_wr_data()
4987 				bfq_bfqq_end_wr(bfqq);  in bfq_update_wr_data()
4993 				switch_back_to_interactive_wr(bfqq, bfqd);  in bfq_update_wr_data()
4994 				bfqq->entity.prio_changed = 1;  in bfq_update_wr_data()
4997 		if (bfqq->wr_coeff > 1 &&  in bfq_update_wr_data()
4998 		    bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time &&  in bfq_update_wr_data()
4999 		    bfqq->service_from_wr > max_service_from_wr) {  in bfq_update_wr_data()
5001 			bfq_bfqq_end_wr(bfqq);  in bfq_update_wr_data()
5012 	if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1))  in bfq_update_wr_data()
5021 						 struct bfq_queue *bfqq)  in bfq_dispatch_rq_from_bfqq()  argument
5023 	struct request *rq = bfqq->next_rq;  in bfq_dispatch_rq_from_bfqq()
5026 	service_to_charge = bfq_serv_to_charge(rq, bfqq);  in bfq_dispatch_rq_from_bfqq()
5028 	bfq_bfqq_served(bfqq, service_to_charge);  in bfq_dispatch_rq_from_bfqq()
5030 	if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) {  in bfq_dispatch_rq_from_bfqq()
5037 	if (bfqq != bfqd->in_service_queue)  in bfq_dispatch_rq_from_bfqq()
5051 	bfq_update_wr_data(bfqd, bfqq);  in bfq_dispatch_rq_from_bfqq()
5058 	if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq)))  in bfq_dispatch_rq_from_bfqq()
5061 	bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED);  in bfq_dispatch_rq_from_bfqq()
5083 	struct bfq_queue *bfqq = NULL;  in __bfq_dispatch_request()  local
5090 		bfqq = RQ_BFQQ(rq);  in __bfq_dispatch_request()
5092 		if (bfqq) {  in __bfq_dispatch_request()
5099 			bfqq->dispatched++;  in __bfq_dispatch_request()
5151 	bfqq = bfq_select_queue(bfqd);  in __bfq_dispatch_request()
5152 	if (!bfqq)  in __bfq_dispatch_request()
5155 	rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq);  in __bfq_dispatch_request()
5173 	struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL;  in bfq_update_dispatch_stats()  local
5175 	if (!idle_timer_disabled && !bfqq)  in bfq_update_dispatch_stats()
5203 	if (bfqq) {  in bfq_update_dispatch_stats()
5204 		struct bfq_group *bfqg = bfqq_group(bfqq);  in bfq_update_dispatch_stats()
5252 void bfq_put_queue(struct bfq_queue *bfqq)  in bfq_put_queue()  argument
5256 	struct bfq_group *bfqg = bfqq_group(bfqq);  in bfq_put_queue()
5258 	bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref);  in bfq_put_queue()
5260 	bfqq->ref--;  in bfq_put_queue()
5261 	if (bfqq->ref)  in bfq_put_queue()
5264 	if (!hlist_unhashed(&bfqq->burst_list_node)) {  in bfq_put_queue()
5265 		hlist_del_init(&bfqq->burst_list_node);  in bfq_put_queue()
5292 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)  in bfq_put_queue()
5293 			bfqq->bfqd->burst_size--;  in bfq_put_queue()
5312 	if (!hlist_unhashed(&bfqq->woken_list_node))  in bfq_put_queue()
5313 		hlist_del_init(&bfqq->woken_list_node);  in bfq_put_queue()
5316 	hlist_for_each_entry_safe(item, n, &bfqq->woken_list,  in bfq_put_queue()
5322 	if (bfqq->bfqd->last_completed_rq_bfqq == bfqq)  in bfq_put_queue()
5323 		bfqq->bfqd->last_completed_rq_bfqq = NULL;  in bfq_put_queue()
5325 	kmem_cache_free(bfq_pool, bfqq);  in bfq_put_queue()
5329 static void bfq_put_stable_ref(struct bfq_queue *bfqq)  in bfq_put_stable_ref()  argument
5331 	bfqq->stable_ref--;  in bfq_put_stable_ref()
5332 	bfq_put_queue(bfqq);  in bfq_put_stable_ref()
5335 void bfq_put_cooperator(struct bfq_queue *bfqq)  in bfq_put_cooperator()  argument
5344 	__bfqq = bfqq->new_bfqq;  in bfq_put_cooperator()
5346 		if (__bfqq == bfqq)  in bfq_put_cooperator()
5354 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_exit_bfqq()  argument
5356 	if (bfqq == bfqd->in_service_queue) {  in bfq_exit_bfqq()
5357 		__bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT);  in bfq_exit_bfqq()
5361 	bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref);  in bfq_exit_bfqq()
5363 	bfq_put_cooperator(bfqq);  in bfq_exit_bfqq()
5365 	bfq_release_process_ref(bfqd, bfqq);  in bfq_exit_bfqq()
5370 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_exit_icq_bfqq()  local
5373 	if (bfqq)  in bfq_exit_icq_bfqq()
5374 		bfqd = bfqq->bfqd; /* NULL if scheduler already exited */  in bfq_exit_icq_bfqq()
5376 	if (bfqq && bfqd) {  in bfq_exit_icq_bfqq()
5380 		bfqq->bic = NULL;  in bfq_exit_icq_bfqq()
5381 		bfq_exit_bfqq(bfqd, bfqq);  in bfq_exit_icq_bfqq()
5418 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)  in bfq_set_next_ioprio_data()  argument
5422 	struct bfq_data *bfqd = bfqq->bfqd;  in bfq_set_next_ioprio_data()
5431 			bdi_dev_name(bfqq->bfqd->queue->disk->bdi),  in bfq_set_next_ioprio_data()
5438 		bfqq->new_ioprio = task_nice_ioprio(tsk);  in bfq_set_next_ioprio_data()
5439 		bfqq->new_ioprio_class = task_nice_ioclass(tsk);  in bfq_set_next_ioprio_data()
5442 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5443 		bfqq->new_ioprio_class = IOPRIO_CLASS_RT;  in bfq_set_next_ioprio_data()
5446 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5447 		bfqq->new_ioprio_class = IOPRIO_CLASS_BE;  in bfq_set_next_ioprio_data()
5450 		bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE;  in bfq_set_next_ioprio_data()
5451 		bfqq->new_ioprio = 7;  in bfq_set_next_ioprio_data()
5455 	if (bfqq->new_ioprio >= IOPRIO_NR_LEVELS) {  in bfq_set_next_ioprio_data()
5457 			bfqq->new_ioprio);  in bfq_set_next_ioprio_data()
5458 		bfqq->new_ioprio = IOPRIO_NR_LEVELS - 1;  in bfq_set_next_ioprio_data()
5461 	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);  in bfq_set_next_ioprio_data()
5462 	bfq_log_bfqq(bfqd, bfqq, "new_ioprio %d new_weight %d",  in bfq_set_next_ioprio_data()
5463 		     bfqq->new_ioprio, bfqq->entity.new_weight);  in bfq_set_next_ioprio_data()
5464 	bfqq->entity.prio_changed = 1;  in bfq_set_next_ioprio_data()
5475 	struct bfq_queue *bfqq;  in bfq_check_ioprio_change()  local
5487 	bfqq = bic_to_bfqq(bic, false);  in bfq_check_ioprio_change()
5488 	if (bfqq) {  in bfq_check_ioprio_change()
5489 		bfq_release_process_ref(bfqd, bfqq);  in bfq_check_ioprio_change()
5490 		bfqq = bfq_get_queue(bfqd, bio, false, bic, true);  in bfq_check_ioprio_change()
5491 		bic_set_bfqq(bic, bfqq, false);  in bfq_check_ioprio_change()
5494 	bfqq = bic_to_bfqq(bic, true);  in bfq_check_ioprio_change()
5495 	if (bfqq)  in bfq_check_ioprio_change()
5496 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_check_ioprio_change()
5499 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_init_bfqq()  argument
5504 	RB_CLEAR_NODE(&bfqq->entity.rb_node);  in bfq_init_bfqq()
5505 	INIT_LIST_HEAD(&bfqq->fifo);  in bfq_init_bfqq()
5506 	INIT_HLIST_NODE(&bfqq->burst_list_node);  in bfq_init_bfqq()
5507 	INIT_HLIST_NODE(&bfqq->woken_list_node);  in bfq_init_bfqq()
5508 	INIT_HLIST_HEAD(&bfqq->woken_list);  in bfq_init_bfqq()
5510 	bfqq->ref = 0;  in bfq_init_bfqq()
5511 	bfqq->bfqd = bfqd;  in bfq_init_bfqq()
5514 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_init_bfqq()
5522 		if (!bfq_class_idle(bfqq))  in bfq_init_bfqq()
5524 			bfq_mark_bfqq_has_short_ttime(bfqq);  in bfq_init_bfqq()
5525 		bfq_mark_bfqq_sync(bfqq);  in bfq_init_bfqq()
5526 		bfq_mark_bfqq_just_created(bfqq);  in bfq_init_bfqq()
5528 		bfq_clear_bfqq_sync(bfqq);  in bfq_init_bfqq()
5531 	bfqq->ttime.last_end_request = now_ns + 1;  in bfq_init_bfqq()
5533 	bfqq->creation_time = jiffies;  in bfq_init_bfqq()
5535 	bfqq->io_start_time = now_ns;  in bfq_init_bfqq()
5537 	bfq_mark_bfqq_IO_bound(bfqq);  in bfq_init_bfqq()
5539 	bfqq->pid = pid;  in bfq_init_bfqq()
5542 	bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3;  in bfq_init_bfqq()
5543 	bfqq->budget_timeout = bfq_smallest_from_now();  in bfq_init_bfqq()
5545 	bfqq->wr_coeff = 1;  in bfq_init_bfqq()
5546 	bfqq->last_wr_start_finish = jiffies;  in bfq_init_bfqq()
5547 	bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now();  in bfq_init_bfqq()
5548 	bfqq->split_time = bfq_smallest_from_now();  in bfq_init_bfqq()
5559 	bfqq->soft_rt_next_start = jiffies;  in bfq_init_bfqq()
5562 	bfqq->seek_history = 1;  in bfq_init_bfqq()
5585 bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_do_early_stable_merge()  argument
5590 		bfq_setup_merge(bfqq, last_bfqq_created);  in bfq_do_early_stable_merge()
5593 		return bfqq;  in bfq_do_early_stable_merge()
5605 	bfqq->bic = bic;  in bfq_do_early_stable_merge()
5606 	bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq);  in bfq_do_early_stable_merge()
5658 						      struct bfq_queue *bfqq,  in bfq_do_or_sched_stable_merge()  argument
5661 	struct bfq_queue **source_bfqq = bfqq->entity.parent ?  in bfq_do_or_sched_stable_merge()
5662 		&bfqq->entity.parent->last_bfqq_created :  in bfq_do_or_sched_stable_merge()
5689 			bfqq->creation_time) ||  in bfq_do_or_sched_stable_merge()
5690 		bfqq->entity.parent != last_bfqq_created->entity.parent ||  in bfq_do_or_sched_stable_merge()
5691 		bfqq->ioprio != last_bfqq_created->ioprio ||  in bfq_do_or_sched_stable_merge()
5692 		bfqq->ioprio_class != last_bfqq_created->ioprio_class)  in bfq_do_or_sched_stable_merge()
5693 		*source_bfqq = bfqq;  in bfq_do_or_sched_stable_merge()
5696 				 bfqq->creation_time)) {  in bfq_do_or_sched_stable_merge()
5704 			bfqq = bfq_do_early_stable_merge(bfqd, bfqq,  in bfq_do_or_sched_stable_merge()
5726 	return bfqq;  in bfq_do_or_sched_stable_merge()
5738 	struct bfq_queue *bfqq;  in bfq_get_queue()  local
5745 		bfqq = *async_bfqq;  in bfq_get_queue()
5746 		if (bfqq)  in bfq_get_queue()
5750 	bfqq = kmem_cache_alloc_node(bfq_pool,  in bfq_get_queue()
5754 	if (bfqq) {  in bfq_get_queue()
5755 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,  in bfq_get_queue()
5757 		bfq_init_entity(&bfqq->entity, bfqg);  in bfq_get_queue()
5758 		bfq_log_bfqq(bfqd, bfqq, "allocated");  in bfq_get_queue()
5760 		bfqq = &bfqd->oom_bfqq;  in bfq_get_queue()
5761 		bfq_log_bfqq(bfqd, bfqq, "using oom bfqq");  in bfq_get_queue()
5770 		bfqq->ref++; /*  in bfq_get_queue()
5777 		bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d",  in bfq_get_queue()
5778 			     bfqq, bfqq->ref);  in bfq_get_queue()
5779 		*async_bfqq = bfqq;  in bfq_get_queue()
5783 	bfqq->ref++; /* get a process reference to this queue */  in bfq_get_queue()
5785 	if (bfqq != &bfqd->oom_bfqq && is_sync && !respawn)  in bfq_get_queue()
5786 		bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic);  in bfq_get_queue()
5787 	return bfqq;  in bfq_get_queue()
5791 				    struct bfq_queue *bfqq)  in bfq_update_io_thinktime()  argument
5793 	struct bfq_ttime *ttime = &bfqq->ttime;  in bfq_update_io_thinktime()
5801 	if (bfqq->dispatched || bfq_bfqq_busy(bfqq))  in bfq_update_io_thinktime()
5803 	elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;  in bfq_update_io_thinktime()
5813 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_update_io_seektime()  argument
5816 	bfqq->seek_history <<= 1;  in bfq_update_io_seektime()
5817 	bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq);  in bfq_update_io_seektime()
5819 	if (bfqq->wr_coeff > 1 &&  in bfq_update_io_seektime()
5820 	    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&  in bfq_update_io_seektime()
5821 	    BFQQ_TOTALLY_SEEKY(bfqq)) {  in bfq_update_io_seektime()
5822 		if (time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +  in bfq_update_io_seektime()
5830 			bfq_bfqq_end_wr(bfqq);  in bfq_update_io_seektime()
5837 			switch_back_to_interactive_wr(bfqq, bfqd);  in bfq_update_io_seektime()
5838 			bfqq->entity.prio_changed = 1;  in bfq_update_io_seektime()
5844 				       struct bfq_queue *bfqq,  in bfq_update_has_short_ttime()  argument
5854 	if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) ||  in bfq_update_has_short_ttime()
5859 	if (time_is_after_eq_jiffies(bfqq->split_time +  in bfq_update_has_short_ttime()
5869 	    (bfq_sample_valid(bfqq->ttime.ttime_samples) &&  in bfq_update_has_short_ttime()
5870 	     bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle>>1))  in bfq_update_has_short_ttime()
5873 	state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq);  in bfq_update_has_short_ttime()
5876 		bfq_mark_bfqq_has_short_ttime(bfqq);  in bfq_update_has_short_ttime()
5878 		bfq_clear_bfqq_has_short_ttime(bfqq);  in bfq_update_has_short_ttime()
5964 	if (state_changed && bfqq->last_serv_time_ns == 0 &&  in bfq_update_has_short_ttime()
5965 	    (time_is_before_eq_jiffies(bfqq->decrease_time_jif +  in bfq_update_has_short_ttime()
5968 		bfq_reset_inject_limit(bfqd, bfqq);  in bfq_update_has_short_ttime()
5975 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_rq_enqueued()  argument
5979 		bfqq->meta_pending++;  in bfq_rq_enqueued()
5981 	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);  in bfq_rq_enqueued()
5983 	if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) {  in bfq_rq_enqueued()
5984 		bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 &&  in bfq_rq_enqueued()
5986 		bool budget_timeout = bfq_bfqq_budget_timeout(bfqq);  in bfq_rq_enqueued()
6004 		if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) &&  in bfq_rq_enqueued()
6015 		bfq_clear_bfqq_wait_request(bfqq);  in bfq_rq_enqueued()
6026 			bfq_bfqq_expire(bfqd, bfqq, false,  in bfq_rq_enqueued()
6031 static void bfqq_request_allocated(struct bfq_queue *bfqq)  in bfqq_request_allocated()  argument
6033 	struct bfq_entity *entity = &bfqq->entity;  in bfqq_request_allocated()
6039 static void bfqq_request_freed(struct bfq_queue *bfqq)  in bfqq_request_freed()  argument
6041 	struct bfq_entity *entity = &bfqq->entity;  in bfqq_request_freed()
6050 	struct bfq_queue *bfqq = RQ_BFQQ(rq),  in __bfq_insert_request()  local
6051 		*new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true,  in __bfq_insert_request()
6061 		bfqq_request_freed(bfqq);  in __bfq_insert_request()
6071 		if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq)  in __bfq_insert_request()
6073 					bfqq, new_bfqq);  in __bfq_insert_request()
6075 		bfq_clear_bfqq_just_created(bfqq);  in __bfq_insert_request()
6080 		bfq_put_queue(bfqq);  in __bfq_insert_request()
6082 		bfqq = new_bfqq;  in __bfq_insert_request()
6085 	bfq_update_io_thinktime(bfqd, bfqq);  in __bfq_insert_request()
6086 	bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq));  in __bfq_insert_request()
6087 	bfq_update_io_seektime(bfqd, bfqq, rq);  in __bfq_insert_request()
6089 	waiting = bfqq && bfq_bfqq_wait_request(bfqq);  in __bfq_insert_request()
6091 	idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq);  in __bfq_insert_request()
6094 	list_add_tail(&rq->queuelist, &bfqq->fifo);  in __bfq_insert_request()
6096 	bfq_rq_enqueued(bfqd, bfqq, rq);  in __bfq_insert_request()
6103 				    struct bfq_queue *bfqq,  in bfq_update_insert_stats()  argument
6107 	if (!bfqq)  in bfq_update_insert_stats()
6121 	bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags);  in bfq_update_insert_stats()
6123 		bfqg_stats_update_idle_time(bfqq_group(bfqq));  in bfq_update_insert_stats()
6128 					   struct bfq_queue *bfqq,  in bfq_update_insert_stats()  argument
6140 	struct bfq_queue *bfqq;  in bfq_insert_request()  local
6150 	bfqq = bfq_init_rq(rq);  in bfq_insert_request()
6159 	if (!bfqq || at_head) {  in bfq_insert_request()
6171 		bfqq = RQ_BFQQ(rq);  in bfq_insert_request()
6188 	bfq_update_insert_stats(q, bfqq, idle_timer_disabled,  in bfq_insert_request()
6206 	struct bfq_queue *bfqq = bfqd->in_service_queue;  in bfq_update_hw_tag()  local
6228 	if (bfqq && bfq_bfqq_has_short_ttime(bfqq) &&  in bfq_update_hw_tag()
6229 	    bfqq->dispatched + bfqq->queued[0] + bfqq->queued[1] <  in bfq_update_hw_tag()
6245 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)  in bfq_completed_request()  argument
6253 	bfqq->dispatched--;  in bfq_completed_request()
6255 	if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) {  in bfq_completed_request()
6262 		bfqq->budget_timeout = jiffies;  in bfq_completed_request()
6264 		bfq_weights_tree_remove(bfqd, bfqq);  in bfq_completed_request()
6269 	bfqq->ttime.last_end_request = now_ns;  in bfq_completed_request()
6307 	if (!bfq_bfqq_coop(bfqq))  in bfq_completed_request()
6308 		bfqd->last_completed_rq_bfqq = bfqq;  in bfq_completed_request()
6323 	if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 &&  in bfq_completed_request()
6324 	    RB_EMPTY_ROOT(&bfqq->sort_list) &&  in bfq_completed_request()
6325 	    bfqq->wr_coeff != bfqd->bfq_wr_coeff)  in bfq_completed_request()
6326 		bfqq->soft_rt_next_start =  in bfq_completed_request()
6327 			bfq_bfqq_softrt_next_start(bfqd, bfqq);  in bfq_completed_request()
6333 	if (bfqd->in_service_queue == bfqq) {  in bfq_completed_request()
6334 		if (bfq_bfqq_must_idle(bfqq)) {  in bfq_completed_request()
6335 			if (bfqq->dispatched == 0)  in bfq_completed_request()
6361 		} else if (bfq_may_expire_for_budg_timeout(bfqq))  in bfq_completed_request()
6362 			bfq_bfqq_expire(bfqd, bfqq, false,  in bfq_completed_request()
6364 		else if (RB_EMPTY_ROOT(&bfqq->sort_list) &&  in bfq_completed_request()
6365 			 (bfqq->dispatched == 0 ||  in bfq_completed_request()
6366 			  !bfq_better_to_idle(bfqq)))  in bfq_completed_request()
6367 			bfq_bfqq_expire(bfqd, bfqq, false,  in bfq_completed_request()
6480 				    struct bfq_queue *bfqq)  in bfq_update_inject_limit()  argument
6483 	unsigned int old_limit = bfqq->inject_limit;  in bfq_update_inject_limit()
6485 	if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) {  in bfq_update_inject_limit()
6486 		u64 threshold = (bfqq->last_serv_time_ns * 3)>>1;  in bfq_update_inject_limit()
6489 			bfqq->inject_limit--;  in bfq_update_inject_limit()
6490 			bfqq->decrease_time_jif = jiffies;  in bfq_update_inject_limit()
6493 			bfqq->inject_limit++;  in bfq_update_inject_limit()
6508 	if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) ||  in bfq_update_inject_limit()
6509 	    tot_time_ns < bfqq->last_serv_time_ns) {  in bfq_update_inject_limit()
6510 		if (bfqq->last_serv_time_ns == 0) {  in bfq_update_inject_limit()
6515 			bfqq->inject_limit = max_t(unsigned int, 1, old_limit);  in bfq_update_inject_limit()
6517 		bfqq->last_serv_time_ns = tot_time_ns;  in bfq_update_inject_limit()
6528 		bfqq->last_serv_time_ns = tot_time_ns;  in bfq_update_inject_limit()
6544 	struct bfq_queue *bfqq = RQ_BFQQ(rq);  in bfq_finish_requeue_request()  local
6553 	if (!rq->elv.icq || !bfqq)  in bfq_finish_requeue_request()
6556 	bfqd = bfqq->bfqd;  in bfq_finish_requeue_request()
6559 		bfqg_stats_update_completion(bfqq_group(bfqq),  in bfq_finish_requeue_request()
6567 			bfq_update_inject_limit(bfqd, bfqq);  in bfq_finish_requeue_request()
6569 		bfq_completed_request(bfqq, bfqd);  in bfq_finish_requeue_request()
6571 	bfqq_request_freed(bfqq);  in bfq_finish_requeue_request()
6572 	bfq_put_queue(bfqq);  in bfq_finish_requeue_request()
6614 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)  in bfq_split_bfqq()  argument
6616 	bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue");  in bfq_split_bfqq()
6618 	if (bfqq_process_refs(bfqq) == 1) {  in bfq_split_bfqq()
6619 		bfqq->pid = current->pid;  in bfq_split_bfqq()
6620 		bfq_clear_bfqq_coop(bfqq);  in bfq_split_bfqq()
6621 		bfq_clear_bfqq_split_coop(bfqq);  in bfq_split_bfqq()
6622 		return bfqq;  in bfq_split_bfqq()
6627 	bfq_put_cooperator(bfqq);  in bfq_split_bfqq()
6629 	bfq_release_process_ref(bfqq->bfqd, bfqq);  in bfq_split_bfqq()
6639 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_get_bfqq_handle_split()  local
6641 	if (likely(bfqq && bfqq != &bfqd->oom_bfqq))  in bfq_get_bfqq_handle_split()
6642 		return bfqq;  in bfq_get_bfqq_handle_split()
6647 	if (bfqq)  in bfq_get_bfqq_handle_split()
6648 		bfq_put_queue(bfqq);  in bfq_get_bfqq_handle_split()
6649 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split);  in bfq_get_bfqq_handle_split()
6651 	bic_set_bfqq(bic, bfqq, is_sync);  in bfq_get_bfqq_handle_split()
6655 			bfq_mark_bfqq_in_large_burst(bfqq);  in bfq_get_bfqq_handle_split()
6657 			bfq_clear_bfqq_in_large_burst(bfqq);  in bfq_get_bfqq_handle_split()
6687 				hlist_add_head(&bfqq->burst_list_node,  in bfq_get_bfqq_handle_split()
6690 		bfqq->split_time = jiffies;  in bfq_get_bfqq_handle_split()
6693 	return bfqq;  in bfq_get_bfqq_handle_split()
6744 	struct bfq_queue *bfqq;  in bfq_init_rq()  local
6767 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,  in bfq_init_rq()
6772 		if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) &&  in bfq_init_rq()
6774 			struct bfq_queue *old_bfqq = bfqq;  in bfq_init_rq()
6777 			if (bfq_bfqq_in_large_burst(bfqq))  in bfq_init_rq()
6780 			bfqq = bfq_split_bfqq(bic, bfqq);  in bfq_init_rq()
6783 			if (!bfqq) {  in bfq_init_rq()
6784 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,  in bfq_init_rq()
6787 				bfqq->waker_bfqq = old_bfqq->waker_bfqq;  in bfq_init_rq()
6788 				bfqq->tentative_waker_bfqq = NULL;  in bfq_init_rq()
6797 				if (bfqq->waker_bfqq)  in bfq_init_rq()
6798 					hlist_add_head(&bfqq->woken_list_node,  in bfq_init_rq()
6799 						       &bfqq->waker_bfqq->woken_list);  in bfq_init_rq()
6805 	bfqq_request_allocated(bfqq);  in bfq_init_rq()
6806 	bfqq->ref++;  in bfq_init_rq()
6808 	bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d",  in bfq_init_rq()
6809 		     rq, bfqq, bfqq->ref);  in bfq_init_rq()
6812 	rq->elv.priv[1] = bfqq;  in bfq_init_rq()
6820 	if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {  in bfq_init_rq()
6821 		bfqq->bic = bic;  in bfq_init_rq()
6828 			bfq_bfqq_resume_state(bfqq, bfqd, bic,  in bfq_init_rq()
6853 	if (unlikely(bfq_bfqq_just_created(bfqq) &&  in bfq_init_rq()
6856 		bfq_handle_burst(bfqd, bfqq);  in bfq_init_rq()
6858 	return bfqq;  in bfq_init_rq()
6862 bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq)  in bfq_idle_slice_timer_body()  argument
6876 	if (bfqq != bfqd->in_service_queue) {  in bfq_idle_slice_timer_body()
6881 	bfq_clear_bfqq_wait_request(bfqq);  in bfq_idle_slice_timer_body()
6883 	if (bfq_bfqq_budget_timeout(bfqq))  in bfq_idle_slice_timer_body()
6890 	else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0)  in bfq_idle_slice_timer_body()
6901 	bfq_bfqq_expire(bfqd, bfqq, true, reason);  in bfq_idle_slice_timer_body()
6916 	struct bfq_queue *bfqq = bfqd->in_service_queue;  in bfq_idle_slice_timer()  local
6926 	if (bfqq)  in bfq_idle_slice_timer()
6927 		bfq_idle_slice_timer_body(bfqd, bfqq);  in bfq_idle_slice_timer()
6935 	struct bfq_queue *bfqq = *bfqq_ptr;  in __bfq_put_async_bfqq()  local
6937 	bfq_log(bfqd, "put_async_bfqq: %p", bfqq);  in __bfq_put_async_bfqq()
6938 	if (bfqq) {  in __bfq_put_async_bfqq()
6939 		bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);  in __bfq_put_async_bfqq()
6941 		bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d",  in __bfq_put_async_bfqq()
6942 			     bfqq, bfqq->ref);  in __bfq_put_async_bfqq()
6943 		bfq_put_queue(bfqq);  in __bfq_put_async_bfqq()
7024 	struct bfq_queue *bfqq, *n;  in bfq_exit_queue()  local
7029 	list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list)  in bfq_exit_queue()
7030 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);  in bfq_exit_queue()