380 struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync)  in bic_to_bfqq()  argument
382 	return bic->bfqq[is_sync];  in bic_to_bfqq()
387 void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync)  in bic_set_bfqq()  argument
402 	bic->bfqq[is_sync] = bfqq;  in bic_set_bfqq()
404 	if (bfqq && bic->stable_merge_bfqq == bfqq) {  in bic_set_bfqq()
413 		bfq_put_stable_ref(bic->stable_merge_bfqq);  in bic_set_bfqq()
415 		bic->stable_merge_bfqq = NULL;  in bic_set_bfqq()
419 struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)  in bic_to_bfqd()  argument
421 	return bic->icq.q->elevator->elevator_data;  in bic_to_bfqd()
1052 		      struct bfq_io_cq *bic, bool bfq_already_existing)  in bfq_bfqq_resume_state()  argument
1057 	if (bic->saved_has_short_ttime)  in bfq_bfqq_resume_state()
1062 	if (bic->saved_IO_bound)  in bfq_bfqq_resume_state()
1067 	bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns;  in bfq_bfqq_resume_state()
1068 	bfqq->inject_limit = bic->saved_inject_limit;  in bfq_bfqq_resume_state()
1069 	bfqq->decrease_time_jif = bic->saved_decrease_time_jif;  in bfq_bfqq_resume_state()
1071 	bfqq->entity.new_weight = bic->saved_weight;  in bfq_bfqq_resume_state()
1072 	bfqq->ttime = bic->saved_ttime;  in bfq_bfqq_resume_state()
1073 	bfqq->io_start_time = bic->saved_io_start_time;  in bfq_bfqq_resume_state()
1074 	bfqq->tot_idle_time = bic->saved_tot_idle_time;  in bfq_bfqq_resume_state()
1080 		bfqq->wr_coeff = bic->saved_wr_coeff;  in bfq_bfqq_resume_state()
1082 	bfqq->service_from_wr = bic->saved_service_from_wr;  in bfq_bfqq_resume_state()
1083 	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;  in bfq_bfqq_resume_state()
1084 	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;  in bfq_bfqq_resume_state()
1085 	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;  in bfq_bfqq_resume_state()
1757 		  (bfqq->bic || RQ_BIC(rq)->stably_merged) &&  in bfq_bfqq_handle_idle_busy_switch()
2335 	struct bfq_io_cq *bic = bfq_bic_lookup(bfqd, current->io_context, q);  in bfq_bio_merge()  local
2340 	if (bic)  in bfq_bio_merge()
2341 		bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf));  in bfq_bio_merge()
2344 	bfqd->bio_bic = bic;  in bfq_bio_merge()
2723 		     void *io_struct, bool request, struct bfq_io_cq *bic)  in bfq_setup_cooperator()  argument
2743 		if (bfq_bfqq_sync(bfqq) && bic->stable_merge_bfqq &&  in bfq_setup_cooperator()
2750 				bic->stable_merge_bfqq;  in bfq_setup_cooperator()
2757 			bic->stable_merge_bfqq = NULL;  in bfq_setup_cooperator()
2765 				bic->stably_merged = true;  in bfq_setup_cooperator()
2766 				if (new_bfqq && new_bfqq->bic)  in bfq_setup_cooperator()
2767 					new_bfqq->bic->stably_merged = true;  in bfq_setup_cooperator()
2867 	struct bfq_io_cq *bic = bfqq->bic;  in bfq_bfqq_save_state()  local
2874 	if (!bic)  in bfq_bfqq_save_state()
2877 	bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns;  in bfq_bfqq_save_state()
2878 	bic->saved_inject_limit = bfqq->inject_limit;  in bfq_bfqq_save_state()
2879 	bic->saved_decrease_time_jif = bfqq->decrease_time_jif;  in bfq_bfqq_save_state()
2881 	bic->saved_weight = bfqq->entity.orig_weight;  in bfq_bfqq_save_state()
2882 	bic->saved_ttime = bfqq->ttime;  in bfq_bfqq_save_state()
2883 	bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);  in bfq_bfqq_save_state()
2884 	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);  in bfq_bfqq_save_state()
2885 	bic->saved_io_start_time = bfqq->io_start_time;  in bfq_bfqq_save_state()
2886 	bic->saved_tot_idle_time = bfqq->tot_idle_time;  in bfq_bfqq_save_state()
2887 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);  in bfq_bfqq_save_state()
2888 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);  in bfq_bfqq_save_state()
2901 		bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff;  in bfq_bfqq_save_state()
2902 		bic->saved_wr_start_at_switch_to_srt = bfq_smallest_from_now();  in bfq_bfqq_save_state()
2903 		bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd);  in bfq_bfqq_save_state()
2904 		bic->saved_last_wr_start_finish = jiffies;  in bfq_bfqq_save_state()
2906 		bic->saved_wr_coeff = bfqq->wr_coeff;  in bfq_bfqq_save_state()
2907 		bic->saved_wr_start_at_switch_to_srt =  in bfq_bfqq_save_state()
2909 		bic->saved_service_from_wr = bfqq->service_from_wr;  in bfq_bfqq_save_state()
2910 		bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;  in bfq_bfqq_save_state()
2911 		bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;  in bfq_bfqq_save_state()
2949 bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,  in bfq_merge_bfqqs()  argument
3017 	bic_set_bfqq(bic, new_bfqq, 1);  in bfq_merge_bfqqs()
3029 	new_bfqq->bic = NULL;  in bfq_merge_bfqqs()
3040 	bfqq->bic = NULL;  in bfq_merge_bfqqs()
4652 			bfqq->bic && bfqq->bic->bfqq[0] &&  in bfq_select_queue()
4653 			bfq_bfqq_busy(bfqq->bic->bfqq[0]) &&  in bfq_select_queue()
4654 			bfqq->bic->bfqq[0]->next_rq ?  in bfq_select_queue()
4655 			bfqq->bic->bfqq[0] : NULL;  in bfq_select_queue()
4744 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&  in bfq_select_queue()
4747 			bfqq = bfqq->bic->bfqq[0];  in bfq_select_queue()
5133 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)  in bfq_put_queue()
5209 static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync)  in bfq_exit_icq_bfqq()  argument
5211 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_exit_icq_bfqq()
5221 		bfqq->bic = NULL;  in bfq_exit_icq_bfqq()
5223 		bic_set_bfqq(bic, NULL, is_sync);  in bfq_exit_icq_bfqq()
5230 	struct bfq_io_cq *bic = icq_to_bic(icq);  in bfq_exit_icq()  local
5232 	if (bic->stable_merge_bfqq) {  in bfq_exit_icq()
5233 		struct bfq_data *bfqd = bic->stable_merge_bfqq->bfqd;  in bfq_exit_icq()
5243 			bfq_put_stable_ref(bic->stable_merge_bfqq);  in bfq_exit_icq()
5246 			bfq_put_stable_ref(bic->stable_merge_bfqq);  in bfq_exit_icq()
5250 	bfq_exit_icq_bfqq(bic, true);  in bfq_exit_icq()
5251 	bfq_exit_icq_bfqq(bic, false);  in bfq_exit_icq()
5259 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)  in bfq_set_next_ioprio_data()  argument
5268 	ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_set_next_ioprio_data()
5283 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5287 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5310 				       struct bfq_io_cq *bic,
5313 static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio)  in bfq_check_ioprio_change()  argument
5315 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_check_ioprio_change()
5317 	int ioprio = bic->icq.ioc->ioprio;  in bfq_check_ioprio_change()
5323 	if (unlikely(!bfqd) || likely(bic->ioprio == ioprio))  in bfq_check_ioprio_change()
5326 	bic->ioprio = ioprio;  in bfq_check_ioprio_change()
5328 	bfqq = bic_to_bfqq(bic, false);  in bfq_check_ioprio_change()
5331 		bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic, true);  in bfq_check_ioprio_change()
5332 		bic_set_bfqq(bic, bfqq, false);  in bfq_check_ioprio_change()
5335 	bfqq = bic_to_bfqq(bic, true);  in bfq_check_ioprio_change()
5337 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_check_ioprio_change()
5341 			  struct bfq_io_cq *bic, pid_t pid, int is_sync)  in bfq_init_bfqq()  argument
5354 	if (bic)  in bfq_init_bfqq()
5355 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_init_bfqq()
5427 			  struct bfq_io_cq *bic,  in bfq_do_early_stable_merge()  argument
5436 	if (new_bfqq->bic)  in bfq_do_early_stable_merge()
5437 		new_bfqq->bic->stably_merged = true;  in bfq_do_early_stable_merge()
5438 	bic->stably_merged = true;  in bfq_do_early_stable_merge()
5446 	bfqq->bic = bic;  in bfq_do_early_stable_merge()
5447 	bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq);  in bfq_do_early_stable_merge()
5500 						      struct bfq_io_cq *bic)  in bfq_do_or_sched_stable_merge()  argument
5546 							 bic,  in bfq_do_or_sched_stable_merge()
5563 			bic->stable_merge_bfqq = last_bfqq_created;  in bfq_do_or_sched_stable_merge()
5573 				       struct bfq_io_cq *bic,  in bfq_get_queue()  argument
5576 	const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_get_queue()
5577 	const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_get_queue()
5603 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,  in bfq_get_queue()
5634 		bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic);  in bfq_get_queue()
5695 				       struct bfq_io_cq *bic)  in bfq_update_has_short_ttime()  argument
5718 	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||  in bfq_update_has_short_ttime()
6486 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)  in bfq_split_bfqq()  argument
6497 	bic_set_bfqq(bic, NULL, 1);  in bfq_split_bfqq()
6506 						   struct bfq_io_cq *bic,  in bfq_get_bfqq_handle_split()  argument
6511 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_get_bfqq_handle_split()
6521 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split);  in bfq_get_bfqq_handle_split()
6523 	bic_set_bfqq(bic, bfqq, is_sync);  in bfq_get_bfqq_handle_split()
6525 		if ((bic->was_in_burst_list && bfqd->large_burst) ||  in bfq_get_bfqq_handle_split()
6526 		    bic->saved_in_large_burst)  in bfq_get_bfqq_handle_split()
6530 			if (bic->was_in_burst_list)  in bfq_get_bfqq_handle_split()
6612 	struct bfq_io_cq *bic;  in bfq_init_rq()  local
6631 	bic = icq_to_bic(rq->elv.icq);  in bfq_init_rq()
6633 	bfq_check_ioprio_change(bic, bio);  in bfq_init_rq()
6635 	bfq_bic_update_cgroup(bic, bio);  in bfq_init_rq()
6637 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,  in bfq_init_rq()
6643 			!bic->stably_merged) {  in bfq_init_rq()
6648 				bic->saved_in_large_burst = true;  in bfq_init_rq()
6650 			bfqq = bfq_split_bfqq(bic, bfqq);  in bfq_init_rq()
6654 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,  in bfq_init_rq()
6680 	rq->elv.priv[0] = bic;  in bfq_init_rq()
6690 		bfqq->bic = bic;  in bfq_init_rq()
6697 			bfq_bfqq_resume_state(bfqq, bfqd, bic,  in bfq_init_rq()