380 struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync)  in bic_to_bfqq()  argument
382 	return bic->bfqq[is_sync];  in bic_to_bfqq()
387 void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync)  in bic_set_bfqq()  argument
402 	bic->bfqq[is_sync] = bfqq;  in bic_set_bfqq()
404 	if (bfqq && bic->stable_merge_bfqq == bfqq) {  in bic_set_bfqq()
413 		bfq_put_stable_ref(bic->stable_merge_bfqq);  in bic_set_bfqq()
415 		bic->stable_merge_bfqq = NULL;  in bic_set_bfqq()
419 struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)  in bic_to_bfqd()  argument
421 	return bic->icq.q->elevator->elevator_data;  in bic_to_bfqd()
674 	struct bfq_io_cq *bic = bfq_bic_lookup(data->q);  in bfq_limit_depth()  local
675 	struct bfq_queue *bfqq = bic ? bic_to_bfqq(bic, op_is_sync(opf)) : NULL;  in bfq_limit_depth()
1163 		      struct bfq_io_cq *bic, bool bfq_already_existing)  in bfq_bfqq_resume_state()  argument
1168 	if (bic->saved_has_short_ttime)  in bfq_bfqq_resume_state()
1173 	if (bic->saved_IO_bound)  in bfq_bfqq_resume_state()
1178 	bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns;  in bfq_bfqq_resume_state()
1179 	bfqq->inject_limit = bic->saved_inject_limit;  in bfq_bfqq_resume_state()
1180 	bfqq->decrease_time_jif = bic->saved_decrease_time_jif;  in bfq_bfqq_resume_state()
1182 	bfqq->entity.new_weight = bic->saved_weight;  in bfq_bfqq_resume_state()
1183 	bfqq->ttime = bic->saved_ttime;  in bfq_bfqq_resume_state()
1184 	bfqq->io_start_time = bic->saved_io_start_time;  in bfq_bfqq_resume_state()
1185 	bfqq->tot_idle_time = bic->saved_tot_idle_time;  in bfq_bfqq_resume_state()
1191 		bfqq->wr_coeff = bic->saved_wr_coeff;  in bfq_bfqq_resume_state()
1193 	bfqq->service_from_wr = bic->saved_service_from_wr;  in bfq_bfqq_resume_state()
1194 	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;  in bfq_bfqq_resume_state()
1195 	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;  in bfq_bfqq_resume_state()
1196 	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;  in bfq_bfqq_resume_state()
1869 		  (bfqq->bic || RQ_BIC(rq)->stably_merged) &&  in bfq_bfqq_handle_idle_busy_switch()
2469 	struct bfq_io_cq *bic = bfq_bic_lookup(q);  in bfq_bio_merge()  local
2474 	if (bic) {  in bfq_bio_merge()
2479 		bfq_bic_update_cgroup(bic, bio);  in bfq_bio_merge()
2481 		bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf));  in bfq_bio_merge()
2485 	bfqd->bio_bic = bic;  in bfq_bio_merge()
2879 		     void *io_struct, bool request, struct bfq_io_cq *bic)  in bfq_setup_cooperator()  argument
2903 		if (bfq_bfqq_sync(bfqq) && bic->stable_merge_bfqq &&  in bfq_setup_cooperator()
2910 				bic->stable_merge_bfqq;  in bfq_setup_cooperator()
2917 			bic->stable_merge_bfqq = NULL;  in bfq_setup_cooperator()
2926 					bic->stably_merged = true;  in bfq_setup_cooperator()
2927 					if (new_bfqq->bic)  in bfq_setup_cooperator()
2928 						new_bfqq->bic->stably_merged =  in bfq_setup_cooperator()
3027 	struct bfq_io_cq *bic = bfqq->bic;  in bfq_bfqq_save_state()  local
3034 	if (!bic)  in bfq_bfqq_save_state()
3037 	bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns;  in bfq_bfqq_save_state()
3038 	bic->saved_inject_limit = bfqq->inject_limit;  in bfq_bfqq_save_state()
3039 	bic->saved_decrease_time_jif = bfqq->decrease_time_jif;  in bfq_bfqq_save_state()
3041 	bic->saved_weight = bfqq->entity.orig_weight;  in bfq_bfqq_save_state()
3042 	bic->saved_ttime = bfqq->ttime;  in bfq_bfqq_save_state()
3043 	bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);  in bfq_bfqq_save_state()
3044 	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);  in bfq_bfqq_save_state()
3045 	bic->saved_io_start_time = bfqq->io_start_time;  in bfq_bfqq_save_state()
3046 	bic->saved_tot_idle_time = bfqq->tot_idle_time;  in bfq_bfqq_save_state()
3047 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);  in bfq_bfqq_save_state()
3048 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);  in bfq_bfqq_save_state()
3061 		bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff;  in bfq_bfqq_save_state()
3062 		bic->saved_wr_start_at_switch_to_srt = bfq_smallest_from_now();  in bfq_bfqq_save_state()
3063 		bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd);  in bfq_bfqq_save_state()
3064 		bic->saved_last_wr_start_finish = jiffies;  in bfq_bfqq_save_state()
3066 		bic->saved_wr_coeff = bfqq->wr_coeff;  in bfq_bfqq_save_state()
3067 		bic->saved_wr_start_at_switch_to_srt =  in bfq_bfqq_save_state()
3069 		bic->saved_service_from_wr = bfqq->service_from_wr;  in bfq_bfqq_save_state()
3070 		bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;  in bfq_bfqq_save_state()
3071 		bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;  in bfq_bfqq_save_state()
3109 bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,  in bfq_merge_bfqqs()  argument
3177 	bic_set_bfqq(bic, new_bfqq, 1);  in bfq_merge_bfqqs()
3189 	new_bfqq->bic = NULL;  in bfq_merge_bfqqs()
3200 	bfqq->bic = NULL;  in bfq_merge_bfqqs()
4812 			bfqq->bic && bfqq->bic->bfqq[0] &&  in bfq_select_queue()
4813 			bfq_bfqq_busy(bfqq->bic->bfqq[0]) &&  in bfq_select_queue()
4814 			bfqq->bic->bfqq[0]->next_rq ?  in bfq_select_queue()
4815 			bfqq->bic->bfqq[0] : NULL;  in bfq_select_queue()
4904 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&  in bfq_select_queue()
4907 			bfqq = bfqq->bic->bfqq[0];  in bfq_select_queue()
5292 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)  in bfq_put_queue()
5368 static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync)  in bfq_exit_icq_bfqq()  argument
5370 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_exit_icq_bfqq()
5380 		bfqq->bic = NULL;  in bfq_exit_icq_bfqq()
5382 		bic_set_bfqq(bic, NULL, is_sync);  in bfq_exit_icq_bfqq()
5389 	struct bfq_io_cq *bic = icq_to_bic(icq);  in bfq_exit_icq()  local
5391 	if (bic->stable_merge_bfqq) {  in bfq_exit_icq()
5392 		struct bfq_data *bfqd = bic->stable_merge_bfqq->bfqd;  in bfq_exit_icq()
5402 			bfq_put_stable_ref(bic->stable_merge_bfqq);  in bfq_exit_icq()
5405 			bfq_put_stable_ref(bic->stable_merge_bfqq);  in bfq_exit_icq()
5409 	bfq_exit_icq_bfqq(bic, true);  in bfq_exit_icq()
5410 	bfq_exit_icq_bfqq(bic, false);  in bfq_exit_icq()
5418 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)  in bfq_set_next_ioprio_data()  argument
5427 	ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_set_next_ioprio_data()
5442 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5446 		bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_set_next_ioprio_data()
5469 				       struct bfq_io_cq *bic,
5472 static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio)  in bfq_check_ioprio_change()  argument
5474 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_check_ioprio_change()
5476 	int ioprio = bic->icq.ioc->ioprio;  in bfq_check_ioprio_change()
5482 	if (unlikely(!bfqd) || likely(bic->ioprio == ioprio))  in bfq_check_ioprio_change()
5485 	bic->ioprio = ioprio;  in bfq_check_ioprio_change()
5487 	bfqq = bic_to_bfqq(bic, false);  in bfq_check_ioprio_change()
5490 		bfqq = bfq_get_queue(bfqd, bio, false, bic, true);  in bfq_check_ioprio_change()
5491 		bic_set_bfqq(bic, bfqq, false);  in bfq_check_ioprio_change()
5494 	bfqq = bic_to_bfqq(bic, true);  in bfq_check_ioprio_change()
5496 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_check_ioprio_change()
5500 			  struct bfq_io_cq *bic, pid_t pid, int is_sync)  in bfq_init_bfqq()  argument
5513 	if (bic)  in bfq_init_bfqq()
5514 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_init_bfqq()
5586 			  struct bfq_io_cq *bic,  in bfq_do_early_stable_merge()  argument
5595 	if (new_bfqq->bic)  in bfq_do_early_stable_merge()
5596 		new_bfqq->bic->stably_merged = true;  in bfq_do_early_stable_merge()
5597 	bic->stably_merged = true;  in bfq_do_early_stable_merge()
5605 	bfqq->bic = bic;  in bfq_do_early_stable_merge()
5606 	bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq);  in bfq_do_early_stable_merge()
5659 						      struct bfq_io_cq *bic)  in bfq_do_or_sched_stable_merge()  argument
5705 							 bic,  in bfq_do_or_sched_stable_merge()
5722 			bic->stable_merge_bfqq = last_bfqq_created;  in bfq_do_or_sched_stable_merge()
5732 				       struct bfq_io_cq *bic,  in bfq_get_queue()  argument
5735 	const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio);  in bfq_get_queue()
5736 	const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_get_queue()
5755 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,  in bfq_get_queue()
5786 		bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic);  in bfq_get_queue()
5845 				       struct bfq_io_cq *bic)  in bfq_update_has_short_ttime()  argument
5868 	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||  in bfq_update_has_short_ttime()
6614 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)  in bfq_split_bfqq()  argument
6625 	bic_set_bfqq(bic, NULL, 1);  in bfq_split_bfqq()
6634 						   struct bfq_io_cq *bic,  in bfq_get_bfqq_handle_split()  argument
6639 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync);  in bfq_get_bfqq_handle_split()
6649 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split);  in bfq_get_bfqq_handle_split()
6651 	bic_set_bfqq(bic, bfqq, is_sync);  in bfq_get_bfqq_handle_split()
6653 		if ((bic->was_in_burst_list && bfqd->large_burst) ||  in bfq_get_bfqq_handle_split()
6654 		    bic->saved_in_large_burst)  in bfq_get_bfqq_handle_split()
6658 			if (bic->was_in_burst_list)  in bfq_get_bfqq_handle_split()
6742 	struct bfq_io_cq *bic;  in bfq_init_rq()  local
6761 	bic = icq_to_bic(rq->elv.icq);  in bfq_init_rq()
6763 	bfq_check_ioprio_change(bic, bio);  in bfq_init_rq()
6765 	bfq_bic_update_cgroup(bic, bio);  in bfq_init_rq()
6767 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,  in bfq_init_rq()
6773 			!bic->stably_merged) {  in bfq_init_rq()
6778 				bic->saved_in_large_burst = true;  in bfq_init_rq()
6780 			bfqq = bfq_split_bfqq(bic, bfqq);  in bfq_init_rq()
6784 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,  in bfq_init_rq()
6807 	bic->requests++;  in bfq_init_rq()
6811 	rq->elv.priv[0] = bic;  in bfq_init_rq()
6821 		bfqq->bic = bic;  in bfq_init_rq()
6828 			bfq_bfqq_resume_state(bfqq, bfqd, bic,  in bfq_init_rq()