889 				       struct cfq_io_cq *cic, struct bio *bio);
905 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()  argument
907 	return cic->cfqq[is_sync];  in cic_to_cfqq()
910 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()  argument
913 	cic->cfqq[is_sync] = cfqq;  in cic_set_cfqq()
916 static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)  in cic_to_cfqd()  argument
918 	return cic->icq.q->elevator->elevator_data;  in cic_to_cfqd()
2462 	struct cfq_io_cq *cic;  in cfq_find_rq_fmerge()  local
2465 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_find_rq_fmerge()
2466 	if (!cic)  in cfq_find_rq_fmerge()
2469 	cfqq = cic_to_cfqq(cic, op_is_sync(bio->bi_opf));  in cfq_find_rq_fmerge()
2584 	struct cfq_io_cq *cic;  in cfq_allow_bio_merge()  local
2597 	cic = cfq_cic_lookup(cfqd, current->io_context);  in cfq_allow_bio_merge()
2598 	if (!cic)  in cfq_allow_bio_merge()
2601 	cfqq = cic_to_cfqq(cic, is_sync);  in cfq_allow_bio_merge()
2910 	struct cfq_io_cq *cic;  in cfq_arm_slice_timer()  local
2946 	cic = cfqd->active_cic;  in cfq_arm_slice_timer()
2947 	if (!cic || !atomic_read(&cic->icq.ioc->active_ref))  in cfq_arm_slice_timer()
2955 	if (sample_valid(cic->ttime.ttime_samples) &&  in cfq_arm_slice_timer()
2956 	    (cfqq->slice_end - now < cic->ttime.ttime_mean)) {  in cfq_arm_slice_timer()
2958 			     cic->ttime.ttime_mean);  in cfq_arm_slice_timer()
3516 		struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_dispatch_request()  local
3518 		atomic_long_inc(&cic->icq.ioc->refcount);  in cfq_dispatch_request()
3519 		cfqd->active_cic = cic;  in cfq_dispatch_request()
3636 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_init_icq()  local
3638 	cic->ttime.last_end_request = ktime_get_ns();  in cfq_init_icq()
3643 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_exit_icq()  local
3644 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in cfq_exit_icq()
3646 	if (cic_to_cfqq(cic, false)) {  in cfq_exit_icq()
3647 		cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, false));  in cfq_exit_icq()
3648 		cic_set_cfqq(cic, NULL, false);  in cfq_exit_icq()
3651 	if (cic_to_cfqq(cic, true)) {  in cfq_exit_icq()
3652 		cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, true));  in cfq_exit_icq()
3653 		cic_set_cfqq(cic, NULL, true);  in cfq_exit_icq()
3657 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()  argument
3665 	ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_init_prio_data()
3678 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3682 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3701 static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_ioprio_changed()  argument
3703 	int ioprio = cic->icq.ioc->ioprio;  in check_ioprio_changed()
3704 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_ioprio_changed()
3711 	if (unlikely(!cfqd) || likely(cic->ioprio == ioprio))  in check_ioprio_changed()
3714 	cfqq = cic_to_cfqq(cic, false);  in check_ioprio_changed()
3717 		cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio);  in check_ioprio_changed()
3718 		cic_set_cfqq(cic, cfqq, false);  in check_ioprio_changed()
3721 	cfqq = cic_to_cfqq(cic, true);  in check_ioprio_changed()
3725 	cic->ioprio = ioprio;  in check_ioprio_changed()
3749 static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_blkcg_changed()  argument
3751 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_blkcg_changed()
3763 	if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr))  in check_blkcg_changed()
3770 	cfqq = cic_to_cfqq(cic, false);  in check_blkcg_changed()
3773 		cic_set_cfqq(cic, NULL, false);  in check_blkcg_changed()
3777 	cfqq = cic_to_cfqq(cic, true);  in check_blkcg_changed()
3780 		cic_set_cfqq(cic, NULL, true);  in check_blkcg_changed()
3784 	cic->blkcg_serial_nr = serial_nr;  in check_blkcg_changed()
3787 static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_blkcg_changed()  argument
3811 cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,  in cfq_get_queue()  argument
3814 	int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_get_queue()
3815 	int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_get_queue()
3828 		if (!ioprio_valid(cic->ioprio)) {  in cfq_get_queue()
3850 	cfq_init_prio_data(cfqq, cic);  in cfq_get_queue()
3879 			struct cfq_io_cq *cic)  in cfq_update_io_thinktime()  argument
3882 		__cfq_update_io_thinktime(&cic->ttime, cfqd->cfq_slice_idle);  in cfq_update_io_thinktime()
3923 		       struct cfq_io_cq *cic)  in cfq_update_idle_window()  argument
3940 	else if (!atomic_read(&cic->icq.ioc->active_ref) ||  in cfq_update_idle_window()
3944 	else if (sample_valid(cic->ttime.ttime_samples)) {  in cfq_update_idle_window()
3945 		if (cic->ttime.ttime_mean > cfqd->cfq_slice_idle)  in cfq_update_idle_window()
4079 	struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_rq_enqueued()  local
4085 	cfq_update_io_thinktime(cfqd, cfqq, cic);  in cfq_rq_enqueued()
4087 	cfq_update_idle_window(cfqd, cfqq, cic);  in cfq_rq_enqueued()
4180 	struct cfq_io_cq *cic = cfqd->active_cic;  in cfq_should_wait_busy()  local
4199 	if (cic && sample_valid(cic->ttime.ttime_samples)  in cfq_should_wait_busy()
4200 	    && (cfqq->slice_end - now < cic->ttime.ttime_mean))  in cfq_should_wait_busy()
4334 	struct cfq_io_cq *cic;  in cfq_may_queue()  local
4343 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_may_queue()
4344 	if (!cic)  in cfq_may_queue()
4347 	cfqq = cic_to_cfqq(cic, op_is_sync(op));  in cfq_may_queue()
4349 		cfq_init_prio_data(cfqq, cic);  in cfq_may_queue()
4381 cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_cq *cic,  in cfq_merge_cfqqs()  argument
4385 	cic_set_cfqq(cic, cfqq->new_cfqq, 1);  in cfq_merge_cfqqs()
4388 	return cic_to_cfqq(cic, 1);  in cfq_merge_cfqqs()
4396 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()  argument
4405 	cic_set_cfqq(cic, NULL, 1);  in split_cfqq()
4420 	struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);  in cfq_set_request()  local
4427 	check_ioprio_changed(cic, bio);  in cfq_set_request()
4428 	check_blkcg_changed(cic, bio);  in cfq_set_request()
4430 	cfqq = cic_to_cfqq(cic, is_sync);  in cfq_set_request()
4434 		cfqq = cfq_get_queue(cfqd, is_sync, cic, bio);  in cfq_set_request()
4435 		cic_set_cfqq(cic, cfqq, is_sync);  in cfq_set_request()
4442 			cfqq = split_cfqq(cic, cfqq);  in cfq_set_request()
4454 			cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq);  in cfq_set_request()