160 	struct cfq_group *cfqg;  member
405 static struct cfq_rb_root *st_for(struct cfq_group *cfqg,  in st_for()  argument
409 	if (!cfqg)  in st_for()
413 		return &cfqg->service_tree_idle;  in st_for()
415 	return &cfqg->service_trees[class][type];  in st_for()
507 static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,  in cfqg_stats_set_start_group_wait_time()  argument
510 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_group_wait_time()
514 	if (cfqg == curr_cfqg)  in cfqg_stats_set_start_group_wait_time()
535 static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)  in cfqg_stats_update_dequeue()  argument
537 	blkg_stat_add(&cfqg->stats.dequeue, 1);  in cfqg_stats_update_dequeue()
540 static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_empty_time()  argument
542 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_empty_time()
559 static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)  in cfqg_stats_update_idle_time()  argument
561 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_idle_time()
573 static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_idle_time()  argument
575 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_idle_time()
583 static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)  in cfqg_stats_update_avg_queue_size()  argument
585 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_avg_queue_size()
595 static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *c…  in cfqg_stats_set_start_group_wait_time()  argument
597 static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }  in cfqg_stats_update_dequeue()  argument
598 static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_empty_time()  argument
599 static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_update_idle_time()  argument
600 static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_idle_time()  argument
601 static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }  in cfqg_stats_update_avg_queue_size()  argument
618 static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)  in cfqg_to_blkg()  argument
620 	return pd_to_blkg(&cfqg->pd);  in cfqg_to_blkg()
635 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)  in cfqg_parent()  argument
637 	struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;  in cfqg_parent()
642 static inline bool cfqg_is_descendant(struct cfq_group *cfqg,  in cfqg_is_descendant()  argument
645 	return cgroup_is_descendant(cfqg_to_blkg(cfqg)->blkcg->css.cgroup,  in cfqg_is_descendant()
649 static inline void cfqg_get(struct cfq_group *cfqg)  in cfqg_get()  argument
651 	return blkg_get(cfqg_to_blkg(cfqg));  in cfqg_get()
654 static inline void cfqg_put(struct cfq_group *cfqg)  in cfqg_put()  argument
656 	return blkg_put(cfqg_to_blkg(cfqg));  in cfqg_put()
661 			cfqg_to_blkg((cfqq)->cfqg)->blkcg,		\
668 #define cfq_log_cfqg(cfqd, cfqg, fmt, args...)	do {			\  argument
670 			cfqg_to_blkg(cfqg)->blkcg, fmt, ##args);	\
673 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()  argument
677 	blkg_rwstat_add(&cfqg->stats.queued, op, 1);  in cfqg_stats_update_io_add()
678 	cfqg_stats_end_empty_time(&cfqg->stats);  in cfqg_stats_update_io_add()
679 	cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);  in cfqg_stats_update_io_add()
682 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()  argument
685 	blkg_stat_add(&cfqg->stats.time, time);  in cfqg_stats_update_timeslice_used()
687 	blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);  in cfqg_stats_update_timeslice_used()
691 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg,  in cfqg_stats_update_io_remove()  argument
694 	blkg_rwstat_add(&cfqg->stats.queued, op, -1);  in cfqg_stats_update_io_remove()
697 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,  in cfqg_stats_update_io_merged()  argument
700 	blkg_rwstat_add(&cfqg->stats.merged, op, 1);  in cfqg_stats_update_io_merged()
703 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()  argument
708 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_completion()
762 static void cfqg_stats_xfer_dead(struct cfq_group *cfqg)  in cfqg_stats_xfer_dead()  argument
764 	struct cfq_group *parent = cfqg_parent(cfqg);  in cfqg_stats_xfer_dead()
766 	lockdep_assert_held(cfqg_to_blkg(cfqg)->q->queue_lock);  in cfqg_stats_xfer_dead()
771 	cfqg_stats_add_aux(&parent->stats, &cfqg->stats);  in cfqg_stats_xfer_dead()
772 	cfqg_stats_reset(&cfqg->stats);  in cfqg_stats_xfer_dead()
777 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg) { return NULL; }  in cfqg_parent()  argument
778 static inline bool cfqg_is_descendant(struct cfq_group *cfqg,  in cfqg_is_descendant()  argument
783 static inline void cfqg_get(struct cfq_group *cfqg) { }  in cfqg_get()  argument
784 static inline void cfqg_put(struct cfq_group *cfqg) { }  in cfqg_put()  argument
791 #define cfq_log_cfqg(cfqd, cfqg, fmt, args...)		do {} while (0)  argument
793 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()  argument
795 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()  argument
797 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg,  in cfqg_stats_update_io_remove()  argument
799 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,  in cfqg_stats_update_io_merged()  argument
801 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()  argument
812 #define for_each_cfqg_st(cfqg, i, j, st) \  argument
814 		for (j = 0, st = i < IDLE_WORKLOAD ? &cfqg->service_trees[i][j]\
815 			: &cfqg->service_tree_idle; \
819 			&cfqg->service_trees[i][j]: NULL) \
870 					struct cfq_group *cfqg)  in cfq_group_busy_queues_wl()  argument
873 		return cfqg->service_tree_idle.count;  in cfq_group_busy_queues_wl()
875 	return cfqg->service_trees[wl_class][ASYNC_WORKLOAD].count +  in cfq_group_busy_queues_wl()
876 		cfqg->service_trees[wl_class][SYNC_NOIDLE_WORKLOAD].count +  in cfq_group_busy_queues_wl()
877 		cfqg->service_trees[wl_class][SYNC_WORKLOAD].count;  in cfq_group_busy_queues_wl()
881 					struct cfq_group *cfqg)  in cfqg_busy_async_queues()  argument
883 	return cfqg->service_trees[RT_WORKLOAD][ASYNC_WORKLOAD].count +  in cfqg_busy_async_queues()
884 		cfqg->service_trees[BE_WORKLOAD][ASYNC_WORKLOAD].count;  in cfqg_busy_async_queues()
989 		struct cfq_group *cfqg = rb_entry_cfqg(st->rb.rb_leftmost);  in update_min_vdisktime()  local
992 						  cfqg->vdisktime);  in update_min_vdisktime()
1003 					struct cfq_group *cfqg, bool rt)  in cfq_group_get_avg_queues()  argument
1008 	unsigned busy = cfq_group_busy_queues_wl(rt, cfqd, cfqg);  in cfq_group_get_avg_queues()
1010 	min_q = min(cfqg->busy_queues_avg[rt], busy);  in cfq_group_get_avg_queues()
1011 	max_q = max(cfqg->busy_queues_avg[rt], busy);  in cfq_group_get_avg_queues()
1012 	cfqg->busy_queues_avg[rt] = (mult * max_q + min_q + round) /  in cfq_group_get_avg_queues()
1014 	return cfqg->busy_queues_avg[rt];  in cfq_group_get_avg_queues()
1018 cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_slice()  argument
1020 	return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT;  in cfq_group_slice()
1032 		unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg,  in cfq_scaled_cfqq_slice()
1036 		u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg);  in cfq_scaled_cfqq_slice()
1230 	return (cfqq->cfqg->nr_cfqq - 1) * (cfq_prio_slice(cfqd, 1, 0) -  in cfq_slice_offset()
1235 cfqg_key(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfqg_key()  argument
1237 	return cfqg->vdisktime - st->min_vdisktime;  in cfqg_key()
1241 __cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in __cfq_group_service_tree_add()  argument
1246 	s64 key = cfqg_key(st, cfqg);  in __cfq_group_service_tree_add()
1263 		st->rb_rightmost = &cfqg->rb_node;  in __cfq_group_service_tree_add()
1265 	rb_link_node(&cfqg->rb_node, parent, node);  in __cfq_group_service_tree_add()
1266 	rb_insert_color_cached(&cfqg->rb_node, &st->rb, leftmost);  in __cfq_group_service_tree_add()
1273 cfq_update_group_weight(struct cfq_group *cfqg)  in cfq_update_group_weight()  argument
1275 	if (cfqg->new_weight) {  in cfq_update_group_weight()
1276 		cfqg->weight = cfqg->new_weight;  in cfq_update_group_weight()
1277 		cfqg->new_weight = 0;  in cfq_update_group_weight()
1282 cfq_update_group_leaf_weight(struct cfq_group *cfqg)  in cfq_update_group_leaf_weight()  argument
1284 	BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));  in cfq_update_group_leaf_weight()
1286 	if (cfqg->new_leaf_weight) {  in cfq_update_group_leaf_weight()
1287 		cfqg->leaf_weight = cfqg->new_leaf_weight;  in cfq_update_group_leaf_weight()
1288 		cfqg->new_leaf_weight = 0;  in cfq_update_group_leaf_weight()
1293 cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_add()  argument
1296 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_add()
1301 	BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));  in cfq_group_service_tree_add()
1308 	cfq_update_group_leaf_weight(cfqg);  in cfq_group_service_tree_add()
1309 	__cfq_group_service_tree_add(st, cfqg);  in cfq_group_service_tree_add()
1340 	cfqg->vfraction = max_t(unsigned, vfr, 1);  in cfq_group_service_tree_add()
1352 cfq_group_notify_queue_add(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_add()  argument
1358 	cfqg->nr_cfqq++;  in cfq_group_notify_queue_add()
1359 	if (!RB_EMPTY_NODE(&cfqg->rb_node))  in cfq_group_notify_queue_add()
1370 		cfqg->vdisktime = __cfqg->vdisktime +  in cfq_group_notify_queue_add()
1373 		cfqg->vdisktime = st->min_vdisktime;  in cfq_group_notify_queue_add()
1374 	cfq_group_service_tree_add(st, cfqg);  in cfq_group_notify_queue_add()
1378 cfq_group_service_tree_del(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_del()  argument
1380 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_del()
1406 	if (!RB_EMPTY_NODE(&cfqg->rb_node))  in cfq_group_service_tree_del()
1407 		cfq_rb_erase(&cfqg->rb_node, st);  in cfq_group_service_tree_del()
1411 cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_del()  argument
1415 	BUG_ON(cfqg->nr_cfqq < 1);  in cfq_group_notify_queue_del()
1416 	cfqg->nr_cfqq--;  in cfq_group_notify_queue_del()
1419 	if (cfqg->nr_cfqq)  in cfq_group_notify_queue_del()
1422 	cfq_log_cfqg(cfqd, cfqg, "del_from_rr group");  in cfq_group_notify_queue_del()
1423 	cfq_group_service_tree_del(st, cfqg);  in cfq_group_notify_queue_del()
1424 	cfqg->saved_wl_slice = 0;  in cfq_group_notify_queue_del()
1425 	cfqg_stats_update_dequeue(cfqg);  in cfq_group_notify_queue_del()
1461 static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,  in cfq_group_served()  argument
1466 	int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)  in cfq_group_served()
1467 			- cfqg->service_tree_idle.count;  in cfq_group_served()
1485 	vfr = cfqg->vfraction;  in cfq_group_served()
1486 	cfq_group_service_tree_del(st, cfqg);  in cfq_group_served()
1487 	cfqg->vdisktime += cfqg_scale_charge(charge, vfr);  in cfq_group_served()
1488 	cfq_group_service_tree_add(st, cfqg);  in cfq_group_served()
1492 		cfqg->saved_wl_slice = cfqd->workload_expires - now;  in cfq_group_served()
1493 		cfqg->saved_wl_type = cfqd->serving_wl_type;  in cfq_group_served()
1494 		cfqg->saved_wl_class = cfqd->serving_wl_class;  in cfq_group_served()
1496 		cfqg->saved_wl_slice = 0;  in cfq_group_served()
1498 	cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,  in cfq_group_served()
1504 	cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);  in cfq_group_served()
1505 	cfqg_stats_set_start_empty_time(cfqg);  in cfq_group_served()
1515 static void cfq_init_cfqg_base(struct cfq_group *cfqg)  in cfq_init_cfqg_base()  argument
1520 	for_each_cfqg_st(cfqg, i, j, st)  in cfq_init_cfqg_base()
1522 	RB_CLEAR_NODE(&cfqg->rb_node);  in cfq_init_cfqg_base()
1524 	cfqg->ttime.last_end_request = ktime_get_ns();  in cfq_init_cfqg_base()
1617 	struct cfq_group *cfqg;  in cfq_pd_alloc()  local
1619 	cfqg = kzalloc_node(sizeof(*cfqg), gfp, node);  in cfq_pd_alloc()
1620 	if (!cfqg)  in cfq_pd_alloc()
1623 	cfq_init_cfqg_base(cfqg);  in cfq_pd_alloc()
1624 	if (cfqg_stats_init(&cfqg->stats, gfp)) {  in cfq_pd_alloc()
1625 		kfree(cfqg);  in cfq_pd_alloc()
1629 	return &cfqg->pd;  in cfq_pd_alloc()
1634 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_init()  local
1637 	cfqg->weight = cgd->weight;  in cfq_pd_init()
1638 	cfqg->leaf_weight = cgd->leaf_weight;  in cfq_pd_init()
1643 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_offline()  local
1647 		if (cfqg->async_cfqq[0][i])  in cfq_pd_offline()
1648 			cfq_put_queue(cfqg->async_cfqq[0][i]);  in cfq_pd_offline()
1649 		if (cfqg->async_cfqq[1][i])  in cfq_pd_offline()
1650 			cfq_put_queue(cfqg->async_cfqq[1][i]);  in cfq_pd_offline()
1653 	if (cfqg->async_idle_cfqq)  in cfq_pd_offline()
1654 		cfq_put_queue(cfqg->async_idle_cfqq);  in cfq_pd_offline()
1662 	cfqg_stats_xfer_dead(cfqg);  in cfq_pd_offline()
1667 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_free()  local
1669 	cfqg_stats_exit(&cfqg->stats);  in cfq_pd_free()
1670 	return kfree(cfqg);  in cfq_pd_free()
1675 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfq_pd_reset_stats()  local
1677 	cfqg_stats_reset(&cfqg->stats);  in cfq_pd_reset_stats()
1691 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()  argument
1693 	cfqq->cfqg = cfqg;  in cfq_link_cfqq_cfqg()
1695 	cfqg_get(cfqg);  in cfq_link_cfqq_cfqg()
1701 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_weight_device()  local
1703 	if (!cfqg->dev_weight)  in cfqg_prfill_weight_device()
1705 	return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);  in cfqg_prfill_weight_device()
1719 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_leaf_weight_device()  local
1721 	if (!cfqg->dev_leaf_weight)  in cfqg_prfill_leaf_weight_device()
1723 	return __blkg_prfill_u64(sf, pd, cfqg->dev_leaf_weight);  in cfqg_prfill_leaf_weight_device()
1768 	struct cfq_group *cfqg;  in __cfqg_set_weight_device()  local
1789 	cfqg = blkg_to_cfqg(ctx.blkg);  in __cfqg_set_weight_device()
1795 			cfqg->dev_weight = v;  in __cfqg_set_weight_device()
1796 			cfqg->new_weight = v ?: cfqgd->weight;  in __cfqg_set_weight_device()
1798 			cfqg->dev_leaf_weight = v;  in __cfqg_set_weight_device()
1799 			cfqg->new_leaf_weight = v ?: cfqgd->leaf_weight;  in __cfqg_set_weight_device()
1846 		struct cfq_group *cfqg = blkg_to_cfqg(blkg);  in __cfq_set_weight()  local
1848 		if (!cfqg)  in __cfq_set_weight()
1853 				cfqg->dev_weight = 0;  in __cfq_set_weight()
1854 			if (!cfqg->dev_weight)  in __cfq_set_weight()
1855 				cfqg->new_weight = cfqgd->weight;  in __cfq_set_weight()
1858 				cfqg->dev_leaf_weight = 0;  in __cfq_set_weight()
1859 			if (!cfqg->dev_leaf_weight)  in __cfq_set_weight()
1860 				cfqg->new_leaf_weight = cfqgd->leaf_weight;  in __cfq_set_weight()
1965 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_avg_queue_size()  local
1966 	u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);  in cfqg_prfill_avg_queue_size()
1970 		v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);  in cfqg_prfill_avg_queue_size()
2191 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()  argument
2192 	cfqq->cfqg = cfqg;  in cfq_link_cfqq_cfqg()
2213 	st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq));  in cfq_service_tree_add()
2274 	cfq_group_notify_queue_add(cfqd, cfqq->cfqg);  in cfq_service_tree_add()
2387 	cfq_group_notify_queue_del(cfqd, cfqq->cfqg);  in cfq_del_cfqq_rr()
2614 	cfqg_stats_update_idle_time(cfqq->cfqg);  in cfq_del_timer()
2623 		cfqg_stats_update_avg_queue_size(cfqq->cfqg);  in __cfq_set_active_queue()
2678 	cfq_group_served(cfqd, cfqq->cfqg, cfqq);  in __cfq_slice_expired()
2724 	struct cfq_group *cfqg;  in cfq_get_next_queue_forced()  local
2732 	cfqg = cfq_get_next_cfqg(cfqd);  in cfq_get_next_queue_forced()
2733 	if (!cfqg)  in cfq_get_next_queue_forced()
2736 	for_each_cfqg_st(cfqg, i, j, st) {  in cfq_get_next_queue_forced()
2838 	if (cur_cfqq->cfqg->nr_cfqq == 1)  in cfq_close_cooperator()
2851 	if (cur_cfqq->cfqg != cfqq->cfqg)  in cfq_close_cooperator()
2967 	    (cfqq->cfqg->nr_cfqq > 1 ||  in cfq_arm_slice_timer()
2980 	cfqg_stats_set_start_idle_time(cfqq->cfqg);  in cfq_arm_slice_timer()
3093 			struct cfq_group *cfqg, enum wl_class_t wl_class)  in cfq_choose_wl_type()  argument
3103 		queue = cfq_rb_first(st_for(cfqg, wl_class, i));  in cfq_choose_wl_type()
3116 choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)  in choose_wl_class_and_type()  argument
3126 	if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))  in choose_wl_class_and_type()
3128 	else if (cfq_group_busy_queues_wl(BE_WORKLOAD, cfqd, cfqg))  in choose_wl_class_and_type()
3144 	st = st_for(cfqg, cfqd->serving_wl_class, cfqd->serving_wl_type);  in choose_wl_class_and_type()
3155 	cfqd->serving_wl_type = cfq_choose_wl_type(cfqd, cfqg,  in choose_wl_class_and_type()
3157 	st = st_for(cfqg, cfqd->serving_wl_class, cfqd->serving_wl_type);  in choose_wl_class_and_type()
3165 	group_slice = cfq_group_slice(cfqd, cfqg);  in choose_wl_class_and_type()
3168 		max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class],  in choose_wl_class_and_type()
3170 					cfqg)));  in choose_wl_class_and_type()
3183 			cfqg_busy_async_queues(cfqd, cfqg);  in choose_wl_class_and_type()
3202 	struct cfq_group *cfqg;  in cfq_get_next_cfqg()  local
3206 	cfqg = cfq_rb_first_group(st);  in cfq_get_next_cfqg()
3208 	return cfqg;  in cfq_get_next_cfqg()
3213 	struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);  in cfq_choose_cfqg()  local
3216 	cfqd->serving_group = cfqg;  in cfq_choose_cfqg()
3219 	if (cfqg->saved_wl_slice) {  in cfq_choose_cfqg()
3220 		cfqd->workload_expires = now + cfqg->saved_wl_slice;  in cfq_choose_cfqg()
3221 		cfqd->serving_wl_type = cfqg->saved_wl_type;  in cfq_choose_cfqg()
3222 		cfqd->serving_wl_class = cfqg->saved_wl_class;  in cfq_choose_cfqg()
3226 	choose_wl_class_and_type(cfqd, cfqg);  in cfq_choose_cfqg()
3264 		if (cfqq->cfqg->nr_cfqq == 1 && RB_EMPTY_ROOT(&cfqq->sort_list)  in cfq_select_queue()
3323 	if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1 &&  in cfq_select_queue()
3324 	    cfqq->cfqg->dispatched &&  in cfq_select_queue()
3325 	    !cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) {  in cfq_select_queue()
3578 	struct cfq_group *cfqg;  in cfq_put_queue()  local
3589 	cfqg = cfqq->cfqg;  in cfq_put_queue()
3598 	cfqg_put(cfqg);  in cfq_put_queue()
3793 cfq_async_queue_prio(struct cfq_group *cfqg, int ioprio_class, int ioprio)  in cfq_async_queue_prio()  argument
3797 		return &cfqg->async_cfqq[0][ioprio];  in cfq_async_queue_prio()
3802 		return &cfqg->async_cfqq[1][ioprio];  in cfq_async_queue_prio()
3804 		return &cfqg->async_idle_cfqq;  in cfq_async_queue_prio()
3818 	struct cfq_group *cfqg;  in cfq_get_queue()  local
3821 	cfqg = cfq_lookup_cfqg(cfqd, bio_blkcg(bio));  in cfq_get_queue()
3822 	if (!cfqg) {  in cfq_get_queue()
3833 		async_cfqq = cfq_async_queue_prio(cfqg, ioprio_class, ioprio);  in cfq_get_queue()
3851 	cfq_link_cfqq_cfqg(cfqq, cfqg);  in cfq_get_queue()
3887 	__cfq_update_io_thinktime(&cfqq->cfqg->ttime, cfqd->cfq_group_idle);  in cfq_update_io_thinktime()
3998 	if (!cfqg_is_descendant(cfqq->cfqg, new_cfqq->cfqg))  in cfq_should_preempt()
4057 		cfqq->cfqg->saved_wl_slice = 0;  in cfq_preempt_queue()
4109 				cfqg_stats_update_idle_time(cfqq->cfqg);  in cfq_rq_enqueued()
4188 	if (cfqq->cfqg->nr_cfqq > 1)  in cfq_should_wait_busy()
4192 	if (cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true))  in cfq_should_wait_busy()
4232 	cfqg_stats_update_completion(cfqq->cfqg, rq->start_time_ns,  in cfq_completed_request()
4245 			st = st_for(cfqq->cfqg, cfqq_class(cfqq),  in cfq_completed_request()
4254 	cfqq->cfqg->ttime.last_end_request = now;  in cfq_completed_request()
4460 	cfqg_get(cfqq->cfqg);  in cfq_set_request()
4462 	rq->elv.priv[1] = cfqq->cfqg;  in cfq_set_request()