138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()  argument
141 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_group_wait_time()
145 	if (bfqg == curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()  argument
168 	bfq_stat_add(&bfqg->stats.dequeue, 1);  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()  argument
173 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()  argument
192 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()  argument
206 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()  argument
214 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
225 	blkg_rwstat_add(&bfqg->stats.queued, op, 1);  in bfqg_stats_update_io_add()
226 	bfqg_stats_end_empty_time(&bfqg->stats);  in bfqg_stats_update_io_add()
227 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))  in bfqg_stats_update_io_add()
228 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_remove()  argument
233 	blkg_rwstat_add(&bfqg->stats.queued, op, -1);  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_merged()  argument
238 	blkg_rwstat_add(&bfqg->stats.merged, op, 1);  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
244 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
259 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_remove()  argument
260 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_merged()  argument
261 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
263 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()  argument
264 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_empty_time()  argument
265 void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_update_idle_time()  argument
266 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()  argument
267 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }  in bfqg_stats_update_avg_queue_size()  argument
284 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()  argument
286 	return pd_to_blkg(&bfqg->pd);  in bfqg_to_blkg()
301 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()  argument
303 	struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;  in bfqg_parent()
322 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()  argument
324 	bfqg->ref++;  in bfqg_get()
327 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()  argument
329 	bfqg->ref--;  in bfqg_put()
331 	if (bfqg->ref == 0)  in bfqg_put()
332 		kfree(bfqg);  in bfqg_put()
335 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()  argument
338 	bfqg_get(bfqg);  in bfqg_and_blkg_get()
340 	blkg_get(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_get()
343 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()  argument
345 	blkg_put(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_put()
347 	bfqg_put(bfqg);  in bfqg_and_blkg_put()
395 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()  argument
399 	if (!bfqg) /* root_group */  in bfqg_stats_xfer_dead()
402 	parent = bfqg_parent(bfqg);  in bfqg_stats_xfer_dead()
404 	lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);  in bfqg_stats_xfer_dead()
409 	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);  in bfqg_stats_xfer_dead()
410 	bfqg_stats_reset(&bfqg->stats);  in bfqg_stats_xfer_dead()
413 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
426 		bfqg_and_blkg_get(bfqg);  in bfq_init_entity()
428 	entity->parent = bfqg->my_entity; /* NULL for root group */  in bfq_init_entity()
429 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
507 	struct bfq_group *bfqg;  in bfq_pd_alloc()  local
509 	bfqg = kzalloc_node(sizeof(*bfqg), gfp, q->node);  in bfq_pd_alloc()
510 	if (!bfqg)  in bfq_pd_alloc()
513 	if (bfqg_stats_init(&bfqg->stats, gfp)) {  in bfq_pd_alloc()
514 		kfree(bfqg);  in bfq_pd_alloc()
519 	bfqg_get(bfqg);  in bfq_pd_alloc()
520 	return &bfqg->pd;  in bfq_pd_alloc()
526 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()  local
528 	struct bfq_entity *entity = &bfqg->entity;  in bfq_pd_init()
532 	entity->my_sched_data = &bfqg->sched_data;  in bfq_pd_init()
533 	bfqg->my_entity = entity; /*  in bfq_pd_init()
537 	bfqg->bfqd = bfqd;  in bfq_pd_init()
538 	bfqg->active_entities = 0;  in bfq_pd_init()
539 	bfqg->rq_pos_tree = RB_ROOT;  in bfq_pd_init()
544 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()  local
546 	bfqg_stats_exit(&bfqg->stats);  in bfq_pd_free()
547 	bfqg_put(bfqg);  in bfq_pd_free()
552 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()  local
554 	bfqg_stats_reset(&bfqg->stats);  in bfq_pd_reset_stats()
557 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()  argument
562 	entity = &bfqg->entity;  in bfq_group_set_parent()
581 	struct bfq_group *bfqg, *parent;  in bfq_find_set_group()  local
584 	bfqg = bfq_lookup_bfqg(bfqd, blkcg);  in bfq_find_set_group()
586 	if (unlikely(!bfqg))  in bfq_find_set_group()
594 	entity = &bfqg->entity;  in bfq_find_set_group()
596 		bfqg = container_of(entity, struct bfq_group, entity);  in bfq_find_set_group()
597 		if (bfqg != bfqd->root_group) {  in bfq_find_set_group()
598 			parent = bfqg_parent(bfqg);  in bfq_find_set_group()
601 			bfq_group_set_parent(bfqg, parent);  in bfq_find_set_group()
605 	return bfqg;  in bfq_find_set_group()
623 		   struct bfq_group *bfqg)  in bfq_bfqq_move()  argument
643 	entity->parent = bfqg->my_entity;  in bfq_bfqq_move()
644 	entity->sched_data = &bfqg->sched_data;  in bfq_bfqq_move()
646 	bfqg_and_blkg_get(bfqg);  in bfq_bfqq_move()
678 	struct bfq_group *bfqg;  in __bfq_bic_change_cgroup()  local
681 	bfqg = bfq_find_set_group(bfqd, blkcg);  in __bfq_bic_change_cgroup()
683 	if (unlikely(!bfqg))  in __bfq_bic_change_cgroup()
684 		bfqg = bfqd->root_group;  in __bfq_bic_change_cgroup()
689 		if (entity->sched_data != &bfqg->sched_data) {  in __bfq_bic_change_cgroup()
700 		if (entity->sched_data != &bfqg->sched_data)  in __bfq_bic_change_cgroup()
701 			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in __bfq_bic_change_cgroup()
704 	return bfqg;  in __bfq_bic_change_cgroup()
710 	struct bfq_group *bfqg = NULL;  in bfq_bic_update_cgroup()  local
723 	bfqg = __bfq_bic_change_cgroup(bfqd, bic, __bio_blkcg(bio));  in bfq_bic_update_cgroup()
774 	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));  in bfq_bic_update_cgroup()
813 					 struct bfq_group *bfqg,  in bfq_reparent_active_entities()  argument
825 	if (bfqg->sched_data.in_service_entity)  in bfq_reparent_active_entities()
827 			bfqg->sched_data.in_service_entity);  in bfq_reparent_active_entities()
841 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()  local
842 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()
843 	struct bfq_entity *entity = bfqg->my_entity;  in bfq_pd_offline()
857 		st = bfqg->sched_data.service_tree + i;  in bfq_pd_offline()
878 		bfq_reparent_active_entities(bfqd, bfqg, st);  in bfq_pd_offline()
884 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
893 	bfqg_stats_xfer_dead(bfqg);  in bfq_pd_offline()
901 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()  local
903 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
925 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()  local
927 	if (!bfqg->entity.dev_weight)  in bfqg_prfill_weight_device()
929 	return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);  in bfqg_prfill_weight_device()
943 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()  argument
947 	bfqg->entity.dev_weight = dev_weight;  in bfq_group_set_weight()
954 	if ((unsigned short)weight != bfqg->entity.new_weight) {  in bfq_group_set_weight()
955 		bfqg->entity.new_weight = (unsigned short)weight;  in bfq_group_set_weight()
972 		bfqg->entity.prio_changed = 1;  in bfq_group_set_weight()
992 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()  local
994 		if (bfqg)  in bfq_io_set_weight_legacy()
995 			bfq_group_set_weight(bfqg, val, 0);  in bfq_io_set_weight_legacy()
1009 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()  local
1028 	bfqg = blkg_to_bfqg(ctx.blkg);  in bfq_io_set_device_weight()
1032 		bfq_group_set_weight(bfqg, bfqg->entity.weight, v);  in bfq_io_set_device_weight()
1163 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()  local
1164 	u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);  in bfqg_prfill_avg_queue_size()
1168 		v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);  in bfqg_prfill_avg_queue_size()
1351 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()  argument
1353 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
1363 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
1385 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()  local
1388 	bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);  in bfq_create_group_hierarchy()
1389 	if (!bfqg)  in bfq_create_group_hierarchy()
1393 		bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;  in bfq_create_group_hierarchy()
1395 	return bfqg;  in bfq_create_group_hierarchy()