71 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()  argument
74 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_group_wait_time()
78 	if (bfqg == curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
99 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()  argument
101 	blkg_stat_add(&bfqg->stats.dequeue, 1);  in bfqg_stats_update_dequeue()
104 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()  argument
106 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_empty_time()
123 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()  argument
125 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_idle_time()
137 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()  argument
139 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_idle_time()
145 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()  argument
147 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_avg_queue_size()
155 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
158 	blkg_rwstat_add(&bfqg->stats.queued, op, 1);  in bfqg_stats_update_io_add()
159 	bfqg_stats_end_empty_time(&bfqg->stats);  in bfqg_stats_update_io_add()
160 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))  in bfqg_stats_update_io_add()
161 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));  in bfqg_stats_update_io_add()
164 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_remove()  argument
166 	blkg_rwstat_add(&bfqg->stats.queued, op, -1);  in bfqg_stats_update_io_remove()
169 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)  in bfqg_stats_update_io_merged()  argument
171 	blkg_rwstat_add(&bfqg->stats.merged, op, 1);  in bfqg_stats_update_io_merged()
174 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
177 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_completion()
190 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
192 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_remove()  argument
193 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }  in bfqg_stats_update_io_merged()  argument
194 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
196 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()  argument
197 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_empty_time()  argument
198 void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_update_idle_time()  argument
199 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()  argument
200 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }  in bfqg_stats_update_avg_queue_size()  argument
217 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()  argument
219 	return pd_to_blkg(&bfqg->pd);  in bfqg_to_blkg()
234 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()  argument
236 	struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;  in bfqg_parent()
255 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()  argument
257 	bfqg->ref++;  in bfqg_get()
260 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()  argument
262 	bfqg->ref--;  in bfqg_put()
264 	if (bfqg->ref == 0)  in bfqg_put()
265 		kfree(bfqg);  in bfqg_put()
268 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()  argument
271 	bfqg_get(bfqg);  in bfqg_and_blkg_get()
273 	blkg_get(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_get()
276 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()  argument
278 	blkg_put(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_put()
280 	bfqg_put(bfqg);  in bfqg_and_blkg_put()
328 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()  argument
332 	if (!bfqg) /* root_group */  in bfqg_stats_xfer_dead()
335 	parent = bfqg_parent(bfqg);  in bfqg_stats_xfer_dead()
337 	lockdep_assert_held(bfqg_to_blkg(bfqg)->q->queue_lock);  in bfqg_stats_xfer_dead()
342 	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);  in bfqg_stats_xfer_dead()
343 	bfqg_stats_reset(&bfqg->stats);  in bfqg_stats_xfer_dead()
346 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
359 		bfqg_and_blkg_get(bfqg);  in bfq_init_entity()
361 	entity->parent = bfqg->my_entity; /* NULL for root group */  in bfq_init_entity()
362 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
439 	struct bfq_group *bfqg;  in bfq_pd_alloc()  local
441 	bfqg = kzalloc_node(sizeof(*bfqg), gfp, node);  in bfq_pd_alloc()
442 	if (!bfqg)  in bfq_pd_alloc()
445 	if (bfqg_stats_init(&bfqg->stats, gfp)) {  in bfq_pd_alloc()
446 		kfree(bfqg);  in bfq_pd_alloc()
451 	bfqg_get(bfqg);  in bfq_pd_alloc()
452 	return &bfqg->pd;  in bfq_pd_alloc()
458 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()  local
460 	struct bfq_entity *entity = &bfqg->entity;  in bfq_pd_init()
464 	entity->my_sched_data = &bfqg->sched_data;  in bfq_pd_init()
465 	bfqg->my_entity = entity; /*  in bfq_pd_init()
469 	bfqg->bfqd = bfqd;  in bfq_pd_init()
470 	bfqg->active_entities = 0;  in bfq_pd_init()
471 	bfqg->rq_pos_tree = RB_ROOT;  in bfq_pd_init()
476 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()  local
478 	bfqg_stats_exit(&bfqg->stats);  in bfq_pd_free()
479 	bfqg_put(bfqg);  in bfq_pd_free()
484 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()  local
486 	bfqg_stats_reset(&bfqg->stats);  in bfq_pd_reset_stats()
489 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()  argument
494 	entity = &bfqg->entity;  in bfq_group_set_parent()
513 	struct bfq_group *bfqg, *parent;  in bfq_find_set_group()  local
516 	bfqg = bfq_lookup_bfqg(bfqd, blkcg);  in bfq_find_set_group()
518 	if (unlikely(!bfqg))  in bfq_find_set_group()
526 	entity = &bfqg->entity;  in bfq_find_set_group()
528 		bfqg = container_of(entity, struct bfq_group, entity);  in bfq_find_set_group()
529 		if (bfqg != bfqd->root_group) {  in bfq_find_set_group()
530 			parent = bfqg_parent(bfqg);  in bfq_find_set_group()
533 			bfq_group_set_parent(bfqg, parent);  in bfq_find_set_group()
537 	return bfqg;  in bfq_find_set_group()
555 		   struct bfq_group *bfqg)  in bfq_bfqq_move()  argument
575 	entity->parent = bfqg->my_entity;  in bfq_bfqq_move()
576 	entity->sched_data = &bfqg->sched_data;  in bfq_bfqq_move()
578 	bfqg_and_blkg_get(bfqg);  in bfq_bfqq_move()
609 	struct bfq_group *bfqg;  in __bfq_bic_change_cgroup()  local
612 	bfqg = bfq_find_set_group(bfqd, blkcg);  in __bfq_bic_change_cgroup()
614 	if (unlikely(!bfqg))  in __bfq_bic_change_cgroup()
615 		bfqg = bfqd->root_group;  in __bfq_bic_change_cgroup()
620 		if (entity->sched_data != &bfqg->sched_data) {  in __bfq_bic_change_cgroup()
631 		if (entity->sched_data != &bfqg->sched_data)  in __bfq_bic_change_cgroup()
632 			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in __bfq_bic_change_cgroup()
635 	return bfqg;  in __bfq_bic_change_cgroup()
641 	struct bfq_group *bfqg = NULL;  in bfq_bic_update_cgroup()  local
654 	bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio));  in bfq_bic_update_cgroup()
705 	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));  in bfq_bic_update_cgroup()
744 					 struct bfq_group *bfqg,  in bfq_reparent_active_entities()  argument
756 	if (bfqg->sched_data.in_service_entity)  in bfq_reparent_active_entities()
758 			bfqg->sched_data.in_service_entity);  in bfq_reparent_active_entities()
772 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()  local
773 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()
774 	struct bfq_entity *entity = bfqg->my_entity;  in bfq_pd_offline()
788 		st = bfqg->sched_data.service_tree + i;  in bfq_pd_offline()
809 		bfq_reparent_active_entities(bfqd, bfqg, st);  in bfq_pd_offline()
815 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
824 	bfqg_stats_xfer_dead(bfqg);  in bfq_pd_offline()
832 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()  local
834 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
869 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()  local
871 		if (!bfqg)  in bfq_io_set_weight_legacy()
879 		if ((unsigned short)val != bfqg->entity.new_weight) {  in bfq_io_set_weight_legacy()
880 			bfqg->entity.new_weight = (unsigned short)val;  in bfq_io_set_weight_legacy()
897 			bfqg->entity.prio_changed = 1;  in bfq_io_set_weight_legacy()
1005 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()  local
1006 	u64 samples = blkg_stat_read(&bfqg->stats.avg_queue_size_samples);  in bfqg_prfill_avg_queue_size()
1010 		v = blkg_stat_read(&bfqg->stats.avg_queue_size_sum);  in bfqg_prfill_avg_queue_size()
1187 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()  argument
1189 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
1199 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
1221 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()  local
1224 	bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);  in bfq_create_group_hierarchy()
1225 	if (!bfqg)  in bfq_create_group_hierarchy()
1229 		bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;  in bfq_create_group_hierarchy()
1231 	return bfqg;  in bfq_create_group_hierarchy()