160 	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))  in bfqg_stats_update_io_add()
247 			      bfqq->bfqd->root_group;  in bfqq_group()
459 	struct bfq_data *bfqd = blkg->q->elevator->elevator_data;  in bfq_pd_init()  local
469 	bfqg->bfqd = bfqd;  in bfq_pd_init()
499 static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd,  in bfq_lookup_bfqg()  argument
504 	blkg = blkg_lookup(blkcg, bfqd->queue);  in bfq_lookup_bfqg()
510 struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,  in bfq_find_set_group()  argument
516 	bfqg = bfq_lookup_bfqg(bfqd, blkcg);  in bfq_find_set_group()
529 		if (bfqg != bfqd->root_group) {  in bfq_find_set_group()
532 				parent = bfqd->root_group;  in bfq_find_set_group()
554 void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_move()  argument
565 	if (bfqq == bfqd->in_service_queue)  in bfq_bfqq_move()
566 		bfq_bfqq_expire(bfqd, bfqd->in_service_queue,  in bfq_bfqq_move()
570 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);  in bfq_bfqq_move()
581 		bfq_pos_tree_add_move(bfqd, bfqq);  in bfq_bfqq_move()
582 		bfq_activate_bfqq(bfqd, bfqq);  in bfq_bfqq_move()
585 	if (!bfqd->in_service_queue && !bfqd->rq_in_driver)  in bfq_bfqq_move()
586 		bfq_schedule_dispatch(bfqd);  in bfq_bfqq_move()
603 static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,  in __bfq_bic_change_cgroup()  argument
612 	bfqg = bfq_find_set_group(bfqd, blkcg);  in __bfq_bic_change_cgroup()
615 		bfqg = bfqd->root_group;  in __bfq_bic_change_cgroup()
622 			bfq_log_bfqq(bfqd, async_bfqq,  in __bfq_bic_change_cgroup()
632 			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in __bfq_bic_change_cgroup()
640 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_bic_update_cgroup()  local
651 	if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))  in bfq_bic_update_cgroup()
654 	bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio));  in bfq_bic_update_cgroup()
728 static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,  in bfq_reparent_leaf_entity()  argument
733 	bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);  in bfq_reparent_leaf_entity()
743 static void bfq_reparent_active_entities(struct bfq_data *bfqd,  in bfq_reparent_active_entities()  argument
754 		bfq_reparent_leaf_entity(bfqd, entity);  in bfq_reparent_active_entities()
757 		bfq_reparent_leaf_entity(bfqd,  in bfq_reparent_active_entities()
773 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()  local
778 	spin_lock_irqsave(&bfqd->lock, flags);  in bfq_pd_offline()
809 		bfq_reparent_active_entities(bfqd, bfqg, st);  in bfq_pd_offline()
815 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
817 	spin_unlock_irqrestore(&bfqd->lock, flags);  in bfq_pd_offline()
827 void bfq_end_wr_async(struct bfq_data *bfqd)  in bfq_end_wr_async()  argument
831 	list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) {  in bfq_end_wr_async()
834 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
836 	bfq_end_wr_async_queues(bfqd, bfqd->root_group);  in bfq_end_wr_async()
1027 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()  argument
1031 	ret = blkcg_activate_policy(bfqd->queue, &blkcg_policy_bfq);  in bfq_create_group_hierarchy()
1035 	return blkg_to_bfqg(bfqd->queue->root_blkg);  in bfq_create_group_hierarchy()
1186 void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_move()  argument
1204 void bfq_end_wr_async(struct bfq_data *bfqd)  in bfq_end_wr_async()  argument
1206 	bfq_end_wr_async_queues(bfqd, bfqd->root_group);  in bfq_end_wr_async()
1209 struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, struct blkcg *blkcg)  in bfq_find_set_group()  argument
1211 	return bfqd->root_group;  in bfq_find_set_group()
1216 	return bfqq->bfqd->root_group;  in bfqq_group()
1219 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()  argument