182 static inline struct iolatency_grp *blkg_to_lat(struct blkcg_gq *blkg)  in blkg_to_lat()  argument
184 	return pd_to_lat(blkg_to_pd(blkg, &blkcg_policy_iolatency));  in blkg_to_lat()
463 	struct blkcg_gq *blkg = bio->bi_blkg;  in blkcg_iolatency_throttle()  local
469 	while (blkg && blkg->parent) {  in blkcg_iolatency_throttle()
470 		struct iolatency_grp *iolat = blkg_to_lat(blkg);  in blkcg_iolatency_throttle()
472 			blkg = blkg->parent;  in blkcg_iolatency_throttle()
479 		blkg = blkg->parent;  in blkcg_iolatency_throttle()
522 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_check_latencies()  local
539 	parent = blkg_to_lat(blkg->parent);  in iolatency_check_latencies()
590 	struct blkcg_gq *blkg;  in blkcg_iolatency_done_bio()  local
599 	blkg = bio->bi_blkg;  in blkcg_iolatency_done_bio()
600 	if (!blkg || !bio_flagged(bio, BIO_TRACKED))  in blkcg_iolatency_done_bio()
611 	while (blkg && blkg->parent) {  in blkcg_iolatency_done_bio()
612 		iolat = blkg_to_lat(blkg);  in blkcg_iolatency_done_bio()
614 			blkg = blkg->parent;  in blkcg_iolatency_done_bio()
637 		blkg = blkg->parent;  in blkcg_iolatency_done_bio()
659 	struct blkcg_gq *blkg;  in blkiolatency_timer_fn()  local
664 	blkg_for_each_descendant_pre(blkg, pos_css,  in blkiolatency_timer_fn()
675 		if (!blkg_tryget(blkg))  in blkiolatency_timer_fn()
678 		iolat = blkg_to_lat(blkg);  in blkiolatency_timer_fn()
712 		blkg_put(blkg);  in blkiolatency_timer_fn()
750 static int iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val)  in iolatency_set_min_lat_nsec()  argument
752 	struct iolatency_grp *iolat = blkg_to_lat(blkg);  in iolatency_set_min_lat_nsec()
763 		blkcg_clear_delay(blkg);  in iolatency_set_min_lat_nsec()
769 static void iolatency_clear_scaling(struct blkcg_gq *blkg)  in iolatency_clear_scaling()  argument
771 	if (blkg->parent) {  in iolatency_clear_scaling()
772 		struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);  in iolatency_clear_scaling()
791 	struct blkcg_gq *blkg;  in iolatency_set_limit()  local
804 	iolat = blkg_to_lat(ctx.blkg);  in iolatency_set_limit()
830 	blkg = ctx.blkg;  in iolatency_set_limit()
833 	enable = iolatency_set_min_lat_nsec(blkg, lat_val);  in iolatency_set_limit()
835 		WARN_ON_ONCE(!blk_get_queue(blkg->q));  in iolatency_set_limit()
836 		blkg_get(blkg);  in iolatency_set_limit()
840 		iolatency_clear_scaling(blkg);  in iolatency_set_limit()
847 		struct iolatency_grp *tmp = blkg_to_lat(blkg);  in iolatency_set_limit()
850 		blk_mq_freeze_queue(blkg->q);  in iolatency_set_limit()
859 		blk_mq_unfreeze_queue(blkg->q);  in iolatency_set_limit()
861 		blkg_put(blkg);  in iolatency_set_limit()
862 		blk_put_queue(blkg->q);  in iolatency_set_limit()
871 	const char *dname = blkg_dev_name(pd->blkg);  in iolatency_prfill_limit()
958 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_init()  local
959 	struct rq_qos *rqos = blkcg_rq_qos(blkg->q);  in iolatency_pd_init()
964 	if (blk_queue_nonrot(blkg->q))  in iolatency_pd_init()
978 	iolat->rq_depth.queue_depth = blkg->q->nr_requests;  in iolatency_pd_init()
989 	if (blkg->parent && blkg_to_pd(blkg->parent, &blkcg_policy_iolatency)) {  in iolatency_pd_init()
990 		struct iolatency_grp *parent = blkg_to_lat(blkg->parent);  in iolatency_pd_init()
1003 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_offline()  local
1007 	ret = iolatency_set_min_lat_nsec(blkg, 0);  in iolatency_pd_offline()
1012 	iolatency_clear_scaling(blkg);  in iolatency_pd_offline()