250 static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)  in sq_to_tg()  argument
252 	if (sq && sq->parent_sq)  in sq_to_tg()
253 		return container_of(sq, struct throtl_grp, service_queue);  in sq_to_tg()
265 static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)  in sq_to_td()  argument
267 	struct throtl_grp *tg = sq_to_tg(sq);  in sq_to_td()
272 		return container_of(sq, struct throtl_data, service_queue);  in sq_to_td()
367 #define throtl_log(sq, fmt, args...)	do {				\  argument
368 	struct throtl_grp *__tg = sq_to_tg((sq));			\
369 	struct throtl_data *__td = sq_to_td((sq));			\
474 static void throtl_service_queue_init(struct throtl_service_queue *sq)  in throtl_service_queue_init()  argument
476 	INIT_LIST_HEAD(&sq->queued[0]);  in throtl_service_queue_init()
477 	INIT_LIST_HEAD(&sq->queued[1]);  in throtl_service_queue_init()
478 	sq->pending_tree = RB_ROOT;  in throtl_service_queue_init()
479 	timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0);  in throtl_service_queue_init()
522 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_pd_init()  local
537 	sq->parent_sq = &td->service_queue;  in throtl_pd_init()
539 		sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue;  in throtl_pd_init()
712 static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,  in throtl_schedule_pending_timer()  argument
715 	unsigned long max_expire = jiffies + 8 * sq_to_td(sq)->throtl_slice;  in throtl_schedule_pending_timer()
726 	mod_timer(&sq->pending_timer, expires);  in throtl_schedule_pending_timer()
727 	throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",  in throtl_schedule_pending_timer()
749 static bool throtl_schedule_next_dispatch(struct throtl_service_queue *sq,  in throtl_schedule_next_dispatch()  argument
753 	if (!sq->nr_pending)  in throtl_schedule_next_dispatch()
756 	update_min_dispatch_time(sq);  in throtl_schedule_next_dispatch()
759 	if (force || time_after(sq->first_pending_disptime, jiffies)) {  in throtl_schedule_next_dispatch()
760 		throtl_schedule_pending_timer(sq, sq->first_pending_disptime);  in throtl_schedule_next_dispatch()
1069 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_add_bio_tg()  local
1081 	if (!sq->nr_queued[rw])  in throtl_add_bio_tg()
1084 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
1086 	sq->nr_queued[rw]++;  in throtl_add_bio_tg()
1092 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_update_disptime()  local
1096 	bio = throtl_peek_queued(&sq->queued[READ]);  in tg_update_disptime()
1100 	bio = throtl_peek_queued(&sq->queued[WRITE]);  in tg_update_disptime()
1128 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_dispatch_one_bio()  local
1129 	struct throtl_service_queue *parent_sq = sq->parent_sq;  in tg_dispatch_one_bio()
1140 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
1141 	sq->nr_queued[rw]--;  in tg_dispatch_one_bio()
1170 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_dispatch_tg()  local
1178 	while ((bio = throtl_peek_queued(&sq->queued[READ])) &&  in throtl_dispatch_tg()
1188 	while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&  in throtl_dispatch_tg()
1207 		struct throtl_service_queue *sq;  in throtl_select_dispatch()  local
1219 		sq = &tg->service_queue;  in throtl_select_dispatch()
1220 		if (sq->nr_queued[0] || sq->nr_queued[1])  in throtl_select_dispatch()
1249 	struct throtl_service_queue *sq = from_timer(sq, t, pending_timer);  in throtl_pending_timer_fn()  local
1250 	struct throtl_grp *tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1251 	struct throtl_data *td = sq_to_td(sq);  in throtl_pending_timer_fn()
1262 	parent_sq = sq->parent_sq;  in throtl_pending_timer_fn()
1266 		throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",  in throtl_pending_timer_fn()
1267 			   sq->nr_queued[READ] + sq->nr_queued[WRITE],  in throtl_pending_timer_fn()
1268 			   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in throtl_pending_timer_fn()
1270 		ret = throtl_select_dispatch(sq);  in throtl_pending_timer_fn()
1272 			throtl_log(sq, "bios disp=%u", ret);  in throtl_pending_timer_fn()
1276 		if (throtl_schedule_next_dispatch(sq, false))  in throtl_pending_timer_fn()
1294 				sq = parent_sq;  in throtl_pending_timer_fn()
1295 				tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1380 	struct throtl_service_queue *sq = &tg->service_queue;  in tg_conf_updated()  local
1430 		throtl_schedule_next_dispatch(sq->parent_sq, true);  in tg_conf_updated()
1818 	struct throtl_service_queue *sq = &tg->service_queue;  in throtl_tg_can_upgrade()  local
1829 	if (read_limit && sq->nr_queued[READ] &&  in throtl_tg_can_upgrade()
1830 	    (!write_limit || sq->nr_queued[WRITE]))  in throtl_tg_can_upgrade()
1832 	if (write_limit && sq->nr_queued[WRITE] &&  in throtl_tg_can_upgrade()
1833 	    (!read_limit || sq->nr_queued[READ]))  in throtl_tg_can_upgrade()
1916 		struct throtl_service_queue *sq = &tg->service_queue;  in throtl_upgrade_state()  local
1919 		throtl_select_dispatch(sq);  in throtl_upgrade_state()
1920 		throtl_schedule_next_dispatch(sq, true);  in throtl_upgrade_state()
2144 	struct throtl_service_queue *sq;  in blk_throtl_bio()  local
2165 	sq = &tg->service_queue;  in blk_throtl_bio()
2174 		if (sq->nr_queued[rw])  in blk_throtl_bio()
2209 		sq = sq->parent_sq;  in blk_throtl_bio()
2210 		tg = sq_to_tg(sq);  in blk_throtl_bio()
2216 	throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",  in blk_throtl_bio()
2221 		   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in blk_throtl_bio()
2343 		struct throtl_service_queue *sq = &tg->service_queue;  in tg_drain_bios()  local
2348 		while ((bio = throtl_peek_queued(&sq->queued[READ])))  in tg_drain_bios()
2350 		while ((bio = throtl_peek_queued(&sq->queued[WRITE])))  in tg_drain_bios()