332 	INIT_LIST_HEAD(&sq->queued[READ]);  in throtl_service_queue_init()
356 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_pd_alloc()
362 	tg->bps[READ][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
364 	tg->iops[READ][LIMIT_MAX] = UINT_MAX;  in throtl_pd_alloc()
366 	tg->bps_conf[READ][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
368 	tg->iops_conf[READ][LIMIT_MAX] = UINT_MAX;  in throtl_pd_alloc()
424 	for (rw = READ; rw <= WRITE; rw++) {  in tg_update_has_rules()
457 		if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] ||  in blk_throtl_update_limit_valid()
458 		    tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) {  in blk_throtl_update_limit_valid()
478 	tg->bps[READ][LIMIT_LOW] = 0;  in throtl_pd_offline()
480 	tg->iops[READ][LIMIT_LOW] = 0;  in throtl_pd_offline()
653 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice_with_credit()
671 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice()
687 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_extend_slice()
791 		   rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice,  in throtl_trim_slice()
820 	if (tg->service_queue.nr_queued[READ])  in tg_update_carryover()
821 		__tg_update_carryover(tg, READ);  in tg_update_carryover()
827 		   tg->carryover_bytes[READ], tg->carryover_bytes[WRITE],  in tg_update_carryover()
828 		   tg->carryover_ios[READ], tg->carryover_ios[WRITE]);  in tg_update_carryover()
1015 	bio = throtl_peek_queued(&sq->queued[READ]);  in tg_update_disptime()
1098 	while ((bio = throtl_peek_queued(&sq->queued[READ])) &&  in throtl_dispatch_tg()
1142 		if (sq->nr_queued[READ] || sq->nr_queued[WRITE])  in throtl_select_dispatch()
1201 			   sq->nr_queued[READ] + sq->nr_queued[WRITE],  in throtl_pending_timer_fn()
1202 			   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in throtl_pending_timer_fn()
1263 	for (rw = READ; rw <= WRITE; rw++)  in blk_throtl_dispatch_work_fn()
1320 		   tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE),  in tg_conf_updated()
1321 		   tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE));  in tg_conf_updated()
1359 	throtl_start_new_slice(tg, READ, false);  in tg_conf_updated()
1445 		.private = offsetof(struct throtl_grp, bps[READ][LIMIT_MAX]),
1457 		.private = offsetof(struct throtl_grp, iops[READ][LIMIT_MAX]),
1512 	if (tg->bps_conf[READ][off] == bps_dft &&  in tg_prfill_limit()
1514 	    tg->iops_conf[READ][off] == iops_dft &&  in tg_prfill_limit()
1521 	if (tg->bps_conf[READ][off] != U64_MAX)  in tg_prfill_limit()
1523 			tg->bps_conf[READ][off]);  in tg_prfill_limit()
1527 	if (tg->iops_conf[READ][off] != UINT_MAX)  in tg_prfill_limit()
1529 			tg->iops_conf[READ][off]);  in tg_prfill_limit()
1581 	v[0] = tg->bps_conf[READ][index];  in tg_set_limit()
1583 	v[2] = tg->iops_conf[READ][index];  in tg_set_limit()
1627 	tg->bps_conf[READ][index] = v[0];  in tg_set_limit()
1629 	tg->iops_conf[READ][index] = v[2];  in tg_set_limit()
1633 		tg->bps[READ][index] = v[0];  in tg_set_limit()
1635 		tg->iops[READ][index] = v[2];  in tg_set_limit()
1638 	tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW],  in tg_set_limit()
1639 		tg->bps_conf[READ][LIMIT_MAX]);  in tg_set_limit()
1642 	tg->iops[READ][LIMIT_LOW] = min(tg->iops_conf[READ][LIMIT_LOW],  in tg_set_limit()
1643 		tg->iops_conf[READ][LIMIT_MAX]);  in tg_set_limit()
1650 	if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] ||  in tg_set_limit()
1654 		tg->bps[READ][LIMIT_LOW] = 0;  in tg_set_limit()
1656 		tg->iops[READ][LIMIT_LOW] = 0;  in tg_set_limit()
1767 	if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW])  in __tg_last_low_overflow_time()
1768 		rtime = tg->last_low_overflow_time[READ];  in __tg_last_low_overflow_time()
1790 		if (!parent->bps[READ][LIMIT_LOW] &&  in tg_last_low_overflow_time()
1791 		    !parent->iops[READ][LIMIT_LOW] &&  in tg_last_low_overflow_time()
1848 	if (throtl_low_limit_reached(tg, READ) &&  in throtl_tg_can_upgrade()
2014 	if (tg->bps[READ][LIMIT_LOW]) {  in throtl_downgrade_check()
2015 		bps = tg->last_bytes_disp[READ] * HZ;  in throtl_downgrade_check()
2017 		if (bps >= tg->bps[READ][LIMIT_LOW])  in throtl_downgrade_check()
2018 			tg->last_low_overflow_time[READ] = now;  in throtl_downgrade_check()
2028 	if (tg->iops[READ][LIMIT_LOW]) {  in throtl_downgrade_check()
2029 		iops = tg->last_io_disp[READ] * HZ / elapsed_time;  in throtl_downgrade_check()
2030 		if (iops >= tg->iops[READ][LIMIT_LOW])  in throtl_downgrade_check()
2031 			tg->last_low_overflow_time[READ] = now;  in throtl_downgrade_check()
2047 	tg->last_bytes_disp[READ] = 0;  in throtl_downgrade_check()
2049 	tg->last_io_disp[READ] = 0;  in throtl_downgrade_check()
2084 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2115 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2141 			td->avg_buckets[READ][i].latency,  in throtl_update_latency_buckets()
2142 			td->avg_buckets[READ][i].valid,  in throtl_update_latency_buckets()
2247 		   rw == READ ? 'R' : 'W',  in __blk_throtl_bio()
2251 		   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in __blk_throtl_bio()
2375 	td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) *  in blk_throtl_init()
2377 	if (!td->latency_buckets[READ]) {  in blk_throtl_init()
2384 		free_percpu(td->latency_buckets[READ]);  in blk_throtl_init()
2403 		free_percpu(td->latency_buckets[READ]);  in blk_throtl_init()
2418 	free_percpu(q->td->latency_buckets[READ]);  in blk_throtl_exit()
2439 			td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY;  in blk_throtl_register()