509 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_pd_alloc()
516 	tg->bps[WRITE][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
518 	tg->iops[WRITE][LIMIT_MAX] = UINT_MAX;  in throtl_pd_alloc()
520 	tg->bps_conf[WRITE][LIMIT_MAX] = U64_MAX;  in throtl_pd_alloc()
522 	tg->iops_conf[WRITE][LIMIT_MAX] = UINT_MAX;  in throtl_pd_alloc()
576 	for (rw = READ; rw <= WRITE; rw++)  in tg_update_has_rules()
604 		if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] ||  in blk_throtl_update_limit_valid()
605 		    tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) {  in blk_throtl_update_limit_valid()
626 	tg->bps[WRITE][LIMIT_LOW] = 0;  in throtl_pd_offline()
628 	tg->iops[WRITE][LIMIT_LOW] = 0;  in throtl_pd_offline()
1127 	bio = throtl_peek_queued(&sq->queued[WRITE]);  in tg_update_disptime()
1215 	while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&  in throtl_dispatch_tg()
1298 			   sq->nr_queued[READ] + sq->nr_queued[WRITE],  in throtl_pending_timer_fn()
1299 			   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in throtl_pending_timer_fn()
1360 	for (rw = READ; rw <= WRITE; rw++)  in blk_throtl_dispatch_work_fn()
1417 		   tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE),  in tg_conf_updated()
1418 		   tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE));  in tg_conf_updated()
1457 	throtl_start_new_slice(tg, WRITE);  in tg_conf_updated()
1545 		.private = offsetof(struct throtl_grp, bps[WRITE][LIMIT_MAX]),
1557 		.private = offsetof(struct throtl_grp, iops[WRITE][LIMIT_MAX]),
1607 	    tg->bps_conf[WRITE][off] == bps_dft &&  in tg_prfill_limit()
1609 	    tg->iops_conf[WRITE][off] == iops_dft &&  in tg_prfill_limit()
1618 	if (tg->bps_conf[WRITE][off] != U64_MAX)  in tg_prfill_limit()
1620 			tg->bps_conf[WRITE][off]);  in tg_prfill_limit()
1624 	if (tg->iops_conf[WRITE][off] != UINT_MAX)  in tg_prfill_limit()
1626 			tg->iops_conf[WRITE][off]);  in tg_prfill_limit()
1673 	v[1] = tg->bps_conf[WRITE][index];  in tg_set_limit()
1675 	v[3] = tg->iops_conf[WRITE][index];  in tg_set_limit()
1719 	tg->bps_conf[WRITE][index] = v[1];  in tg_set_limit()
1721 	tg->iops_conf[WRITE][index] = v[3];  in tg_set_limit()
1725 		tg->bps[WRITE][index] = v[1];  in tg_set_limit()
1727 		tg->iops[WRITE][index] = v[3];  in tg_set_limit()
1731 	tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW],  in tg_set_limit()
1732 		tg->bps_conf[WRITE][LIMIT_MAX]);  in tg_set_limit()
1735 	tg->iops[WRITE][LIMIT_LOW] = min(tg->iops_conf[WRITE][LIMIT_LOW],  in tg_set_limit()
1736 		tg->iops_conf[WRITE][LIMIT_MAX]);  in tg_set_limit()
1742 	      tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) ||  in tg_set_limit()
1746 		tg->bps[WRITE][LIMIT_LOW] = 0;  in tg_set_limit()
1748 		tg->iops[WRITE][LIMIT_LOW] = 0;  in tg_set_limit()
1814 	if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW])  in __tg_last_low_overflow_time()
1815 		wtime = tg->last_low_overflow_time[WRITE];  in __tg_last_low_overflow_time()
1838 		    !parent->bps[WRITE][LIMIT_LOW] &&  in tg_last_low_overflow_time()
1839 		    !parent->iops[WRITE][LIMIT_LOW])  in tg_last_low_overflow_time()
1883 	write_limit = tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW];  in throtl_tg_can_upgrade()
1887 	    (!write_limit || sq->nr_queued[WRITE]))  in throtl_tg_can_upgrade()
1889 	if (write_limit && sq->nr_queued[WRITE] &&  in throtl_tg_can_upgrade()
2058 	if (tg->bps[WRITE][LIMIT_LOW]) {  in throtl_downgrade_check()
2059 		bps = tg->last_bytes_disp[WRITE] * HZ;  in throtl_downgrade_check()
2061 		if (bps >= tg->bps[WRITE][LIMIT_LOW])  in throtl_downgrade_check()
2062 			tg->last_low_overflow_time[WRITE] = now;  in throtl_downgrade_check()
2072 	if (tg->iops[WRITE][LIMIT_LOW]) {  in throtl_downgrade_check()
2073 		tg->last_io_disp[WRITE] += atomic_xchg(&tg->last_io_split_cnt[WRITE], 0);  in throtl_downgrade_check()
2074 		iops = tg->last_io_disp[WRITE] * HZ / elapsed_time;  in throtl_downgrade_check()
2075 		if (iops >= tg->iops[WRITE][LIMIT_LOW])  in throtl_downgrade_check()
2076 			tg->last_low_overflow_time[WRITE] = now;  in throtl_downgrade_check()
2087 	tg->last_bytes_disp[WRITE] = 0;  in throtl_downgrade_check()
2089 	tg->last_io_disp[WRITE] = 0;  in throtl_downgrade_check()
2124 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2155 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2183 			td->avg_buckets[WRITE][i].latency,  in throtl_update_latency_buckets()
2184 			td->avg_buckets[WRITE][i].valid);  in throtl_update_latency_buckets()
2299 		   sq->nr_queued[READ], sq->nr_queued[WRITE]);  in blk_throtl_bio()
2429 	td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) *  in blk_throtl_init()
2431 	if (!td->latency_buckets[WRITE]) {  in blk_throtl_init()
2452 		free_percpu(td->latency_buckets[WRITE]);  in blk_throtl_init()
2465 	free_percpu(q->td->latency_buckets[WRITE]);  in blk_throtl_exit()
2485 			td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY;  in blk_throtl_register_queue()