293 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)  in tg_bps_limit()  argument
303 	ret = tg->bps[rw][td->limit_index];  in tg_bps_limit()
307 		    tg->iops[rw][td->limit_index])  in tg_bps_limit()
313 	if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] &&  in tg_bps_limit()
314 	    tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) {  in tg_bps_limit()
317 		adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td);  in tg_bps_limit()
318 		ret = min(tg->bps[rw][LIMIT_MAX], adjusted);  in tg_bps_limit()
323 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)  in tg_iops_limit()  argument
333 	ret = tg->iops[rw][td->limit_index];  in tg_iops_limit()
337 		    tg->bps[rw][td->limit_index])  in tg_iops_limit()
343 	if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] &&  in tg_iops_limit()
344 	    tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) {  in tg_iops_limit()
347 		adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td);  in tg_iops_limit()
350 		ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted);  in tg_iops_limit()
485 	int rw;  in throtl_pd_alloc()  local
493 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_pd_alloc()
494 		throtl_qnode_init(&tg->qnode_on_self[rw], tg);  in throtl_pd_alloc()
495 		throtl_qnode_init(&tg->qnode_on_parent[rw], tg);  in throtl_pd_alloc()
552 	int rw;  in tg_update_has_rules()  local
554 	for (rw = READ; rw <= WRITE; rw++)  in tg_update_has_rules()
555 		tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) ||  in tg_update_has_rules()
557 			 (tg_bps_limit(tg, rw) != U64_MAX ||  in tg_update_has_rules()
558 			  tg_iops_limit(tg, rw) != UINT_MAX));  in tg_update_has_rules()
769 		bool rw, unsigned long start)  in throtl_start_new_slice_with_credit()  argument
771 	tg->bytes_disp[rw] = 0;  in throtl_start_new_slice_with_credit()
772 	tg->io_disp[rw] = 0;  in throtl_start_new_slice_with_credit()
780 	if (time_after_eq(start, tg->slice_start[rw]))  in throtl_start_new_slice_with_credit()
781 		tg->slice_start[rw] = start;  in throtl_start_new_slice_with_credit()
783 	tg->slice_end[rw] = jiffies + tg->td->throtl_slice;  in throtl_start_new_slice_with_credit()
786 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice_with_credit()
787 		   tg->slice_end[rw], jiffies);  in throtl_start_new_slice_with_credit()
790 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)  in throtl_start_new_slice()  argument
792 	tg->bytes_disp[rw] = 0;  in throtl_start_new_slice()
793 	tg->io_disp[rw] = 0;  in throtl_start_new_slice()
794 	tg->slice_start[rw] = jiffies;  in throtl_start_new_slice()
795 	tg->slice_end[rw] = jiffies + tg->td->throtl_slice;  in throtl_start_new_slice()
798 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice()
799 		   tg->slice_end[rw], jiffies);  in throtl_start_new_slice()
802 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,  in throtl_set_slice_end()  argument
805 	tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);  in throtl_set_slice_end()
808 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,  in throtl_extend_slice()  argument
811 	tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);  in throtl_extend_slice()
814 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_extend_slice()
815 		   tg->slice_end[rw], jiffies);  in throtl_extend_slice()
819 static bool throtl_slice_used(struct throtl_grp *tg, bool rw)  in throtl_slice_used()  argument
821 	if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))  in throtl_slice_used()
828 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)  in throtl_trim_slice()  argument
833 	BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw]));  in throtl_trim_slice()
840 	if (throtl_slice_used(tg, rw))  in throtl_trim_slice()
851 	throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice);  in throtl_trim_slice()
853 	time_elapsed = jiffies - tg->slice_start[rw];  in throtl_trim_slice()
859 	tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices;  in throtl_trim_slice()
863 	io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices) /  in throtl_trim_slice()
869 	if (tg->bytes_disp[rw] >= bytes_trim)  in throtl_trim_slice()
870 		tg->bytes_disp[rw] -= bytes_trim;  in throtl_trim_slice()
872 		tg->bytes_disp[rw] = 0;  in throtl_trim_slice()
874 	if (tg->io_disp[rw] >= io_trim)  in throtl_trim_slice()
875 		tg->io_disp[rw] -= io_trim;  in throtl_trim_slice()
877 		tg->io_disp[rw] = 0;  in throtl_trim_slice()
879 	tg->slice_start[rw] += nr_slices * tg->td->throtl_slice;  in throtl_trim_slice()
883 		   rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim,  in throtl_trim_slice()
884 		   tg->slice_start[rw], tg->slice_end[rw], jiffies);  in throtl_trim_slice()
890 	bool rw = bio_data_dir(bio);  in tg_with_in_iops_limit()  local
895 	jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];  in tg_with_in_iops_limit()
910 	tmp = (u64)tg_iops_limit(tg, rw) * jiffy_elapsed_rnd;  in tg_with_in_iops_limit()
918 	if (tg->io_disp[rw] + 1 <= io_allowed) {  in tg_with_in_iops_limit()
935 	bool rw = bio_data_dir(bio);  in tg_with_in_bps_limit()  local
940 	jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];  in tg_with_in_bps_limit()
948 	tmp = tg_bps_limit(tg, rw) * jiffy_elapsed_rnd;  in tg_with_in_bps_limit()
952 	if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) {  in tg_with_in_bps_limit()
959 	extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed;  in tg_with_in_bps_limit()
960 	jiffy_wait = div64_u64(extra_bytes * HZ, tg_bps_limit(tg, rw));  in tg_with_in_bps_limit()
982 	bool rw = bio_data_dir(bio);  in tg_may_dispatch()  local
991 	BUG_ON(tg->service_queue.nr_queued[rw] &&  in tg_may_dispatch()
992 	       bio != throtl_peek_queued(&tg->service_queue.queued[rw]));  in tg_may_dispatch()
995 	if (tg_bps_limit(tg, rw) == U64_MAX &&  in tg_may_dispatch()
996 	    tg_iops_limit(tg, rw) == UINT_MAX) {  in tg_may_dispatch()
1009 	if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw]))  in tg_may_dispatch()
1010 		throtl_start_new_slice(tg, rw);  in tg_may_dispatch()
1012 		if (time_before(tg->slice_end[rw],  in tg_may_dispatch()
1014 			throtl_extend_slice(tg, rw,  in tg_may_dispatch()
1030 	if (time_before(tg->slice_end[rw], jiffies + max_wait))  in tg_may_dispatch()
1031 		throtl_extend_slice(tg, rw, jiffies + max_wait);  in tg_may_dispatch()
1038 	bool rw = bio_data_dir(bio);  in throtl_charge_bio()  local
1042 	tg->bytes_disp[rw] += bio_size;  in throtl_charge_bio()
1043 	tg->io_disp[rw]++;  in throtl_charge_bio()
1044 	tg->last_bytes_disp[rw] += bio_size;  in throtl_charge_bio()
1045 	tg->last_io_disp[rw]++;  in throtl_charge_bio()
1070 	bool rw = bio_data_dir(bio);  in throtl_add_bio_tg()  local
1073 		qn = &tg->qnode_on_self[rw];  in throtl_add_bio_tg()
1081 	if (!sq->nr_queued[rw])  in throtl_add_bio_tg()
1084 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
1086 	sq->nr_queued[rw]++;  in throtl_add_bio_tg()
1117 					struct throtl_grp *parent_tg, bool rw)  in start_parent_slice_with_credit()  argument
1119 	if (throtl_slice_used(parent_tg, rw)) {  in start_parent_slice_with_credit()
1120 		throtl_start_new_slice_with_credit(parent_tg, rw,  in start_parent_slice_with_credit()
1121 				child_tg->slice_start[rw]);  in start_parent_slice_with_credit()
1126 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)  in tg_dispatch_one_bio()  argument
1140 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
1141 	sq->nr_queued[rw]--;  in tg_dispatch_one_bio()
1153 		throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg);  in tg_dispatch_one_bio()
1154 		start_parent_slice_with_credit(tg, parent_tg, rw);  in tg_dispatch_one_bio()
1156 		throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw],  in tg_dispatch_one_bio()
1157 				     &parent_sq->queued[rw]);  in tg_dispatch_one_bio()
1158 		BUG_ON(tg->td->nr_queued[rw] <= 0);  in tg_dispatch_one_bio()
1159 		tg->td->nr_queued[rw]--;  in tg_dispatch_one_bio()
1162 	throtl_trim_slice(tg, rw);  in tg_dispatch_one_bio()
1324 	int rw;  in blk_throtl_dispatch_work_fn()  local
1329 	for (rw = READ; rw <= WRITE; rw++)  in blk_throtl_dispatch_work_fn()
1330 		while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL)))  in blk_throtl_dispatch_work_fn()
2050 	int i, cpu, rw;  in throtl_update_latency_buckets()  local
2061 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2063 			struct latency_bucket *tmp = &td->tmp_buckets[rw][i];  in throtl_update_latency_buckets()
2069 				bucket = per_cpu_ptr(td->latency_buckets[rw],  in throtl_update_latency_buckets()
2080 				latency[rw] = tmp->total_latency;  in throtl_update_latency_buckets()
2084 				latency[rw] /= samples;  in throtl_update_latency_buckets()
2085 				if (latency[rw] == 0)  in throtl_update_latency_buckets()
2087 				avg_latency[rw][i].latency = latency[rw];  in throtl_update_latency_buckets()
2092 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2094 			if (!avg_latency[rw][i].latency) {  in throtl_update_latency_buckets()
2095 				if (td->avg_buckets[rw][i].latency < last_latency[rw])  in throtl_update_latency_buckets()
2096 					td->avg_buckets[rw][i].latency =  in throtl_update_latency_buckets()
2097 						last_latency[rw];  in throtl_update_latency_buckets()
2101 			if (!td->avg_buckets[rw][i].valid)  in throtl_update_latency_buckets()
2102 				latency[rw] = avg_latency[rw][i].latency;  in throtl_update_latency_buckets()
2104 				latency[rw] = (td->avg_buckets[rw][i].latency * 7 +  in throtl_update_latency_buckets()
2105 					avg_latency[rw][i].latency) >> 3;  in throtl_update_latency_buckets()
2107 			td->avg_buckets[rw][i].latency = max(latency[rw],  in throtl_update_latency_buckets()
2108 				last_latency[rw]);  in throtl_update_latency_buckets()
2109 			td->avg_buckets[rw][i].valid = true;  in throtl_update_latency_buckets()
2110 			last_latency[rw] = td->avg_buckets[rw][i].latency;  in throtl_update_latency_buckets()
2145 	bool rw = bio_data_dir(bio);  in blk_throtl_bio()  local
2152 	if (bio_flagged(bio, BIO_THROTTLED) || !tg->has_rules[rw])  in blk_throtl_bio()
2169 		if (tg->last_low_overflow_time[rw] == 0)  in blk_throtl_bio()
2170 			tg->last_low_overflow_time[rw] = jiffies;  in blk_throtl_bio()
2174 		if (sq->nr_queued[rw])  in blk_throtl_bio()
2179 			tg->last_low_overflow_time[rw] = jiffies;  in blk_throtl_bio()
2201 		throtl_trim_slice(tg, rw);  in blk_throtl_bio()
2208 		qn = &tg->qnode_on_parent[rw];  in blk_throtl_bio()
2217 		   rw == READ ? 'R' : 'W',  in blk_throtl_bio()
2218 		   tg->bytes_disp[rw], bio->bi_iter.bi_size,  in blk_throtl_bio()
2219 		   tg_bps_limit(tg, rw),  in blk_throtl_bio()
2220 		   tg->io_disp[rw], tg_iops_limit(tg, rw),  in blk_throtl_bio()
2223 	tg->last_low_overflow_time[rw] = jiffies;  in blk_throtl_bio()
2225 	td->nr_queued[rw]++;  in blk_throtl_bio()
2288 	int rw = bio_data_dir(bio);  in blk_throtl_bio_endio()  local
2314 		threshold = tg->td->avg_buckets[rw][bucket].latency +  in blk_throtl_bio_endio()
2368 	int rw;  in blk_throtl_drain()  local
2389 	for (rw = READ; rw <= WRITE; rw++)  in blk_throtl_drain()
2390 		while ((bio = throtl_pop_queued(&td->service_queue.queued[rw],  in blk_throtl_drain()