147 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)  in tg_bps_limit()  argument
157 	ret = tg->bps[rw][td->limit_index];  in tg_bps_limit()
161 		    tg->iops[rw][td->limit_index])  in tg_bps_limit()
167 	if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] &&  in tg_bps_limit()
168 	    tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) {  in tg_bps_limit()
171 		adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td);  in tg_bps_limit()
172 		ret = min(tg->bps[rw][LIMIT_MAX], adjusted);  in tg_bps_limit()
177 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)  in tg_iops_limit()  argument
187 	ret = tg->iops[rw][td->limit_index];  in tg_iops_limit()
191 		    tg->bps[rw][td->limit_index])  in tg_iops_limit()
197 	if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] &&  in tg_iops_limit()
198 	    tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) {  in tg_iops_limit()
201 		adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td);  in tg_iops_limit()
204 		ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted);  in tg_iops_limit()
342 	int rw;  in throtl_pd_alloc()  local
356 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_pd_alloc()
357 		throtl_qnode_init(&tg->qnode_on_self[rw], tg);  in throtl_pd_alloc()
358 		throtl_qnode_init(&tg->qnode_on_parent[rw], tg);  in throtl_pd_alloc()
422 	int rw;  in tg_update_has_rules()  local
424 	for (rw = READ; rw <= WRITE; rw++) {  in tg_update_has_rules()
425 		tg->has_rules_iops[rw] =  in tg_update_has_rules()
426 			(parent_tg && parent_tg->has_rules_iops[rw]) ||  in tg_update_has_rules()
428 			  tg_iops_limit(tg, rw) != UINT_MAX);  in tg_update_has_rules()
429 		tg->has_rules_bps[rw] =  in tg_update_has_rules()
430 			(parent_tg && parent_tg->has_rules_bps[rw]) ||  in tg_update_has_rules()
432 			 (tg_bps_limit(tg, rw) != U64_MAX));  in tg_update_has_rules()
634 		bool rw, unsigned long start)  in throtl_start_new_slice_with_credit()  argument
636 	tg->bytes_disp[rw] = 0;  in throtl_start_new_slice_with_credit()
637 	tg->io_disp[rw] = 0;  in throtl_start_new_slice_with_credit()
638 	tg->carryover_bytes[rw] = 0;  in throtl_start_new_slice_with_credit()
639 	tg->carryover_ios[rw] = 0;  in throtl_start_new_slice_with_credit()
647 	if (time_after(start, tg->slice_start[rw]))  in throtl_start_new_slice_with_credit()
648 		tg->slice_start[rw] = start;  in throtl_start_new_slice_with_credit()
650 	tg->slice_end[rw] = jiffies + tg->td->throtl_slice;  in throtl_start_new_slice_with_credit()
653 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice_with_credit()
654 		   tg->slice_end[rw], jiffies);  in throtl_start_new_slice_with_credit()
657 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw,  in throtl_start_new_slice()  argument
660 	tg->bytes_disp[rw] = 0;  in throtl_start_new_slice()
661 	tg->io_disp[rw] = 0;  in throtl_start_new_slice()
662 	tg->slice_start[rw] = jiffies;  in throtl_start_new_slice()
663 	tg->slice_end[rw] = jiffies + tg->td->throtl_slice;  in throtl_start_new_slice()
665 		tg->carryover_bytes[rw] = 0;  in throtl_start_new_slice()
666 		tg->carryover_ios[rw] = 0;  in throtl_start_new_slice()
671 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_start_new_slice()
672 		   tg->slice_end[rw], jiffies);  in throtl_start_new_slice()
675 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,  in throtl_set_slice_end()  argument
678 	tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);  in throtl_set_slice_end()
681 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,  in throtl_extend_slice()  argument
684 	throtl_set_slice_end(tg, rw, jiffy_end);  in throtl_extend_slice()
687 		   rw == READ ? 'R' : 'W', tg->slice_start[rw],  in throtl_extend_slice()
688 		   tg->slice_end[rw], jiffies);  in throtl_extend_slice()
692 static bool throtl_slice_used(struct throtl_grp *tg, bool rw)  in throtl_slice_used()  argument
694 	if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))  in throtl_slice_used()
736 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)  in throtl_trim_slice()  argument
742 	BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw]));  in throtl_trim_slice()
749 	if (throtl_slice_used(tg, rw))  in throtl_trim_slice()
760 	throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice);  in throtl_trim_slice()
762 	time_elapsed = rounddown(jiffies - tg->slice_start[rw],  in throtl_trim_slice()
767 	bytes_trim = calculate_bytes_allowed(tg_bps_limit(tg, rw),  in throtl_trim_slice()
769 		     tg->carryover_bytes[rw];  in throtl_trim_slice()
770 	io_trim = calculate_io_allowed(tg_iops_limit(tg, rw), time_elapsed) +  in throtl_trim_slice()
771 		  tg->carryover_ios[rw];  in throtl_trim_slice()
775 	tg->carryover_bytes[rw] = 0;  in throtl_trim_slice()
776 	if ((long long)tg->bytes_disp[rw] >= bytes_trim)  in throtl_trim_slice()
777 		tg->bytes_disp[rw] -= bytes_trim;  in throtl_trim_slice()
779 		tg->bytes_disp[rw] = 0;  in throtl_trim_slice()
781 	tg->carryover_ios[rw] = 0;  in throtl_trim_slice()
782 	if ((int)tg->io_disp[rw] >= io_trim)  in throtl_trim_slice()
783 		tg->io_disp[rw] -= io_trim;  in throtl_trim_slice()
785 		tg->io_disp[rw] = 0;  in throtl_trim_slice()
787 	tg->slice_start[rw] += time_elapsed;  in throtl_trim_slice()
791 		   rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice,  in throtl_trim_slice()
792 		   bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw],  in throtl_trim_slice()
796 static void __tg_update_carryover(struct throtl_grp *tg, bool rw)  in __tg_update_carryover()  argument
798 	unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw];  in __tg_update_carryover()
799 	u64 bps_limit = tg_bps_limit(tg, rw);  in __tg_update_carryover()
800 	u32 iops_limit = tg_iops_limit(tg, rw);  in __tg_update_carryover()
809 		tg->carryover_bytes[rw] +=  in __tg_update_carryover()
811 			tg->bytes_disp[rw];  in __tg_update_carryover()
813 		tg->carryover_ios[rw] +=  in __tg_update_carryover()
815 			tg->io_disp[rw];  in __tg_update_carryover()
834 	bool rw = bio_data_dir(bio);  in tg_within_iops_limit()  local
842 	jiffy_elapsed = jiffies - tg->slice_start[rw];  in tg_within_iops_limit()
847 		     tg->carryover_ios[rw];  in tg_within_iops_limit()
848 	if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed)  in tg_within_iops_limit()
859 	bool rw = bio_data_dir(bio);  in tg_within_bps_limit()  local
870 	jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];  in tg_within_bps_limit()
878 			tg->carryover_bytes[rw];  in tg_within_bps_limit()
879 	if (bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed)  in tg_within_bps_limit()
883 	extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed;  in tg_within_bps_limit()
904 	bool rw = bio_data_dir(bio);  in tg_may_dispatch()  local
906 	u64 bps_limit = tg_bps_limit(tg, rw);  in tg_may_dispatch()
907 	u32 iops_limit = tg_iops_limit(tg, rw);  in tg_may_dispatch()
915 	BUG_ON(tg->service_queue.nr_queued[rw] &&  in tg_may_dispatch()
916 	       bio != throtl_peek_queued(&tg->service_queue.queued[rw]));  in tg_may_dispatch()
933 	if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw]))  in tg_may_dispatch()
934 		throtl_start_new_slice(tg, rw, true);  in tg_may_dispatch()
936 		if (time_before(tg->slice_end[rw],  in tg_may_dispatch()
938 			throtl_extend_slice(tg, rw,  in tg_may_dispatch()
955 	if (time_before(tg->slice_end[rw], jiffies + max_wait))  in tg_may_dispatch()
956 		throtl_extend_slice(tg, rw, jiffies + max_wait);  in tg_may_dispatch()
963 	bool rw = bio_data_dir(bio);  in throtl_charge_bio()  local
968 		tg->bytes_disp[rw] += bio_size;  in throtl_charge_bio()
969 		tg->last_bytes_disp[rw] += bio_size;  in throtl_charge_bio()
972 	tg->io_disp[rw]++;  in throtl_charge_bio()
973 	tg->last_io_disp[rw]++;  in throtl_charge_bio()
989 	bool rw = bio_data_dir(bio);  in throtl_add_bio_tg()  local
992 		qn = &tg->qnode_on_self[rw];  in throtl_add_bio_tg()
1000 	if (!sq->nr_queued[rw])  in throtl_add_bio_tg()
1003 	throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);  in throtl_add_bio_tg()
1005 	sq->nr_queued[rw]++;  in throtl_add_bio_tg()
1036 					struct throtl_grp *parent_tg, bool rw)  in start_parent_slice_with_credit()  argument
1038 	if (throtl_slice_used(parent_tg, rw)) {  in start_parent_slice_with_credit()
1039 		throtl_start_new_slice_with_credit(parent_tg, rw,  in start_parent_slice_with_credit()
1040 				child_tg->slice_start[rw]);  in start_parent_slice_with_credit()
1045 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)  in tg_dispatch_one_bio()  argument
1059 	bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);  in tg_dispatch_one_bio()
1060 	sq->nr_queued[rw]--;  in tg_dispatch_one_bio()
1072 		throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg);  in tg_dispatch_one_bio()
1073 		start_parent_slice_with_credit(tg, parent_tg, rw);  in tg_dispatch_one_bio()
1076 		throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw],  in tg_dispatch_one_bio()
1077 				     &parent_sq->queued[rw]);  in tg_dispatch_one_bio()
1078 		BUG_ON(tg->td->nr_queued[rw] <= 0);  in tg_dispatch_one_bio()
1079 		tg->td->nr_queued[rw]--;  in tg_dispatch_one_bio()
1082 	throtl_trim_slice(tg, rw);  in tg_dispatch_one_bio()
1258 	int rw;  in blk_throtl_dispatch_work_fn()  local
1263 	for (rw = READ; rw <= WRITE; rw++)  in blk_throtl_dispatch_work_fn()
1264 		while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL)))  in blk_throtl_dispatch_work_fn()
1827 static bool throtl_low_limit_reached(struct throtl_grp *tg, int rw)  in throtl_low_limit_reached()  argument
1830 	bool limit = tg->bps[rw][LIMIT_LOW] || tg->iops[rw][LIMIT_LOW];  in throtl_low_limit_reached()
1838 	return !limit || sq->nr_queued[rw];  in throtl_low_limit_reached()
2073 	int i, cpu, rw;  in throtl_update_latency_buckets()  local
2084 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2086 			struct latency_bucket *tmp = &td->tmp_buckets[rw][i];  in throtl_update_latency_buckets()
2092 				bucket = per_cpu_ptr(td->latency_buckets[rw],  in throtl_update_latency_buckets()
2103 				latency[rw] = tmp->total_latency;  in throtl_update_latency_buckets()
2107 				latency[rw] /= samples;  in throtl_update_latency_buckets()
2108 				if (latency[rw] == 0)  in throtl_update_latency_buckets()
2110 				avg_latency[rw][i].latency = latency[rw];  in throtl_update_latency_buckets()
2115 	for (rw = READ; rw <= WRITE; rw++) {  in throtl_update_latency_buckets()
2117 			if (!avg_latency[rw][i].latency) {  in throtl_update_latency_buckets()
2118 				if (td->avg_buckets[rw][i].latency < last_latency[rw])  in throtl_update_latency_buckets()
2119 					td->avg_buckets[rw][i].latency =  in throtl_update_latency_buckets()
2120 						last_latency[rw];  in throtl_update_latency_buckets()
2124 			if (!td->avg_buckets[rw][i].valid)  in throtl_update_latency_buckets()
2125 				latency[rw] = avg_latency[rw][i].latency;  in throtl_update_latency_buckets()
2127 				latency[rw] = (td->avg_buckets[rw][i].latency * 7 +  in throtl_update_latency_buckets()
2128 					avg_latency[rw][i].latency) >> 3;  in throtl_update_latency_buckets()
2130 			td->avg_buckets[rw][i].latency = max(latency[rw],  in throtl_update_latency_buckets()
2131 				last_latency[rw]);  in throtl_update_latency_buckets()
2132 			td->avg_buckets[rw][i].valid = true;  in throtl_update_latency_buckets()
2133 			last_latency[rw] = td->avg_buckets[rw][i].latency;  in throtl_update_latency_buckets()
2181 	bool rw = bio_data_dir(bio);  in __blk_throtl_bio()  local
2197 		if (tg->last_low_overflow_time[rw] == 0)  in __blk_throtl_bio()
2198 			tg->last_low_overflow_time[rw] = jiffies;  in __blk_throtl_bio()
2202 		if (sq->nr_queued[rw])  in __blk_throtl_bio()
2207 			tg->last_low_overflow_time[rw] = jiffies;  in __blk_throtl_bio()
2229 		throtl_trim_slice(tg, rw);  in __blk_throtl_bio()
2236 		qn = &tg->qnode_on_parent[rw];  in __blk_throtl_bio()
2247 		   rw == READ ? 'R' : 'W',  in __blk_throtl_bio()
2248 		   tg->bytes_disp[rw], bio->bi_iter.bi_size,  in __blk_throtl_bio()
2249 		   tg_bps_limit(tg, rw),  in __blk_throtl_bio()
2250 		   tg->io_disp[rw], tg_iops_limit(tg, rw),  in __blk_throtl_bio()
2253 	tg->last_low_overflow_time[rw] = jiffies;  in __blk_throtl_bio()
2255 	td->nr_queued[rw]++;  in __blk_throtl_bio()
2285 	const bool rw = op_is_write(op);  in throtl_track_latency()  local
2296 	latency = get_cpu_ptr(td->latency_buckets[rw]);  in throtl_track_latency()
2299 	put_cpu_ptr(td->latency_buckets[rw]);  in throtl_track_latency()
2319 	int rw = bio_data_dir(bio);  in blk_throtl_bio_endio()  local
2347 		threshold = tg->td->avg_buckets[rw][bucket].latency +  in blk_throtl_bio_endio()