193 static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat)  in lat_to_blkg()  argument
195 	return pd_to_blkg(&iolat->pd);  in lat_to_blkg()
198 static inline void latency_stat_init(struct iolatency_grp *iolat,  in latency_stat_init()  argument
201 	if (iolat->ssd) {  in latency_stat_init()
208 static inline void latency_stat_sum(struct iolatency_grp *iolat,  in latency_stat_sum()  argument
212 	if (iolat->ssd) {  in latency_stat_sum()
219 static inline void latency_stat_record_time(struct iolatency_grp *iolat,  in latency_stat_record_time()  argument
222 	struct latency_stat *stat = get_cpu_ptr(iolat->stats);  in latency_stat_record_time()
223 	if (iolat->ssd) {  in latency_stat_record_time()
224 		if (req_time >= iolat->min_lat_nsec)  in latency_stat_record_time()
232 static inline bool latency_sum_ok(struct iolatency_grp *iolat,  in latency_sum_ok()  argument
235 	if (iolat->ssd) {  in latency_sum_ok()
240 	return stat->rqs.mean <= iolat->min_lat_nsec;  in latency_sum_ok()
243 static inline u64 latency_stat_samples(struct iolatency_grp *iolat,  in latency_stat_samples()  argument
246 	if (iolat->ssd)  in latency_stat_samples()
251 static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat,  in iolat_update_total_lat_avg()  argument
256 	if (iolat->ssd)  in iolat_update_total_lat_avg()
267 			div64_u64(iolat->cur_win_nsec,  in iolat_update_total_lat_avg()
269 	iolat->lat_avg = calc_load(iolat->lat_avg,  in iolat_update_total_lat_avg()
282 	struct iolatency_grp *iolat = private_data;  in iolat_acquire_inflight()  local
283 	return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);  in iolat_acquire_inflight()
287 				       struct iolatency_grp *iolat,  in __blkcg_iolatency_throttle()  argument
291 	struct rq_wait *rqw = &iolat->rq_wait;  in __blkcg_iolatency_throttle()
292 	unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay);  in __blkcg_iolatency_throttle()
309 	rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb);  in __blkcg_iolatency_throttle()
371 static void scale_change(struct iolatency_grp *iolat, bool up)  in scale_change()  argument
373 	unsigned long qd = iolat->blkiolat->rqos.q->nr_requests;  in scale_change()
375 	unsigned long old = iolat->rq_depth.max_depth;  in scale_change()
381 		if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat)))  in scale_change()
387 			iolat->rq_depth.max_depth = old;  in scale_change()
388 			wake_up_all(&iolat->rq_wait.wait);  in scale_change()
392 		iolat->rq_depth.max_depth = max(old, 1UL);  in scale_change()
397 static void check_scale_change(struct iolatency_grp *iolat)  in check_scale_change()  argument
402 	unsigned int our_cookie = atomic_read(&iolat->scale_cookie);  in check_scale_change()
406 	if (lat_to_blkg(iolat)->parent == NULL)  in check_scale_change()
409 	parent = blkg_to_lat(lat_to_blkg(iolat)->parent);  in check_scale_change()
424 	if (!atomic_try_cmpxchg(&iolat->scale_cookie, &our_cookie, cur_cookie)) {  in check_scale_change()
429 	if (direction < 0 && iolat->min_lat_nsec) {  in check_scale_change()
432 		if (!scale_lat || iolat->min_lat_nsec <= scale_lat)  in check_scale_change()
443 		if (iolat->nr_samples <= samples_thresh)  in check_scale_change()
448 	if (iolat->rq_depth.max_depth == 1 && direction < 0) {  in check_scale_change()
449 		blkcg_use_delay(lat_to_blkg(iolat));  in check_scale_change()
455 		blkcg_clear_delay(lat_to_blkg(iolat));  in check_scale_change()
456 		iolat->rq_depth.max_depth = UINT_MAX;  in check_scale_change()
457 		wake_up_all(&iolat->rq_wait.wait);  in check_scale_change()
461 	scale_change(iolat, direction > 0);  in check_scale_change()
474 		struct iolatency_grp *iolat = blkg_to_lat(blkg);  in blkcg_iolatency_throttle()  local
475 		if (!iolat) {  in blkcg_iolatency_throttle()
480 		check_scale_change(iolat);  in blkcg_iolatency_throttle()
481 		__blkcg_iolatency_throttle(rqos, iolat, issue_as_root,  in blkcg_iolatency_throttle()
489 static void iolatency_record_time(struct iolatency_grp *iolat,  in iolatency_record_time()  argument
511 	if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) {  in iolatency_record_time()
512 		u64 sub = iolat->min_lat_nsec;  in iolatency_record_time()
514 			blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time);  in iolatency_record_time()
518 	latency_stat_record_time(iolat, req_time);  in iolatency_record_time()
524 static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)  in iolatency_check_latencies()  argument
526 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_check_latencies()
533 	latency_stat_init(iolat, &stat);  in iolatency_check_latencies()
537 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_check_latencies()
538 		latency_stat_sum(iolat, &stat, s);  in iolatency_check_latencies()
539 		latency_stat_init(iolat, s);  in iolatency_check_latencies()
549 	iolat_update_total_lat_avg(iolat, &stat);  in iolatency_check_latencies()
552 	if (latency_sum_ok(iolat, &stat) &&  in iolatency_check_latencies()
559 	latency_stat_sum(iolat, &iolat->cur_stat, &stat);  in iolatency_check_latencies()
560 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
561 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
562 	iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
568 	if (latency_sum_ok(iolat, &iolat->cur_stat) &&  in iolatency_check_latencies()
569 	    latency_sum_ok(iolat, &stat)) {  in iolatency_check_latencies()
570 		if (latency_stat_samples(iolat, &iolat->cur_stat) <  in iolatency_check_latencies()
573 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
575 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
578 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
581 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
582 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
583 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
585 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
587 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
596 	struct iolatency_grp *iolat;  in blkcg_iolatency_done_bio()  local
606 	iolat = blkg_to_lat(bio->bi_blkg);  in blkcg_iolatency_done_bio()
607 	if (!iolat)  in blkcg_iolatency_done_bio()
610 	if (!iolat->blkiolat->enabled)  in blkcg_iolatency_done_bio()
615 		iolat = blkg_to_lat(blkg);  in blkcg_iolatency_done_bio()
616 		if (!iolat) {  in blkcg_iolatency_done_bio()
620 		rqw = &iolat->rq_wait;  in blkcg_iolatency_done_bio()
628 		if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) {  in blkcg_iolatency_done_bio()
629 			iolatency_record_time(iolat, &bio->bi_issue, now,  in blkcg_iolatency_done_bio()
631 			window_start = atomic64_read(&iolat->window_start);  in blkcg_iolatency_done_bio()
633 			    (now - window_start) >= iolat->cur_win_nsec) {  in blkcg_iolatency_done_bio()
634 				if (atomic64_try_cmpxchg(&iolat->window_start,  in blkcg_iolatency_done_bio()
636 					iolatency_check_latencies(iolat, now);  in blkcg_iolatency_done_bio()
670 		struct iolatency_grp *iolat;  in blkiolatency_timer_fn()  local
682 		iolat = blkg_to_lat(blkg);  in blkiolatency_timer_fn()
683 		if (!iolat)  in blkiolatency_timer_fn()
686 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
701 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
796 	struct iolatency_grp *iolat = blkg_to_lat(blkg);  in iolatency_set_min_lat_nsec()  local
797 	struct blk_iolatency *blkiolat = iolat->blkiolat;  in iolatency_set_min_lat_nsec()
798 	u64 oldval = iolat->min_lat_nsec;  in iolatency_set_min_lat_nsec()
800 	iolat->min_lat_nsec = val;  in iolatency_set_min_lat_nsec()
801 	iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE);  in iolatency_set_min_lat_nsec()
802 	iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec,  in iolatency_set_min_lat_nsec()
819 		struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);  in iolatency_clear_scaling()  local
821 		if (!iolat)  in iolatency_clear_scaling()
824 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
840 	struct iolatency_grp *iolat;  in iolatency_set_limit()  local
850 	iolat = blkg_to_lat(ctx.blkg);  in iolatency_set_limit()
877 	oldval = iolat->min_lat_nsec;  in iolatency_set_limit()
880 	if (oldval != iolat->min_lat_nsec)  in iolatency_set_limit()
891 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_prfill_limit()  local
894 	if (!dname || !iolat->min_lat_nsec)  in iolatency_prfill_limit()
897 		   dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC));  in iolatency_prfill_limit()
909 static void iolatency_ssd_stat(struct iolatency_grp *iolat, struct seq_file *s)  in iolatency_ssd_stat()  argument
914 	latency_stat_init(iolat, &stat);  in iolatency_ssd_stat()
918 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_ssd_stat()
919 		latency_stat_sum(iolat, &stat, s);  in iolatency_ssd_stat()
923 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_ssd_stat()
931 			iolat->rq_depth.max_depth);  in iolatency_ssd_stat()
936 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_stat()  local
943 	if (iolat->ssd)  in iolatency_pd_stat()
944 		return iolatency_ssd_stat(iolat, s);  in iolatency_pd_stat()
946 	avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC);  in iolatency_pd_stat()
947 	cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC);  in iolatency_pd_stat()
948 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_pd_stat()
953 			iolat->rq_depth.max_depth, avg_lat, cur_win);  in iolatency_pd_stat()
960 	struct iolatency_grp *iolat;  in iolatency_pd_alloc()  local
962 	iolat = kzalloc_node(sizeof(*iolat), gfp, q->node);  in iolatency_pd_alloc()
963 	if (!iolat)  in iolatency_pd_alloc()
965 	iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat),  in iolatency_pd_alloc()
967 	if (!iolat->stats) {  in iolatency_pd_alloc()
968 		kfree(iolat);  in iolatency_pd_alloc()
971 	return &iolat->pd;  in iolatency_pd_alloc()
976 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_init()  local
977 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_init()
984 		iolat->ssd = true;  in iolatency_pd_init()
986 		iolat->ssd = false;  in iolatency_pd_init()
990 		stat = per_cpu_ptr(iolat->stats, cpu);  in iolatency_pd_init()
991 		latency_stat_init(iolat, stat);  in iolatency_pd_init()
994 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_pd_init()
995 	rq_wait_init(&iolat->rq_wait);  in iolatency_pd_init()
996 	spin_lock_init(&iolat->child_lat.lock);  in iolatency_pd_init()
997 	iolat->rq_depth.queue_depth = blkg->q->nr_requests;  in iolatency_pd_init()
998 	iolat->rq_depth.max_depth = UINT_MAX;  in iolatency_pd_init()
999 	iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth;  in iolatency_pd_init()
1000 	iolat->blkiolat = blkiolat;  in iolatency_pd_init()
1001 	iolat->cur_win_nsec = 100 * NSEC_PER_MSEC;  in iolatency_pd_init()
1002 	atomic64_set(&iolat->window_start, now);  in iolatency_pd_init()
1010 		atomic_set(&iolat->scale_cookie,  in iolatency_pd_init()
1013 		atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
1016 	atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
1021 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_offline()  local
1022 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_offline()
1030 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_free()  local
1031 	free_percpu(iolat->stats);  in iolatency_pd_free()
1032 	kfree(iolat);  in iolatency_pd_free()