187 static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat)  in lat_to_blkg()  argument
189 	return pd_to_blkg(&iolat->pd);  in lat_to_blkg()
192 static inline void latency_stat_init(struct iolatency_grp *iolat,  in latency_stat_init()  argument
195 	if (iolat->ssd) {  in latency_stat_init()
202 static inline void latency_stat_sum(struct iolatency_grp *iolat,  in latency_stat_sum()  argument
206 	if (iolat->ssd) {  in latency_stat_sum()
213 static inline void latency_stat_record_time(struct iolatency_grp *iolat,  in latency_stat_record_time()  argument
216 	struct latency_stat *stat = get_cpu_ptr(iolat->stats);  in latency_stat_record_time()
217 	if (iolat->ssd) {  in latency_stat_record_time()
218 		if (req_time >= iolat->min_lat_nsec)  in latency_stat_record_time()
226 static inline bool latency_sum_ok(struct iolatency_grp *iolat,  in latency_sum_ok()  argument
229 	if (iolat->ssd) {  in latency_sum_ok()
234 	return stat->rqs.mean <= iolat->min_lat_nsec;  in latency_sum_ok()
237 static inline u64 latency_stat_samples(struct iolatency_grp *iolat,  in latency_stat_samples()  argument
240 	if (iolat->ssd)  in latency_stat_samples()
245 static inline void iolat_update_total_lat_avg(struct iolatency_grp *iolat,  in iolat_update_total_lat_avg()  argument
250 	if (iolat->ssd)  in iolat_update_total_lat_avg()
261 			div64_u64(iolat->cur_win_nsec,  in iolat_update_total_lat_avg()
263 	iolat->lat_avg = calc_load(iolat->lat_avg,  in iolat_update_total_lat_avg()
276 	struct iolatency_grp *iolat = private_data;  in iolat_acquire_inflight()  local
277 	return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);  in iolat_acquire_inflight()
281 				       struct iolatency_grp *iolat,  in __blkcg_iolatency_throttle()  argument
285 	struct rq_wait *rqw = &iolat->rq_wait;  in __blkcg_iolatency_throttle()
286 	unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay);  in __blkcg_iolatency_throttle()
303 	rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb);  in __blkcg_iolatency_throttle()
365 static void scale_change(struct iolatency_grp *iolat, bool up)  in scale_change()  argument
367 	unsigned long qd = iolat->blkiolat->rqos.q->nr_requests;  in scale_change()
369 	unsigned long old = iolat->rq_depth.max_depth;  in scale_change()
375 		if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat)))  in scale_change()
381 			iolat->rq_depth.max_depth = old;  in scale_change()
382 			wake_up_all(&iolat->rq_wait.wait);  in scale_change()
386 		iolat->rq_depth.max_depth = max(old, 1UL);  in scale_change()
391 static void check_scale_change(struct iolatency_grp *iolat)  in check_scale_change()  argument
396 	unsigned int our_cookie = atomic_read(&iolat->scale_cookie);  in check_scale_change()
401 	if (lat_to_blkg(iolat)->parent == NULL)  in check_scale_change()
404 	parent = blkg_to_lat(lat_to_blkg(iolat)->parent);  in check_scale_change()
419 	old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie);  in check_scale_change()
425 	if (direction < 0 && iolat->min_lat_nsec) {  in check_scale_change()
428 		if (!scale_lat || iolat->min_lat_nsec <= scale_lat)  in check_scale_change()
439 		if (iolat->nr_samples <= samples_thresh)  in check_scale_change()
444 	if (iolat->rq_depth.max_depth == 1 && direction < 0) {  in check_scale_change()
445 		blkcg_use_delay(lat_to_blkg(iolat));  in check_scale_change()
451 		blkcg_clear_delay(lat_to_blkg(iolat));  in check_scale_change()
452 		iolat->rq_depth.max_depth = UINT_MAX;  in check_scale_change()
453 		wake_up_all(&iolat->rq_wait.wait);  in check_scale_change()
457 	scale_change(iolat, direction > 0);  in check_scale_change()
470 		struct iolatency_grp *iolat = blkg_to_lat(blkg);  in blkcg_iolatency_throttle()  local
471 		if (!iolat) {  in blkcg_iolatency_throttle()
476 		check_scale_change(iolat);  in blkcg_iolatency_throttle()
477 		__blkcg_iolatency_throttle(rqos, iolat, issue_as_root,  in blkcg_iolatency_throttle()
485 static void iolatency_record_time(struct iolatency_grp *iolat,  in iolatency_record_time()  argument
507 	if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) {  in iolatency_record_time()
508 		u64 sub = iolat->min_lat_nsec;  in iolatency_record_time()
510 			blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time);  in iolatency_record_time()
514 	latency_stat_record_time(iolat, req_time);  in iolatency_record_time()
520 static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)  in iolatency_check_latencies()  argument
522 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_check_latencies()
529 	latency_stat_init(iolat, &stat);  in iolatency_check_latencies()
533 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_check_latencies()
534 		latency_stat_sum(iolat, &stat, s);  in iolatency_check_latencies()
535 		latency_stat_init(iolat, s);  in iolatency_check_latencies()
545 	iolat_update_total_lat_avg(iolat, &stat);  in iolatency_check_latencies()
548 	if (latency_sum_ok(iolat, &stat) &&  in iolatency_check_latencies()
555 	latency_stat_sum(iolat, &iolat->cur_stat, &stat);  in iolatency_check_latencies()
556 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
557 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
558 	iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
564 	if (latency_sum_ok(iolat, &iolat->cur_stat) &&  in iolatency_check_latencies()
565 	    latency_sum_ok(iolat, &stat)) {  in iolatency_check_latencies()
566 		if (latency_stat_samples(iolat, &iolat->cur_stat) <  in iolatency_check_latencies()
569 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
571 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
574 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
577 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
578 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
579 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
581 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
583 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
592 	struct iolatency_grp *iolat;  in blkcg_iolatency_done_bio()  local
603 	iolat = blkg_to_lat(bio->bi_blkg);  in blkcg_iolatency_done_bio()
604 	if (!iolat)  in blkcg_iolatency_done_bio()
607 	enabled = blk_iolatency_enabled(iolat->blkiolat);  in blkcg_iolatency_done_bio()
612 		iolat = blkg_to_lat(blkg);  in blkcg_iolatency_done_bio()
613 		if (!iolat) {  in blkcg_iolatency_done_bio()
617 		rqw = &iolat->rq_wait;  in blkcg_iolatency_done_bio()
625 		if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) {  in blkcg_iolatency_done_bio()
626 			iolatency_record_time(iolat, &bio->bi_issue, now,  in blkcg_iolatency_done_bio()
628 			window_start = atomic64_read(&iolat->window_start);  in blkcg_iolatency_done_bio()
630 			    (now - window_start) >= iolat->cur_win_nsec) {  in blkcg_iolatency_done_bio()
631 				if (atomic64_cmpxchg(&iolat->window_start,  in blkcg_iolatency_done_bio()
633 					iolatency_check_latencies(iolat, now);  in blkcg_iolatency_done_bio()
666 		struct iolatency_grp *iolat;  in blkiolatency_timer_fn()  local
678 		iolat = blkg_to_lat(blkg);  in blkiolatency_timer_fn()
679 		if (!iolat)  in blkiolatency_timer_fn()
682 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
697 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
752 	struct iolatency_grp *iolat = blkg_to_lat(blkg);  in iolatency_set_min_lat_nsec()  local
753 	u64 oldval = iolat->min_lat_nsec;  in iolatency_set_min_lat_nsec()
755 	iolat->min_lat_nsec = val;  in iolatency_set_min_lat_nsec()
756 	iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE);  in iolatency_set_min_lat_nsec()
757 	iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec,  in iolatency_set_min_lat_nsec()
772 		struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);  in iolatency_clear_scaling()  local
774 		if (!iolat)  in iolatency_clear_scaling()
777 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
793 	struct iolatency_grp *iolat;  in iolatency_set_limit()  local
804 	iolat = blkg_to_lat(ctx.blkg);  in iolatency_set_limit()
831 	oldval = iolat->min_lat_nsec;  in iolatency_set_limit()
839 	if (oldval != iolat->min_lat_nsec) {  in iolatency_set_limit()
870 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_prfill_limit()  local
873 	if (!dname || !iolat->min_lat_nsec)  in iolatency_prfill_limit()
876 		   dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC));  in iolatency_prfill_limit()
888 static size_t iolatency_ssd_stat(struct iolatency_grp *iolat, char *buf,  in iolatency_ssd_stat()  argument
894 	latency_stat_init(iolat, &stat);  in iolatency_ssd_stat()
898 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_ssd_stat()
899 		latency_stat_sum(iolat, &stat, s);  in iolatency_ssd_stat()
903 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_ssd_stat()
910 			 iolat->rq_depth.max_depth);  in iolatency_ssd_stat()
916 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_stat()  local
923 	if (iolat->ssd)  in iolatency_pd_stat()
924 		return iolatency_ssd_stat(iolat, buf, size);  in iolatency_pd_stat()
926 	avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC);  in iolatency_pd_stat()
927 	cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC);  in iolatency_pd_stat()
928 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_pd_stat()
933 			 iolat->rq_depth.max_depth, avg_lat, cur_win);  in iolatency_pd_stat()
941 	struct iolatency_grp *iolat;  in iolatency_pd_alloc()  local
943 	iolat = kzalloc_node(sizeof(*iolat), gfp, q->node);  in iolatency_pd_alloc()
944 	if (!iolat)  in iolatency_pd_alloc()
946 	iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat),  in iolatency_pd_alloc()
948 	if (!iolat->stats) {  in iolatency_pd_alloc()
949 		kfree(iolat);  in iolatency_pd_alloc()
952 	return &iolat->pd;  in iolatency_pd_alloc()
957 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_init()  local
958 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_init()
965 		iolat->ssd = true;  in iolatency_pd_init()
967 		iolat->ssd = false;  in iolatency_pd_init()
971 		stat = per_cpu_ptr(iolat->stats, cpu);  in iolatency_pd_init()
972 		latency_stat_init(iolat, stat);  in iolatency_pd_init()
975 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_pd_init()
976 	rq_wait_init(&iolat->rq_wait);  in iolatency_pd_init()
977 	spin_lock_init(&iolat->child_lat.lock);  in iolatency_pd_init()
978 	iolat->rq_depth.queue_depth = blkg->q->nr_requests;  in iolatency_pd_init()
979 	iolat->rq_depth.max_depth = UINT_MAX;  in iolatency_pd_init()
980 	iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth;  in iolatency_pd_init()
981 	iolat->blkiolat = blkiolat;  in iolatency_pd_init()
982 	iolat->cur_win_nsec = 100 * NSEC_PER_MSEC;  in iolatency_pd_init()
983 	atomic64_set(&iolat->window_start, now);  in iolatency_pd_init()
991 		atomic_set(&iolat->scale_cookie,  in iolatency_pd_init()
994 		atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
997 	atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
1002 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_offline()  local
1003 	struct blkcg_gq *blkg = lat_to_blkg(iolat);  in iolatency_pd_offline()
1004 	struct blk_iolatency *blkiolat = iolat->blkiolat;  in iolatency_pd_offline()
1017 	struct iolatency_grp *iolat = pd_to_lat(pd);  in iolatency_pd_free()  local
1018 	free_percpu(iolat->stats);  in iolatency_pd_free()
1019 	kfree(iolat);  in iolatency_pd_free()