1 // SPDX-License-Identifier: GPL-2.0
3  * Block rq-qos base io controller
7  * - It's bio based, so the latency covers the whole block layer in addition to
9  * - We will throttle all IO that comes in here if we need to.
10  * - We use the mean latency over the 100ms window.  This is because writes can
13  * - By default there's no throttling, we set the queue_depth to UINT_MAX so
17  * The hierarchy works like the cpu controller does, we track the latency at
19  * queue depth.  This means that we only care about our latency targets at the
32  * an average latency of 5ms.  If it does then we will throttle the "slow"
44  * number of IO's we're allowed to have in flight.  This starts at (u64)-1 down
52  * down more then we induce a latency at userspace return.  We accumulate the
55  * total_time += min_lat_nsec - actual_io_completion
69 #include <linux/backing-dev.h>
76 #include <linux/blk-mq.h>
77 #include "blk-rq-qos.h"
78 #include "blk-stat.h"
79 #include "blk-cgroup.h"
92 	 * ->enabled is the master enable switch gating the throttling logic and
94 	 * tracked in ->enable_cnt, and ->enable is flipped on/off accordingly
95 	 * from ->enable_work with the request_queue frozen. For details, See
109 	spinlock_t lock;  member
114 	/* The latency that we missed. */
120 	/* The guy who actually changed the latency numbers. */
151 	/* total running average of our io latency. */
164  * These are the constants used to fake the fixed-point moving average
166  * (FIXED_1 (2048) - exp_factor) * new_sample into lat_avg.  The sampling
168  * latency such that 1/exp (decay rate) is [1 min, 2.5 min) when windows
174 				      (BLKIOLATENCY_NR_EXP_FACTORS - 1))
176 	2045, // exp(1/600) - 600 samples
177 	2039, // exp(1/240) - 240 samples
178 	2031, // exp(1/120) - 120 samples
179 	2023, // exp(1/80)  - 80 samples
180 	2014, // exp(1/60)  - 60 samples
195 	return pd_to_blkg(&iolat->pd);  in lat_to_blkg()
201 	if (iolat->ssd) {  in latency_stat_init()
202 		stat->ps.total = 0;  in latency_stat_init()
203 		stat->ps.missed = 0;  in latency_stat_init()
205 		blk_rq_stat_init(&stat->rqs);  in latency_stat_init()
212 	if (iolat->ssd) {  in latency_stat_sum()
213 		sum->ps.total += stat->ps.total;  in latency_stat_sum()
214 		sum->ps.missed += stat->ps.missed;  in latency_stat_sum()
216 		blk_rq_stat_sum(&sum->rqs, &stat->rqs);  in latency_stat_sum()
222 	struct latency_stat *stat = get_cpu_ptr(iolat->stats);  in latency_stat_record_time()
223 	if (iolat->ssd) {  in latency_stat_record_time()
224 		if (req_time >= iolat->min_lat_nsec)  in latency_stat_record_time()
225 			stat->ps.missed++;  in latency_stat_record_time()
226 		stat->ps.total++;  in latency_stat_record_time()
228 		blk_rq_stat_add(&stat->rqs, req_time);  in latency_stat_record_time()
235 	if (iolat->ssd) {  in latency_sum_ok()
236 		u64 thresh = div64_u64(stat->ps.total, 10);  in latency_sum_ok()
238 		return stat->ps.missed < thresh;  in latency_sum_ok()
240 	return stat->rqs.mean <= iolat->min_lat_nsec;  in latency_sum_ok()
246 	if (iolat->ssd)  in latency_stat_samples()
247 		return stat->ps.total;  in latency_stat_samples()
248 	return stat->rqs.nr_samples;  in latency_stat_samples()
256 	if (iolat->ssd)  in iolat_update_total_lat_avg()
261 	 * Because we are using this for IO time in ns, the values stored  in iolat_update_total_lat_avg()
266 	exp_idx = min_t(int, BLKIOLATENCY_NR_EXP_FACTORS - 1,  in iolat_update_total_lat_avg()
267 			div64_u64(iolat->cur_win_nsec,  in iolat_update_total_lat_avg()
269 	iolat->lat_avg = calc_load(iolat->lat_avg,  in iolat_update_total_lat_avg()
271 				   stat->rqs.mean);  in iolat_update_total_lat_avg()
276 	atomic_dec(&rqw->inflight);  in iolat_cleanup_cb()
277 	wake_up(&rqw->wait);  in iolat_cleanup_cb()
283 	return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);  in iolat_acquire_inflight()
291 	struct rq_wait *rqw = &iolat->rq_wait;  in __blkcg_iolatency_throttle()
292 	unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay);  in __blkcg_iolatency_throttle()
295 		blkcg_schedule_throttle(rqos->q->disk, use_memdelay);  in __blkcg_iolatency_throttle()
305 		atomic_inc(&rqw->inflight);  in __blkcg_iolatency_throttle()
333 	unsigned long qd = blkiolat->rqos.q->nr_requests;  in scale_cookie_change()
335 	unsigned long old = atomic_read(&lat_info->scale_cookie);  in scale_cookie_change()
340 		diff = DEFAULT_SCALE_COOKIE - old;  in scale_cookie_change()
344 			atomic_set(&lat_info->scale_cookie,  in scale_cookie_change()
347 			atomic_inc(&lat_info->scale_cookie);  in scale_cookie_change()
349 			atomic_add(scale, &lat_info->scale_cookie);  in scale_cookie_change()
359 				atomic_dec(&lat_info->scale_cookie);  in scale_cookie_change()
361 			atomic_sub(scale, &lat_info->scale_cookie);  in scale_cookie_change()
373 	unsigned long qd = iolat->blkiolat->rqos.q->nr_requests;  in scale_change()
375 	unsigned long old = iolat->rq_depth.max_depth;  in scale_change()
387 			iolat->rq_depth.max_depth = old;  in scale_change()
388 			wake_up_all(&iolat->rq_wait.wait);  in scale_change()
392 		iolat->rq_depth.max_depth = max(old, 1UL);  in scale_change()
402 	unsigned int our_cookie = atomic_read(&iolat->scale_cookie);  in check_scale_change()
406 	if (lat_to_blkg(iolat)->parent == NULL)  in check_scale_change()
409 	parent = blkg_to_lat(lat_to_blkg(iolat)->parent);  in check_scale_change()
413 	lat_info = &parent->child_lat;  in check_scale_change()
414 	cur_cookie = atomic_read(&lat_info->scale_cookie);  in check_scale_change()
415 	scale_lat = READ_ONCE(lat_info->scale_lat);  in check_scale_change()
418 		direction = -1;  in check_scale_change()
424 	if (!atomic_try_cmpxchg(&iolat->scale_cookie, &our_cookie, cur_cookie)) {  in check_scale_change()
429 	if (direction < 0 && iolat->min_lat_nsec) {  in check_scale_change()
432 		if (!scale_lat || iolat->min_lat_nsec <= scale_lat)  in check_scale_change()
441 		samples_thresh = lat_info->nr_samples * 5;  in check_scale_change()
443 		if (iolat->nr_samples <= samples_thresh)  in check_scale_change()
448 	if (iolat->rq_depth.max_depth == 1 && direction < 0) {  in check_scale_change()
456 		iolat->rq_depth.max_depth = UINT_MAX;  in check_scale_change()
457 		wake_up_all(&iolat->rq_wait.wait);  in check_scale_change()
467 	struct blkcg_gq *blkg = bio->bi_blkg;  in blkcg_iolatency_throttle()
470 	if (!blkiolat->enabled)  in blkcg_iolatency_throttle()
473 	while (blkg && blkg->parent) {  in blkcg_iolatency_throttle()
476 			blkg = blkg->parent;  in blkcg_iolatency_throttle()
482 				     (bio->bi_opf & REQ_SWAP) == REQ_SWAP);  in blkcg_iolatency_throttle()
483 		blkg = blkg->parent;  in blkcg_iolatency_throttle()
485 	if (!timer_pending(&blkiolat->timer))  in blkcg_iolatency_throttle()
486 		mod_timer(&blkiolat->timer, jiffies + HZ);  in blkcg_iolatency_throttle()
505 	req_time = now - start;  in iolatency_record_time()
508 	 * We don't want to count issue_as_root bio's in the cgroups latency  in iolatency_record_time()
511 	if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) {  in iolatency_record_time()
512 		u64 sub = iolat->min_lat_nsec;  in iolatency_record_time()
514 			blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time);  in iolatency_record_time()
537 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_check_latencies()
543 	parent = blkg_to_lat(blkg->parent);  in iolatency_check_latencies()
547 	lat_info = &parent->child_lat;  in iolatency_check_latencies()
553 	    atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE)  in iolatency_check_latencies()
557 	spin_lock_irqsave(&lat_info->lock, flags);  in iolatency_check_latencies()
559 	latency_stat_sum(iolat, &iolat->cur_stat, &stat);  in iolatency_check_latencies()
560 	lat_info->nr_samples -= iolat->nr_samples;  in iolatency_check_latencies()
561 	lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
562 	iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
564 	if ((lat_info->last_scale_event >= now ||  in iolatency_check_latencies()
565 	    now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME))  in iolatency_check_latencies()
568 	if (latency_sum_ok(iolat, &iolat->cur_stat) &&  in iolatency_check_latencies()
570 		if (latency_stat_samples(iolat, &iolat->cur_stat) <  in iolatency_check_latencies()
573 		if (lat_info->scale_grp == iolat) {  in iolatency_check_latencies()
574 			lat_info->last_scale_event = now;  in iolatency_check_latencies()
575 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in iolatency_check_latencies()
577 	} else if (lat_info->scale_lat == 0 ||  in iolatency_check_latencies()
578 		   lat_info->scale_lat >= iolat->min_lat_nsec) {  in iolatency_check_latencies()
579 		lat_info->last_scale_event = now;  in iolatency_check_latencies()
580 		if (!lat_info->scale_grp ||  in iolatency_check_latencies()
581 		    lat_info->scale_lat > iolat->min_lat_nsec) {  in iolatency_check_latencies()
582 			WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);  in iolatency_check_latencies()
583 			lat_info->scale_grp = iolat;  in iolatency_check_latencies()
585 		scale_cookie_change(iolat->blkiolat, lat_info, false);  in iolatency_check_latencies()
587 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_check_latencies()
589 	spin_unlock_irqrestore(&lat_info->lock, flags);  in iolatency_check_latencies()
602 	blkg = bio->bi_blkg;  in blkcg_iolatency_done_bio()
606 	iolat = blkg_to_lat(bio->bi_blkg);  in blkcg_iolatency_done_bio()
610 	if (!iolat->blkiolat->enabled)  in blkcg_iolatency_done_bio()
614 	while (blkg && blkg->parent) {  in blkcg_iolatency_done_bio()
617 			blkg = blkg->parent;  in blkcg_iolatency_done_bio()
620 		rqw = &iolat->rq_wait;  in blkcg_iolatency_done_bio()
622 		inflight = atomic_dec_return(&rqw->inflight);  in blkcg_iolatency_done_bio()
628 		if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) {  in blkcg_iolatency_done_bio()
629 			iolatency_record_time(iolat, &bio->bi_issue, now,  in blkcg_iolatency_done_bio()
631 			window_start = atomic64_read(&iolat->window_start);  in blkcg_iolatency_done_bio()
633 			    (now - window_start) >= iolat->cur_win_nsec) {  in blkcg_iolatency_done_bio()
634 				if (atomic64_try_cmpxchg(&iolat->window_start,  in blkcg_iolatency_done_bio()
639 		wake_up(&rqw->wait);  in blkcg_iolatency_done_bio()
640 		blkg = blkg->parent;  in blkcg_iolatency_done_bio()
648 	del_timer_sync(&blkiolat->timer);  in blkcg_iolatency_exit()
649 	flush_work(&blkiolat->enable_work);  in blkcg_iolatency_exit()
650 	blkcg_deactivate_policy(rqos->q, &blkcg_policy_iolatency);  in blkcg_iolatency_exit()
669 				     blkiolat->rqos.q->root_blkg) {  in blkiolatency_timer_fn()
686 		lat_info = &iolat->child_lat;  in blkiolatency_timer_fn()
687 		cookie = atomic_read(&lat_info->scale_cookie);  in blkiolatency_timer_fn()
692 		spin_lock_irqsave(&lat_info->lock, flags);  in blkiolatency_timer_fn()
693 		if (lat_info->last_scale_event >= now)  in blkiolatency_timer_fn()
700 		if (lat_info->scale_grp == NULL) {  in blkiolatency_timer_fn()
701 			scale_cookie_change(iolat->blkiolat, lat_info, true);  in blkiolatency_timer_fn()
710 		if (now - lat_info->last_scale_event >=  in blkiolatency_timer_fn()
712 			lat_info->scale_grp = NULL;  in blkiolatency_timer_fn()
714 		spin_unlock_irqrestore(&lat_info->lock, flags);  in blkiolatency_timer_fn()
722  * blkiolatency_enable_work_fn - Enable or disable iolatency on the device
725  * iolatency needs to keep track of the number of in-flight IOs per cgroup. This
728  * want to disable the in-flight tracking.
730  * We have to make sure that the counting is balanced - we don't want to leak
731  * the in-flight counts by disabling accounting in the completion path while IOs
733  * freezing the queue while flipping ->enabled. As this requires a sleepable
734  * context, ->enabled flipping is punted to this work function.
745 	 * ->enabled_cnt modification. Acting on the latest ->enable_cnt is  in blkiolatency_enable_work_fn()
748 	 * Also, we know @blkiolat is safe to access as ->enable_work is flushed  in blkiolatency_enable_work_fn()
751 	enabled = atomic_read(&blkiolat->enable_cnt);  in blkiolatency_enable_work_fn()
752 	if (enabled != blkiolat->enabled) {  in blkiolatency_enable_work_fn()
753 		blk_mq_freeze_queue(blkiolat->rqos.q);  in blkiolatency_enable_work_fn()
754 		blkiolat->enabled = enabled;  in blkiolatency_enable_work_fn()
755 		blk_mq_unfreeze_queue(blkiolat->rqos.q);  in blkiolatency_enable_work_fn()
761 	struct request_queue *q = disk->queue;  in blk_iolatency_init()
768 		return -ENOMEM;  in blk_iolatency_init()
770 	rqos = &blkiolat->rqos;  in blk_iolatency_init()
771 	rqos->id = RQ_QOS_LATENCY;  in blk_iolatency_init()
772 	rqos->ops = &blkcg_iolatency_ops;  in blk_iolatency_init()
773 	rqos->q = q;  in blk_iolatency_init()
782 	timer_setup(&blkiolat->timer, blkiolatency_timer_fn, 0);  in blk_iolatency_init()
783 	INIT_WORK(&blkiolat->enable_work, blkiolatency_enable_work_fn);  in blk_iolatency_init()
797 	struct blk_iolatency *blkiolat = iolat->blkiolat;  in iolatency_set_min_lat_nsec()
798 	u64 oldval = iolat->min_lat_nsec;  in iolatency_set_min_lat_nsec()
800 	iolat->min_lat_nsec = val;  in iolatency_set_min_lat_nsec()
801 	iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE);  in iolatency_set_min_lat_nsec()
802 	iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec,  in iolatency_set_min_lat_nsec()
806 		if (atomic_inc_return(&blkiolat->enable_cnt) == 1)  in iolatency_set_min_lat_nsec()
807 			schedule_work(&blkiolat->enable_work);  in iolatency_set_min_lat_nsec()
811 		if (atomic_dec_return(&blkiolat->enable_cnt) == 0)  in iolatency_set_min_lat_nsec()
812 			schedule_work(&blkiolat->enable_work);  in iolatency_set_min_lat_nsec()
818 	if (blkg->parent) {  in iolatency_clear_scaling()
819 		struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);  in iolatency_clear_scaling()
824 		lat_info = &iolat->child_lat;  in iolatency_clear_scaling()
825 		spin_lock(&lat_info->lock);  in iolatency_clear_scaling()
826 		atomic_set(&lat_info->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_clear_scaling()
827 		lat_info->last_scale_event = 0;  in iolatency_clear_scaling()
828 		lat_info->scale_grp = NULL;  in iolatency_clear_scaling()
829 		lat_info->scale_lat = 0;  in iolatency_clear_scaling()
830 		spin_unlock(&lat_info->lock);  in iolatency_clear_scaling()
853 	ret = -EINVAL;  in iolatency_set_limit()
877 	oldval = iolat->min_lat_nsec;  in iolatency_set_limit()
880 	if (oldval != iolat->min_lat_nsec)  in iolatency_set_limit()
892 	const char *dname = blkg_dev_name(pd->blkg);  in iolatency_prfill_limit()
894 	if (!dname || !iolat->min_lat_nsec)  in iolatency_prfill_limit()
897 		   dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC));  in iolatency_prfill_limit()
905 			  &blkcg_policy_iolatency, seq_cft(sf)->private, false);  in iolatency_print_limit()
918 		s = per_cpu_ptr(iolat->stats, cpu);  in iolatency_ssd_stat()
923 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_ssd_stat()
931 			iolat->rq_depth.max_depth);  in iolatency_ssd_stat()
943 	if (iolat->ssd)  in iolatency_pd_stat()
946 	avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC);  in iolatency_pd_stat()
947 	cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC);  in iolatency_pd_stat()
948 	if (iolat->rq_depth.max_depth == UINT_MAX)  in iolatency_pd_stat()
953 			iolat->rq_depth.max_depth, avg_lat, cur_win);  in iolatency_pd_stat()
962 	iolat = kzalloc_node(sizeof(*iolat), gfp, q->node);  in iolatency_pd_alloc()
965 	iolat->stats = __alloc_percpu_gfp(sizeof(struct latency_stat),  in iolatency_pd_alloc()
967 	if (!iolat->stats) {  in iolatency_pd_alloc()
971 	return &iolat->pd;  in iolatency_pd_alloc()
978 	struct rq_qos *rqos = blkcg_rq_qos(blkg->q);  in iolatency_pd_init()
983 	if (blk_queue_nonrot(blkg->q))  in iolatency_pd_init()
984 		iolat->ssd = true;  in iolatency_pd_init()
986 		iolat->ssd = false;  in iolatency_pd_init()
990 		stat = per_cpu_ptr(iolat->stats, cpu);  in iolatency_pd_init()
994 	latency_stat_init(iolat, &iolat->cur_stat);  in iolatency_pd_init()
995 	rq_wait_init(&iolat->rq_wait);  in iolatency_pd_init()
996 	spin_lock_init(&iolat->child_lat.lock);  in iolatency_pd_init()
997 	iolat->rq_depth.queue_depth = blkg->q->nr_requests;  in iolatency_pd_init()
998 	iolat->rq_depth.max_depth = UINT_MAX;  in iolatency_pd_init()
999 	iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth;  in iolatency_pd_init()
1000 	iolat->blkiolat = blkiolat;  in iolatency_pd_init()
1001 	iolat->cur_win_nsec = 100 * NSEC_PER_MSEC;  in iolatency_pd_init()
1002 	atomic64_set(&iolat->window_start, now);  in iolatency_pd_init()
1008 	if (blkg->parent && blkg_to_pd(blkg->parent, &blkcg_policy_iolatency)) {  in iolatency_pd_init()
1009 		struct iolatency_grp *parent = blkg_to_lat(blkg->parent);  in iolatency_pd_init()
1010 		atomic_set(&iolat->scale_cookie,  in iolatency_pd_init()
1011 			   atomic_read(&parent->child_lat.scale_cookie));  in iolatency_pd_init()
1013 		atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
1016 	atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE);  in iolatency_pd_init()
1031 	free_percpu(iolat->stats);  in iolatency_pd_free()
1037 		.name = "latency",