392 struct ioc {  struct
432 	struct ioc			*ioc;  argument
611 static struct ioc *rqos_to_ioc(struct rq_qos *rqos)  in rqos_to_ioc()
613 	return container_of(rqos, struct ioc, rqos);  in rqos_to_ioc()
616 static struct ioc *q_to_ioc(struct request_queue *q)  in q_to_ioc()
629 static const char __maybe_unused *ioc_name(struct ioc *ioc)  in ioc_name()  argument
631 	return q_name(ioc->rqos.q);  in ioc_name()
682 static void ioc_refresh_period_us(struct ioc *ioc)  in ioc_refresh_period_us()  argument
686 	lockdep_assert_held(&ioc->lock);  in ioc_refresh_period_us()
689 	if (ioc->params.qos[QOS_RLAT] >= ioc->params.qos[QOS_WLAT]) {  in ioc_refresh_period_us()
690 		ppm = ioc->params.qos[QOS_RPPM];  in ioc_refresh_period_us()
691 		lat = ioc->params.qos[QOS_RLAT];  in ioc_refresh_period_us()
693 		ppm = ioc->params.qos[QOS_WPPM];  in ioc_refresh_period_us()
694 		lat = ioc->params.qos[QOS_WLAT];  in ioc_refresh_period_us()
713 	ioc->period_us = period_us;  in ioc_refresh_period_us()
714 	ioc->margin_us = period_us * MARGIN_PCT / 100;  in ioc_refresh_period_us()
715 	ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(  in ioc_refresh_period_us()
719 static int ioc_autop_idx(struct ioc *ioc)  in ioc_autop_idx()  argument
721 	int idx = ioc->autop_idx;  in ioc_autop_idx()
727 	if (!blk_queue_nonrot(ioc->rqos.q))  in ioc_autop_idx()
731 	if (blk_queue_depth(ioc->rqos.q) == 1)  in ioc_autop_idx()
739 	if (ioc->user_qos_params || ioc->user_cost_model)  in ioc_autop_idx()
743 	vrate_pct = div64_u64(atomic64_read(&ioc->vtime_rate) * 100,  in ioc_autop_idx()
748 		if (!ioc->autop_too_fast_at)  in ioc_autop_idx()
749 			ioc->autop_too_fast_at = now_ns;  in ioc_autop_idx()
750 		if (now_ns - ioc->autop_too_fast_at >= AUTOP_CYCLE_NSEC)  in ioc_autop_idx()
753 		ioc->autop_too_fast_at = 0;  in ioc_autop_idx()
757 		if (!ioc->autop_too_slow_at)  in ioc_autop_idx()
758 			ioc->autop_too_slow_at = now_ns;  in ioc_autop_idx()
759 		if (now_ns - ioc->autop_too_slow_at >= AUTOP_CYCLE_NSEC)  in ioc_autop_idx()
762 		ioc->autop_too_slow_at = 0;  in ioc_autop_idx()
805 static void ioc_refresh_lcoefs(struct ioc *ioc)  in ioc_refresh_lcoefs()  argument
807 	u64 *u = ioc->params.i_lcoefs;  in ioc_refresh_lcoefs()
808 	u64 *c = ioc->params.lcoefs;  in ioc_refresh_lcoefs()
816 static bool ioc_refresh_params(struct ioc *ioc, bool force)  in ioc_refresh_params()  argument
821 	lockdep_assert_held(&ioc->lock);  in ioc_refresh_params()
823 	idx = ioc_autop_idx(ioc);  in ioc_refresh_params()
826 	if (idx == ioc->autop_idx && !force)  in ioc_refresh_params()
829 	if (idx != ioc->autop_idx)  in ioc_refresh_params()
830 		atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);  in ioc_refresh_params()
832 	ioc->autop_idx = idx;  in ioc_refresh_params()
833 	ioc->autop_too_fast_at = 0;  in ioc_refresh_params()
834 	ioc->autop_too_slow_at = 0;  in ioc_refresh_params()
836 	if (!ioc->user_qos_params)  in ioc_refresh_params()
837 		memcpy(ioc->params.qos, p->qos, sizeof(p->qos));  in ioc_refresh_params()
838 	if (!ioc->user_cost_model)  in ioc_refresh_params()
839 		memcpy(ioc->params.i_lcoefs, p->i_lcoefs, sizeof(p->i_lcoefs));  in ioc_refresh_params()
841 	ioc_refresh_period_us(ioc);  in ioc_refresh_params()
842 	ioc_refresh_lcoefs(ioc);  in ioc_refresh_params()
844 	ioc->vrate_min = DIV64_U64_ROUND_UP((u64)ioc->params.qos[QOS_MIN] *  in ioc_refresh_params()
846 	ioc->vrate_max = div64_u64((u64)ioc->params.qos[QOS_MAX] *  in ioc_refresh_params()
853 static void ioc_now(struct ioc *ioc, struct ioc_now *now)  in ioc_now()  argument
859 	now->vrate = atomic64_read(&ioc->vtime_rate);  in ioc_now()
870 		seq = read_seqcount_begin(&ioc->period_seqcount);  in ioc_now()
871 		now->vnow = ioc->period_at_vtime +  in ioc_now()
872 			(now->now - ioc->period_at) * now->vrate;  in ioc_now()
873 	} while (read_seqcount_retry(&ioc->period_seqcount, seq));  in ioc_now()
876 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)  in ioc_start_period()  argument
878 	lockdep_assert_held(&ioc->lock);  in ioc_start_period()
879 	WARN_ON_ONCE(ioc->running != IOC_RUNNING);  in ioc_start_period()
881 	write_seqcount_begin(&ioc->period_seqcount);  in ioc_start_period()
882 	ioc->period_at = now->now;  in ioc_start_period()
883 	ioc->period_at_vtime = now->vnow;  in ioc_start_period()
884 	write_seqcount_end(&ioc->period_seqcount);  in ioc_start_period()
886 	ioc->timer.expires = jiffies + usecs_to_jiffies(ioc->period_us);  in ioc_start_period()
887 	add_timer(&ioc->timer);  in ioc_start_period()
896 	struct ioc *ioc = iocg->ioc;  in __propagate_active_weight()  local
899 	lockdep_assert_held(&ioc->lock);  in __propagate_active_weight()
936 	ioc->weights_updated = true;  in __propagate_active_weight()
939 static void commit_active_weights(struct ioc *ioc)  in commit_active_weights()  argument
941 	lockdep_assert_held(&ioc->lock);  in commit_active_weights()
943 	if (ioc->weights_updated) {  in commit_active_weights()
946 		atomic_inc(&ioc->hweight_gen);  in commit_active_weights()
947 		ioc->weights_updated = false;  in commit_active_weights()
954 	commit_active_weights(iocg->ioc);  in propagate_active_weight()
959 	struct ioc *ioc = iocg->ioc;  in current_hweight()  local
965 	ioc_gen = atomic_read(&ioc->hweight_gen);  in current_hweight()
1013 	struct ioc *ioc = iocg->ioc;  in weight_updated()  local
1018 	lockdep_assert_held(&ioc->lock);  in weight_updated()
1029 	struct ioc *ioc = iocg->ioc;  in iocg_activate()  local
1039 		ioc_now(ioc, now);  in iocg_activate()
1040 		cur_period = atomic64_read(&ioc->cur_period);  in iocg_activate()
1050 	spin_lock_irq(&ioc->lock);  in iocg_activate()
1052 	ioc_now(ioc, now);  in iocg_activate()
1055 	cur_period = atomic64_read(&ioc->cur_period);  in iocg_activate()
1075 	max_period_delta = DIV64_U64_ROUND_UP(VTIME_VALID_DUR, ioc->period_us);  in iocg_activate()
1077 	vmargin = ioc->margin_us * now->vrate;  in iocg_activate()
1092 	iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;  in iocg_activate()
1093 	list_add(&iocg->active_list, &ioc->active_iocgs);  in iocg_activate()
1102 	if (ioc->running == IOC_IDLE) {  in iocg_activate()
1103 		ioc->running = IOC_RUNNING;  in iocg_activate()
1104 		ioc_start_period(ioc, now);  in iocg_activate()
1108 	spin_unlock_irq(&ioc->lock);  in iocg_activate()
1112 	spin_unlock_irq(&ioc->lock);  in iocg_activate()
1144 	struct ioc *ioc = iocg->ioc;  in iocg_kick_waitq()  local
1146 	u64 margin_ns = (u64)(ioc->period_us *  in iocg_kick_waitq()
1206 	ioc_now(iocg->ioc, &now);  in iocg_waitq_timer_fn()
1217 	struct ioc *ioc = iocg->ioc;  in iocg_kick_delay()  local
1220 	u64 vmargin = ioc->margin_us * now->vrate;  in iocg_kick_delay()
1221 	u64 margin_ns = ioc->margin_us * NSEC_PER_USEC;  in iocg_kick_delay()
1264 	ioc_now(iocg->ioc, &now);  in iocg_delay_timer_fn()
1270 static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p)  in ioc_lat_stat()  argument
1278 		struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu);  in ioc_lat_stat()
1306 				   ioc->period_us * NSEC_PER_USEC);  in ioc_lat_stat()
1312 	struct ioc *ioc = iocg->ioc;  in iocg_is_idle()  local
1316 	    atomic64_read(&ioc->cur_period))  in iocg_is_idle()
1342 	struct ioc *ioc = container_of(timer, struct ioc, timer);  in ioc_timer_fn()  local
1346 	u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];  in ioc_timer_fn()
1347 	u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];  in ioc_timer_fn()
1353 	ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);  in ioc_timer_fn()
1356 	spin_lock_irq(&ioc->lock);  in ioc_timer_fn()
1358 	ioc_now(ioc, &now);  in ioc_timer_fn()
1360 	period_vtime = now.vnow - ioc->period_at_vtime;  in ioc_timer_fn()
1362 		spin_unlock_irq(&ioc->lock);  in ioc_timer_fn()
1372 	list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {  in ioc_timer_fn()
1393 	commit_active_weights(ioc);  in ioc_timer_fn()
1396 	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {  in ioc_timer_fn()
1448 		vmargin = ioc->margin_us * now.vrate;  in ioc_timer_fn()
1499 	list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {  in ioc_timer_fn()
1531 	commit_active_weights(ioc);  in ioc_timer_fn()
1539 	prev_busy_level = ioc->busy_level;  in ioc_timer_fn()
1543 		ioc->busy_level = max(ioc->busy_level, 0);  in ioc_timer_fn()
1544 		ioc->busy_level++;  in ioc_timer_fn()
1550 			ioc->busy_level = min(ioc->busy_level, 0);  in ioc_timer_fn()
1553 				ioc->busy_level--;  in ioc_timer_fn()
1556 		ioc->busy_level = 0;  in ioc_timer_fn()
1559 	ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);  in ioc_timer_fn()
1561 	if (ioc->busy_level > 0 || (ioc->busy_level < 0 && !nr_lagging)) {  in ioc_timer_fn()
1562 		u64 vrate = atomic64_read(&ioc->vtime_rate);  in ioc_timer_fn()
1563 		u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;  in ioc_timer_fn()
1583 			int idx = min_t(int, abs(ioc->busy_level),  in ioc_timer_fn()
1587 			if (ioc->busy_level > 0)  in ioc_timer_fn()
1596 		trace_iocost_ioc_vrate_adj(ioc, vrate, &missed_ppm, rq_wait_pct,  in ioc_timer_fn()
1600 		atomic64_set(&ioc->vtime_rate, vrate);  in ioc_timer_fn()
1601 		ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(  in ioc_timer_fn()
1602 			ioc->period_us * vrate * INUSE_MARGIN_PCT, 100);  in ioc_timer_fn()
1603 	} else if (ioc->busy_level != prev_busy_level || nr_lagging) {  in ioc_timer_fn()
1604 		trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),  in ioc_timer_fn()
1609 	ioc_refresh_params(ioc, false);  in ioc_timer_fn()
1615 	atomic64_inc(&ioc->cur_period);  in ioc_timer_fn()
1617 	if (ioc->running != IOC_STOP) {  in ioc_timer_fn()
1618 		if (!list_empty(&ioc->active_iocgs)) {  in ioc_timer_fn()
1619 			ioc_start_period(ioc, &now);  in ioc_timer_fn()
1621 			ioc->busy_level = 0;  in ioc_timer_fn()
1622 			ioc->running = IOC_IDLE;  in ioc_timer_fn()
1626 	spin_unlock_irq(&ioc->lock);  in ioc_timer_fn()
1632 	struct ioc *ioc = iocg->ioc;  in calc_vtime_cost_builtin()  local
1640 		coef_seqio	= ioc->params.lcoefs[LCOEF_RSEQIO];  in calc_vtime_cost_builtin()
1641 		coef_randio	= ioc->params.lcoefs[LCOEF_RRANDIO];  in calc_vtime_cost_builtin()
1642 		coef_page	= ioc->params.lcoefs[LCOEF_RPAGE];  in calc_vtime_cost_builtin()
1645 		coef_seqio	= ioc->params.lcoefs[LCOEF_WSEQIO];  in calc_vtime_cost_builtin()
1646 		coef_randio	= ioc->params.lcoefs[LCOEF_WRANDIO];  in calc_vtime_cost_builtin()
1647 		coef_page	= ioc->params.lcoefs[LCOEF_WPAGE];  in calc_vtime_cost_builtin()
1681 	struct ioc *ioc = rqos_to_ioc(rqos);  in ioc_rqos_throttle()  local
1689 	if (!ioc->enabled || !iocg->level)  in ioc_rqos_throttle()
1707 	    time_after_eq64(vtime + ioc->inuse_margin_vtime, now.vnow)) {  in ioc_rqos_throttle()
1710 		spin_lock_irq(&ioc->lock);  in ioc_rqos_throttle()
1712 		spin_unlock_irq(&ioc->lock);  in ioc_rqos_throttle()
1799 	struct ioc *ioc = iocg->ioc;  in ioc_rqos_merge()  local
1806 	if (!ioc->enabled || !iocg->level)  in ioc_rqos_merge()
1813 	ioc_now(ioc, &now);  in ioc_rqos_merge()
1844 	struct ioc *ioc = rqos_to_ioc(rqos);  in ioc_rqos_done()  local
1848 	if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)  in ioc_rqos_done()
1867 	if (on_q_ns <= ioc->params.qos[pidx] * NSEC_PER_USEC)  in ioc_rqos_done()
1868 		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);  in ioc_rqos_done()
1870 		this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);  in ioc_rqos_done()
1872 	this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);  in ioc_rqos_done()
1877 	struct ioc *ioc = rqos_to_ioc(rqos);  in ioc_rqos_queue_depth_changed()  local
1879 	spin_lock_irq(&ioc->lock);  in ioc_rqos_queue_depth_changed()
1880 	ioc_refresh_params(ioc, false);  in ioc_rqos_queue_depth_changed()
1881 	spin_unlock_irq(&ioc->lock);  in ioc_rqos_queue_depth_changed()
1886 	struct ioc *ioc = rqos_to_ioc(rqos);  in ioc_rqos_exit()  local
1890 	spin_lock_irq(&ioc->lock);  in ioc_rqos_exit()
1891 	ioc->running = IOC_STOP;  in ioc_rqos_exit()
1892 	spin_unlock_irq(&ioc->lock);  in ioc_rqos_exit()
1894 	del_timer_sync(&ioc->timer);  in ioc_rqos_exit()
1895 	free_percpu(ioc->pcpu_stat);  in ioc_rqos_exit()
1896 	kfree(ioc);  in ioc_rqos_exit()
1910 	struct ioc *ioc;  in blk_iocost_init()  local
1914 	ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);  in blk_iocost_init()
1915 	if (!ioc)  in blk_iocost_init()
1918 	ioc->pcpu_stat = alloc_percpu(struct ioc_pcpu_stat);  in blk_iocost_init()
1919 	if (!ioc->pcpu_stat) {  in blk_iocost_init()
1920 		kfree(ioc);  in blk_iocost_init()
1924 	rqos = &ioc->rqos;  in blk_iocost_init()
1929 	spin_lock_init(&ioc->lock);  in blk_iocost_init()
1930 	timer_setup(&ioc->timer, ioc_timer_fn, 0);  in blk_iocost_init()
1931 	INIT_LIST_HEAD(&ioc->active_iocgs);  in blk_iocost_init()
1933 	ioc->running = IOC_IDLE;  in blk_iocost_init()
1934 	atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);  in blk_iocost_init()
1935 	seqcount_init(&ioc->period_seqcount);  in blk_iocost_init()
1936 	ioc->period_at = ktime_to_us(ktime_get());  in blk_iocost_init()
1937 	atomic64_set(&ioc->cur_period, 0);  in blk_iocost_init()
1938 	atomic_set(&ioc->hweight_gen, 0);  in blk_iocost_init()
1940 	spin_lock_irq(&ioc->lock);  in blk_iocost_init()
1941 	ioc->autop_idx = AUTOP_INVALID;  in blk_iocost_init()
1942 	ioc_refresh_params(ioc, true);  in blk_iocost_init()
1943 	spin_unlock_irq(&ioc->lock);  in blk_iocost_init()
1949 		free_percpu(ioc->pcpu_stat);  in blk_iocost_init()
1950 		kfree(ioc);  in blk_iocost_init()
1991 	struct ioc *ioc = q_to_ioc(blkg->q);  in ioc_pd_init()  local
1996 	ioc_now(ioc, &now);  in ioc_pd_init()
1998 	iocg->ioc = ioc;  in ioc_pd_init()
2002 	atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));  in ioc_pd_init()
2020 	spin_lock_irqsave(&ioc->lock, flags);  in ioc_pd_init()
2022 	spin_unlock_irqrestore(&ioc->lock, flags);  in ioc_pd_init()
2028 	struct ioc *ioc = iocg->ioc;  in ioc_pd_free()  local
2030 	if (ioc) {  in ioc_pd_free()
2031 		spin_lock(&ioc->lock);  in ioc_pd_free()
2036 		spin_unlock(&ioc->lock);  in ioc_pd_free()
2092 				spin_lock_irq(&iocg->ioc->lock);  in ioc_weight_write()
2094 				spin_unlock_irq(&iocg->ioc->lock);  in ioc_weight_write()
2117 	spin_lock(&iocg->ioc->lock);  in ioc_weight_write()
2120 	spin_unlock(&iocg->ioc->lock);  in ioc_weight_write()
2134 	struct ioc *ioc = pd_to_iocg(pd)->ioc;  in ioc_qos_prfill()  local
2140 		   dname, ioc->enabled, ioc->user_qos_params ? "user" : "auto",  in ioc_qos_prfill()
2141 		   ioc->params.qos[QOS_RPPM] / 10000,  in ioc_qos_prfill()
2142 		   ioc->params.qos[QOS_RPPM] % 10000 / 100,  in ioc_qos_prfill()
2143 		   ioc->params.qos[QOS_RLAT],  in ioc_qos_prfill()
2144 		   ioc->params.qos[QOS_WPPM] / 10000,  in ioc_qos_prfill()
2145 		   ioc->params.qos[QOS_WPPM] % 10000 / 100,  in ioc_qos_prfill()
2146 		   ioc->params.qos[QOS_WLAT],  in ioc_qos_prfill()
2147 		   ioc->params.qos[QOS_MIN] / 10000,  in ioc_qos_prfill()
2148 		   ioc->params.qos[QOS_MIN] % 10000 / 100,  in ioc_qos_prfill()
2149 		   ioc->params.qos[QOS_MAX] / 10000,  in ioc_qos_prfill()
2150 		   ioc->params.qos[QOS_MAX] % 10000 / 100);  in ioc_qos_prfill()
2183 	struct ioc *ioc;  in ioc_qos_write()  local
2193 	ioc = q_to_ioc(disk->queue);  in ioc_qos_write()
2194 	if (!ioc) {  in ioc_qos_write()
2198 		ioc = q_to_ioc(disk->queue);  in ioc_qos_write()
2201 	spin_lock_irq(&ioc->lock);  in ioc_qos_write()
2202 	memcpy(qos, ioc->params.qos, sizeof(qos));  in ioc_qos_write()
2203 	enable = ioc->enabled;  in ioc_qos_write()
2204 	user = ioc->user_qos_params;  in ioc_qos_write()
2205 	spin_unlock_irq(&ioc->lock);  in ioc_qos_write()
2272 	spin_lock_irq(&ioc->lock);  in ioc_qos_write()
2275 		blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);  in ioc_qos_write()
2276 		ioc->enabled = true;  in ioc_qos_write()
2278 		blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);  in ioc_qos_write()
2279 		ioc->enabled = false;  in ioc_qos_write()
2283 		memcpy(ioc->params.qos, qos, sizeof(qos));  in ioc_qos_write()
2284 		ioc->user_qos_params = true;  in ioc_qos_write()
2286 		ioc->user_qos_params = false;  in ioc_qos_write()
2289 	ioc_refresh_params(ioc, true);  in ioc_qos_write()
2290 	spin_unlock_irq(&ioc->lock);  in ioc_qos_write()
2305 	struct ioc *ioc = pd_to_iocg(pd)->ioc;  in ioc_cost_model_prfill()  local
2306 	u64 *u = ioc->params.i_lcoefs;  in ioc_cost_model_prfill()
2314 		   dname, ioc->user_cost_model ? "user" : "auto",  in ioc_cost_model_prfill()
2349 	struct ioc *ioc;  in ioc_cost_model_write()  local
2359 	ioc = q_to_ioc(disk->queue);  in ioc_cost_model_write()
2360 	if (!ioc) {  in ioc_cost_model_write()
2364 		ioc = q_to_ioc(disk->queue);  in ioc_cost_model_write()
2367 	spin_lock_irq(&ioc->lock);  in ioc_cost_model_write()
2368 	memcpy(u, ioc->params.i_lcoefs, sizeof(u));  in ioc_cost_model_write()
2369 	user = ioc->user_cost_model;  in ioc_cost_model_write()
2370 	spin_unlock_irq(&ioc->lock);  in ioc_cost_model_write()
2407 	spin_lock_irq(&ioc->lock);  in ioc_cost_model_write()
2409 		memcpy(ioc->params.i_lcoefs, u, sizeof(u));  in ioc_cost_model_write()
2410 		ioc->user_cost_model = true;  in ioc_cost_model_write()
2412 		ioc->user_cost_model = false;  in ioc_cost_model_write()
2414 	ioc_refresh_params(ioc, true);  in ioc_cost_model_write()
2415 	spin_unlock_irq(&ioc->lock);  in ioc_cost_model_write()