560 	u64				now;  member
946 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)  in ioc_refresh_vrate()  argument
948 	s64 pleft = ioc->period_at + ioc->period_us - now->now;  in ioc_refresh_vrate()
977 static void ioc_now(struct ioc *ioc, struct ioc_now *now)  in ioc_now()  argument
981 	now->now_ns = ktime_get();  in ioc_now()
982 	now->now = ktime_to_us(now->now_ns);  in ioc_now()
983 	now->vrate = atomic64_read(&ioc->vtime_rate);  in ioc_now()
995 		now->vnow = ioc->period_at_vtime +  in ioc_now()
996 			(now->now - ioc->period_at) * now->vrate;  in ioc_now()
1000 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)  in ioc_start_period()  argument
1005 	ioc->period_at = now->now;  in ioc_start_period()
1006 	ioc->period_at_vtime = now->vnow;  in ioc_start_period()
1019 				bool save, struct ioc_now *now)  in __propagate_weights()  argument
1030 		iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);  in __propagate_weights()
1084 			      bool save, struct ioc_now *now)  in propagate_weights()  argument
1086 	__propagate_weights(iocg, active, inuse, save, now);  in propagate_weights()
1170 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)  in weight_updated()  argument
1181 		propagate_weights(iocg, weight, iocg->inuse, true, now);  in weight_updated()
1185 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_activate()  argument
1197 		ioc_now(ioc, now);  in iocg_activate()
1210 	ioc_now(ioc, now);  in iocg_activate()
1231 	vtarget = now->vnow - ioc->margins.target;  in iocg_activate()
1247 			  iocg->last_inuse ?: iocg->weight, true, now);  in iocg_activate()
1249 	TRACE_IOCG_PATH(iocg_activate, iocg, now,  in iocg_activate()
1252 	iocg->activated_at = now->now;  in iocg_activate()
1256 		ioc->dfgv_period_at = now->now;  in iocg_activate()
1258 		ioc_start_period(ioc, now);  in iocg_activate()
1270 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_kick_delay()  argument
1281 	tdelta = now->now - iocg->delay_at;  in iocg_kick_delay()
1290 		abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;  in iocg_kick_delay()
1307 		iocg->delay_at = now->now;  in iocg_kick_delay()
1313 			iocg->indelay_since = now->now;  in iocg_kick_delay()
1318 			iocg->local_stat.indelay_us += now->now - iocg->indelay_since;  in iocg_kick_delay()
1328 			    struct ioc_now *now)  in iocg_incur_debt()  argument
1341 		iocg->indebt_since = now->now;  in iocg_incur_debt()
1342 		propagate_weights(iocg, iocg->active, 0, false, now);  in iocg_incur_debt()
1353 			  struct ioc_now *now)  in iocg_pay_debt()  argument
1366 		iocg->local_stat.indebt_us += now->now - iocg->indebt_since;  in iocg_pay_debt()
1370 				  false, now);  in iocg_pay_debt()
1406 			    struct ioc_now *now)  in iocg_kick_waitq()  argument
1417 	vbudget = now->vnow - atomic64_read(&iocg->vtime);  in iocg_kick_waitq()
1429 		iocg_pay_debt(iocg, abs_vpay, now);  in iocg_kick_waitq()
1434 		iocg_kick_delay(iocg, now);  in iocg_kick_waitq()
1459 			iocg->local_stat.wait_us += now->now - iocg->wait_since;  in iocg_kick_waitq()
1466 		iocg->wait_since = now->now;  in iocg_kick_waitq()
1473 	expires = now->now_ns +  in iocg_kick_waitq()
1492 	struct ioc_now now;  in iocg_waitq_timer_fn()  local
1495 	ioc_now(iocg->ioc, &now);  in iocg_waitq_timer_fn()
1498 	iocg_kick_waitq(iocg, pay_debt, &now);  in iocg_waitq_timer_fn()
1588 static void iocg_flush_stat_one(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_flush_stat_one()  argument
1638 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)  in iocg_flush_stat()  argument
1645 		iocg_flush_stat_one(iocg, now);  in iocg_flush_stat()
1651 		iocg_flush_stat_one(iocg, now);  in iocg_flush_stat()
1662 				  u32 usage, struct ioc_now *now)  in hweight_after_donation()  argument
1674 	    time_after64(vtime, now->vnow - ioc->margins.min))  in hweight_after_donation()
1678 	excess = now->vnow - vtime - ioc->margins.target;  in hweight_after_donation()
1702 	delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),  in hweight_after_donation()
1703 			  now->vnow - ioc->period_at_vtime);  in hweight_after_donation()
1767 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)  in transfer_surpluses()  argument
1969 		TRACE_IOCG_PATH(inuse_transfer, iocg, now,  in transfer_surpluses()
1974 		__propagate_weights(iocg, iocg->active, inuse, true, now);  in transfer_surpluses()
1994 			      struct ioc_now *now)  in ioc_forgive_debts()  argument
2001 		ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2017 	if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))  in ioc_forgive_debts()
2024 	dur = now->now - ioc->dfgv_period_at;  in ioc_forgive_debts()
2027 	ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2064 		iocg_kick_waitq(iocg, true, now);  in ioc_forgive_debts()
2066 		TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,  in ioc_forgive_debts()
2078 	struct ioc_now now;  in ioc_timer_fn()  local
2094 	ioc_now(ioc, &now);  in ioc_timer_fn()
2096 	period_vtime = now.vnow - ioc->period_at_vtime;  in ioc_timer_fn()
2117 			iocg->local_stat.wait_us += now.now - iocg->wait_since;  in ioc_timer_fn()
2118 			iocg->wait_since = now.now;  in ioc_timer_fn()
2122 				now.now - iocg->indebt_since;  in ioc_timer_fn()
2123 			iocg->indebt_since = now.now;  in ioc_timer_fn()
2127 				now.now - iocg->indelay_since;  in ioc_timer_fn()
2128 			iocg->indelay_since = now.now;  in ioc_timer_fn()
2134 			iocg_kick_waitq(iocg, true, &now);  in ioc_timer_fn()
2148 			excess = now.vnow - vtime - ioc->margins.target;  in ioc_timer_fn()
2157 			__propagate_weights(iocg, 0, 0, false, &now);  in ioc_timer_fn()
2169 	iocg_flush_stat(&ioc->active_iocgs, &now);  in ioc_timer_fn()
2193 		    time_after64(vtime, now.vnow -  in ioc_timer_fn()
2195 		    time_before64(vdone, now.vnow - period_vtime))  in ioc_timer_fn()
2214 			usage_dur = max_t(u64, now.now - iocg->activated_at, 1);  in ioc_timer_fn()
2216 			usage_dur = max_t(u64, now.now - ioc->period_at, 1);  in ioc_timer_fn()
2227 		     time_before64(vtime, now.vnow - ioc->margins.low))) {  in ioc_timer_fn()
2237 							 usage, &now);  in ioc_timer_fn()
2243 				TRACE_IOCG_PATH(inuse_shortage, iocg, &now,  in ioc_timer_fn()
2248 						    iocg->active, true, &now);  in ioc_timer_fn()
2258 		transfer_surpluses(&surpluses, &now);  in ioc_timer_fn()
2360 	ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);  in ioc_timer_fn()
2370 			ioc_start_period(ioc, &now);  in ioc_timer_fn()
2377 		ioc_refresh_vrate(ioc, &now);  in ioc_timer_fn()
2384 				      u64 abs_cost, struct ioc_now *now)  in adjust_inuse_and_calc_cost()  argument
2396 	margin = now->vnow - vtime - cost;  in adjust_inuse_and_calc_cost()
2429 		propagate_weights(iocg, iocg->active, new_inuse, true, now);  in adjust_inuse_and_calc_cost()
2432 	} while (time_after64(vtime + cost, now->vnow) &&  in adjust_inuse_and_calc_cost()
2437 	TRACE_IOCG_PATH(inuse_adjust, iocg, now,  in adjust_inuse_and_calc_cost()
2522 	struct ioc_now now;  in ioc_rqos_throttle()  local
2537 	if (!iocg_activate(iocg, &now))  in ioc_rqos_throttle()
2542 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_throttle()
2550 	    time_before_eq64(vtime + cost, now.vnow)) {  in ioc_rqos_throttle()
2598 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_throttle()
2599 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_throttle()
2614 				  &now);  in ioc_rqos_throttle()
2637 	iocg_kick_waitq(iocg, ioc_locked, &now);  in ioc_rqos_throttle()
2658 	struct ioc_now now;  in ioc_rqos_merge()  local
2670 	ioc_now(ioc, &now);  in ioc_rqos_merge()
2673 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_merge()
2685 	    time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {  in ioc_rqos_merge()
2699 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_merge()
2700 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_merge()
2892 	struct ioc_now now;  in ioc_pd_init()  local
2896 	ioc_now(ioc, &now);  in ioc_pd_init()
2899 	atomic64_set(&iocg->vtime, now.vnow);  in ioc_pd_init()
2900 	atomic64_set(&iocg->done_vtime, now.vnow);  in ioc_pd_init()
2920 	weight_updated(iocg, &now);  in ioc_pd_init()
2934 			struct ioc_now now;  in ioc_pd_free()  local
2936 			ioc_now(ioc, &now);  in ioc_pd_free()
2937 			propagate_weights(iocg, 0, 0, false, &now);  in ioc_pd_free()
3011 	struct ioc_now now;  in ioc_weight_write()  local
3032 				ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3033 				weight_updated(iocg, &now);  in ioc_weight_write()
3059 	ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3060 	weight_updated(iocg, &now);  in ioc_weight_write()