557 	u64				now;  member
939 static void ioc_refresh_vrate(struct ioc *ioc, struct ioc_now *now)  in ioc_refresh_vrate()  argument
941 	s64 pleft = ioc->period_at + ioc->period_us - now->now;  in ioc_refresh_vrate()
1018 static void ioc_now(struct ioc *ioc, struct ioc_now *now)  in ioc_now()  argument
1022 	now->now_ns = ktime_get();  in ioc_now()
1023 	now->now = ktime_to_us(now->now_ns);  in ioc_now()
1024 	now->vrate = atomic64_read(&ioc->vtime_rate);  in ioc_now()
1036 		now->vnow = ioc->period_at_vtime +  in ioc_now()
1037 			(now->now - ioc->period_at) * now->vrate;  in ioc_now()
1041 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)  in ioc_start_period()  argument
1046 	ioc->period_at = now->now;  in ioc_start_period()
1047 	ioc->period_at_vtime = now->vnow;  in ioc_start_period()
1060 				bool save, struct ioc_now *now)  in __propagate_weights()  argument
1081 		iocg->saved_margin = now->vnow - atomic64_read(&iocg->vtime);  in __propagate_weights()
1135 			      bool save, struct ioc_now *now)  in propagate_weights()  argument
1137 	__propagate_weights(iocg, active, inuse, save, now);  in propagate_weights()
1221 static void weight_updated(struct ioc_gq *iocg, struct ioc_now *now)  in weight_updated()  argument
1232 		propagate_weights(iocg, weight, iocg->inuse, true, now);  in weight_updated()
1236 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_activate()  argument
1248 		ioc_now(ioc, now);  in iocg_activate()
1261 	ioc_now(ioc, now);  in iocg_activate()
1282 	vtarget = now->vnow - ioc->margins.target;  in iocg_activate()
1298 			  iocg->last_inuse ?: iocg->weight, true, now);  in iocg_activate()
1300 	TRACE_IOCG_PATH(iocg_activate, iocg, now,  in iocg_activate()
1303 	iocg->activated_at = now->now;  in iocg_activate()
1307 		ioc->dfgv_period_at = now->now;  in iocg_activate()
1309 		ioc_start_period(ioc, now);  in iocg_activate()
1321 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_kick_delay()  argument
1332 	tdelta = now->now - iocg->delay_at;  in iocg_kick_delay()
1341 		abs_cost_to_cost(iocg->abs_vdebt, hwa) - now->vnow;  in iocg_kick_delay()
1358 		iocg->delay_at = now->now;  in iocg_kick_delay()
1364 			iocg->indelay_since = now->now;  in iocg_kick_delay()
1369 			iocg->stat.indelay_us += now->now - iocg->indelay_since;  in iocg_kick_delay()
1379 			    struct ioc_now *now)  in iocg_incur_debt()  argument
1392 		iocg->indebt_since = now->now;  in iocg_incur_debt()
1393 		propagate_weights(iocg, iocg->active, 0, false, now);  in iocg_incur_debt()
1404 			  struct ioc_now *now)  in iocg_pay_debt()  argument
1417 		iocg->stat.indebt_us += now->now - iocg->indebt_since;  in iocg_pay_debt()
1421 				  false, now);  in iocg_pay_debt()
1458 			    struct ioc_now *now)  in iocg_kick_waitq()  argument
1469 	vbudget = now->vnow - atomic64_read(&iocg->vtime);  in iocg_kick_waitq()
1481 		iocg_pay_debt(iocg, abs_vpay, now);  in iocg_kick_waitq()
1486 		iocg_kick_delay(iocg, now);  in iocg_kick_waitq()
1511 			iocg->stat.wait_us += now->now - iocg->wait_since;  in iocg_kick_waitq()
1518 		iocg->wait_since = now->now;  in iocg_kick_waitq()
1525 	expires = now->now_ns +  in iocg_kick_waitq()
1544 	struct ioc_now now;  in iocg_waitq_timer_fn()  local
1547 	ioc_now(iocg->ioc, &now);  in iocg_waitq_timer_fn()
1550 	iocg_kick_waitq(iocg, pay_debt, &now);  in iocg_waitq_timer_fn()
1660 static void iocg_flush_stat_leaf(struct ioc_gq *iocg, struct ioc_now *now)  in iocg_flush_stat_leaf()  argument
1684 static void iocg_flush_stat(struct list_head *target_iocgs, struct ioc_now *now)  in iocg_flush_stat()  argument
1691 		iocg_flush_stat_leaf(iocg, now);  in iocg_flush_stat()
1708 				  u32 usage, struct ioc_now *now)  in hweight_after_donation()  argument
1720 	    time_after64(vtime, now->vnow - ioc->margins.min))  in hweight_after_donation()
1724 	excess = now->vnow - vtime - ioc->margins.target;  in hweight_after_donation()
1748 	delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),  in hweight_after_donation()
1749 			  now->vnow - ioc->period_at_vtime);  in hweight_after_donation()
1813 static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)  in transfer_surpluses()  argument
1991 	 * All inner nodes now have ->hweight_inuse and ->child_adjusted_sum and  in transfer_surpluses()
2015 		TRACE_IOCG_PATH(inuse_transfer, iocg, now,  in transfer_surpluses()
2020 		__propagate_weights(iocg, iocg->active, inuse, true, now);  in transfer_surpluses()
2040 			      struct ioc_now *now)  in ioc_forgive_debts()  argument
2047 		ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2063 	if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))  in ioc_forgive_debts()
2070 	dur = now->now - ioc->dfgv_period_at;  in ioc_forgive_debts()
2073 	ioc->dfgv_period_at = now->now;  in ioc_forgive_debts()
2110 		iocg_kick_waitq(iocg, true, now);  in ioc_forgive_debts()
2112 		TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,  in ioc_forgive_debts()
2130 static int ioc_check_iocgs(struct ioc *ioc, struct ioc_now *now)  in ioc_check_iocgs()  argument
2144 			iocg->stat.wait_us += now->now - iocg->wait_since;  in ioc_check_iocgs()
2145 			iocg->wait_since = now->now;  in ioc_check_iocgs()
2149 				now->now - iocg->indebt_since;  in ioc_check_iocgs()
2150 			iocg->indebt_since = now->now;  in ioc_check_iocgs()
2154 				now->now - iocg->indelay_since;  in ioc_check_iocgs()
2155 			iocg->indelay_since = now->now;  in ioc_check_iocgs()
2161 			iocg_kick_waitq(iocg, true, now);  in ioc_check_iocgs()
2175 			excess = now->vnow - vtime - ioc->margins.target;  in ioc_check_iocgs()
2184 			TRACE_IOCG_PATH(iocg_idle, iocg, now,  in ioc_check_iocgs()
2187 			__propagate_weights(iocg, 0, 0, false, now);  in ioc_check_iocgs()
2202 	struct ioc_now now;  in ioc_timer_fn()  local
2218 	ioc_now(ioc, &now);  in ioc_timer_fn()
2220 	period_vtime = now.vnow - ioc->period_at_vtime;  in ioc_timer_fn()
2226 	nr_debtors = ioc_check_iocgs(ioc, &now);  in ioc_timer_fn()
2232 	iocg_flush_stat(&ioc->active_iocgs, &now);  in ioc_timer_fn()
2256 		    time_after64(vtime, now.vnow -  in ioc_timer_fn()
2258 		    time_before64(vdone, now.vnow - period_vtime))  in ioc_timer_fn()
2272 		     time_before64(vtime, now.vnow - ioc->margins.low))) {  in ioc_timer_fn()
2286 				usage_dur = max_t(u64, now.now - iocg->activated_at, 1);  in ioc_timer_fn()
2288 				usage_dur = max_t(u64, now.now - ioc->period_at, 1);  in ioc_timer_fn()
2302 							 usage, &now);  in ioc_timer_fn()
2325 				TRACE_IOCG_PATH(inuse_shortage, iocg, &now,  in ioc_timer_fn()
2330 						    iocg->active, true, &now);  in ioc_timer_fn()
2340 		transfer_surpluses(&surpluses, &now);  in ioc_timer_fn()
2399 	ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);  in ioc_timer_fn()
2409 			ioc_start_period(ioc, &now);  in ioc_timer_fn()
2416 		ioc_refresh_vrate(ioc, &now);  in ioc_timer_fn()
2423 				      u64 abs_cost, struct ioc_now *now)  in adjust_inuse_and_calc_cost()  argument
2435 	margin = now->vnow - vtime - cost;  in adjust_inuse_and_calc_cost()
2468 		propagate_weights(iocg, iocg->active, new_inuse, true, now);  in adjust_inuse_and_calc_cost()
2471 	} while (time_after64(vtime + cost, now->vnow) &&  in adjust_inuse_and_calc_cost()
2476 	TRACE_IOCG_PATH(inuse_adjust, iocg, now,  in adjust_inuse_and_calc_cost()
2561 	struct ioc_now now;  in ioc_rqos_throttle()  local
2576 	if (!iocg_activate(iocg, &now))  in ioc_rqos_throttle()
2581 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_throttle()
2589 	    time_before_eq64(vtime + cost, now.vnow)) {  in ioc_rqos_throttle()
2637 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_throttle()
2638 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_throttle()
2653 				  &now);  in ioc_rqos_throttle()
2676 	iocg_kick_waitq(iocg, ioc_locked, &now);  in ioc_rqos_throttle()
2697 	struct ioc_now now;  in ioc_rqos_merge()  local
2709 	ioc_now(ioc, &now);  in ioc_rqos_merge()
2712 	cost = adjust_inuse_and_calc_cost(iocg, vtime, abs_cost, &now);  in ioc_rqos_merge()
2724 	    time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow)) {  in ioc_rqos_merge()
2738 		iocg_incur_debt(iocg, abs_cost, &now);  in ioc_rqos_merge()
2739 		if (iocg_kick_delay(iocg, &now))  in ioc_rqos_merge()
2944 	struct ioc_now now;  in ioc_pd_init()  local
2948 	ioc_now(ioc, &now);  in ioc_pd_init()
2951 	atomic64_set(&iocg->vtime, now.vnow);  in ioc_pd_init()
2952 	atomic64_set(&iocg->done_vtime, now.vnow);  in ioc_pd_init()
2972 	weight_updated(iocg, &now);  in ioc_pd_init()
2986 			struct ioc_now now;  in ioc_pd_free()  local
2988 			ioc_now(ioc, &now);  in ioc_pd_free()
2989 			propagate_weights(iocg, 0, 0, false, &now);  in ioc_pd_free()
3057 	struct ioc_now now;  in ioc_weight_write()  local
3078 				ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3079 				weight_updated(iocg, &now);  in ioc_weight_write()
3105 	ioc_now(iocg->ioc, &now);  in ioc_weight_write()
3106 	weight_updated(iocg, &now);  in ioc_weight_write()