4601 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
4603 	if (cfs_b->quota != RUNTIME_INF)  in __refill_cfs_bandwidth_runtime()
4604 		cfs_b->runtime = cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
4613 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,  in __assign_cfs_rq_runtime()  argument
4618 	lockdep_assert_held(&cfs_b->lock);  in __assign_cfs_rq_runtime()
4623 	if (cfs_b->quota == RUNTIME_INF)  in __assign_cfs_rq_runtime()
4626 		start_cfs_bandwidth(cfs_b);  in __assign_cfs_rq_runtime()
4628 		if (cfs_b->runtime > 0) {  in __assign_cfs_rq_runtime()
4629 			amount = min(cfs_b->runtime, min_amount);  in __assign_cfs_rq_runtime()
4630 			cfs_b->runtime -= amount;  in __assign_cfs_rq_runtime()
4631 			cfs_b->idle = 0;  in __assign_cfs_rq_runtime()
4643 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in assign_cfs_rq_runtime()  local
4646 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4647 	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());  in assign_cfs_rq_runtime()
4648 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4744 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
4748 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
4750 	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {  in throttle_cfs_rq()
4762 				  &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4764 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
4813 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
4823 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
4824 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
4826 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
4895 static void distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)  in distribute_cfs_runtime()  argument
4901 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
4913 		raw_spin_lock(&cfs_b->lock);  in distribute_cfs_runtime()
4915 		if (runtime > cfs_b->runtime)  in distribute_cfs_runtime()
4916 			runtime = cfs_b->runtime;  in distribute_cfs_runtime()
4917 		cfs_b->runtime -= runtime;  in distribute_cfs_runtime()
4918 		remaining = cfs_b->runtime;  in distribute_cfs_runtime()
4919 		raw_spin_unlock(&cfs_b->lock);  in distribute_cfs_runtime()
4942 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)  in do_sched_cfs_period_timer()  argument
4947 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
4950 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4951 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
4957 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
4960 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
4964 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
4969 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
4974 	while (throttled && cfs_b->runtime > 0) {  in do_sched_cfs_period_timer()
4975 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
4977 		distribute_cfs_runtime(cfs_b);  in do_sched_cfs_period_timer()
4978 		raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
4980 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4989 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
5011 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
5013 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
5028 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
5033 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
5037 	if (cfs_b->slack_started)  in start_cfs_slack_bandwidth()
5039 	cfs_b->slack_started = true;  in start_cfs_slack_bandwidth()
5041 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
5049 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
5055 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
5056 	if (cfs_b->quota != RUNTIME_INF) {  in __return_cfs_rq_runtime()
5057 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
5060 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
5061 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
5062 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
5064 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
5085 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
5091 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5092 	cfs_b->slack_started = false;  in do_sched_cfs_slack_timer()
5094 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
5095 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5099 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
5100 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
5102 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5107 	distribute_cfs_runtime(cfs_b);  in do_sched_cfs_slack_timer()
5109 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5110 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5175 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
5178 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
5187 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
5194 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in sched_cfs_period_timer()
5196 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
5200 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);  in sched_cfs_period_timer()
5203 			u64 new, old = ktime_to_ns(cfs_b->period);  in sched_cfs_period_timer()
5212 				cfs_b->period = ns_to_ktime(new);  in sched_cfs_period_timer()
5213 				cfs_b->quota *= 2;  in sched_cfs_period_timer()
5219 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
5225 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
5233 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
5234 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in sched_cfs_period_timer()
5239 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in init_cfs_bandwidth()  argument
5241 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
5242 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
5243 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
5244 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
5246 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
5247 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);  in init_cfs_bandwidth()
5248 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
5249 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
5250 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
5251 	cfs_b->slack_started = false;  in init_cfs_bandwidth()
5260 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
5262 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
5264 	if (cfs_b->period_active)  in start_cfs_bandwidth()
5267 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
5268 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
5269 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
5272 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
5275 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
5278 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
5279 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
5298 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
5301 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
5302 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
5303 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
5368 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in init_cfs_bandwidth()  argument
5378 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument