4688 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
4690 	if (unlikely(cfs_b->quota == RUNTIME_INF))  in __refill_cfs_bandwidth_runtime()
4693 	cfs_b->runtime += cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
4694 	cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);  in __refill_cfs_bandwidth_runtime()
4703 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,  in __assign_cfs_rq_runtime()  argument
4708 	lockdep_assert_held(&cfs_b->lock);  in __assign_cfs_rq_runtime()
4713 	if (cfs_b->quota == RUNTIME_INF)  in __assign_cfs_rq_runtime()
4716 		start_cfs_bandwidth(cfs_b);  in __assign_cfs_rq_runtime()
4718 		if (cfs_b->runtime > 0) {  in __assign_cfs_rq_runtime()
4719 			amount = min(cfs_b->runtime, min_amount);  in __assign_cfs_rq_runtime()
4720 			cfs_b->runtime -= amount;  in __assign_cfs_rq_runtime()
4721 			cfs_b->idle = 0;  in __assign_cfs_rq_runtime()
4733 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in assign_cfs_rq_runtime()  local
4736 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4737 	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());  in assign_cfs_rq_runtime()
4738 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4834 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
4838 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
4840 	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {  in throttle_cfs_rq()
4852 				  &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4854 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
4921 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
4931 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
4932 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
4934 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
5013 static void distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)  in distribute_cfs_runtime()  argument
5019 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
5031 		raw_spin_lock(&cfs_b->lock);  in distribute_cfs_runtime()
5033 		if (runtime > cfs_b->runtime)  in distribute_cfs_runtime()
5034 			runtime = cfs_b->runtime;  in distribute_cfs_runtime()
5035 		cfs_b->runtime -= runtime;  in distribute_cfs_runtime()
5036 		remaining = cfs_b->runtime;  in distribute_cfs_runtime()
5037 		raw_spin_unlock(&cfs_b->lock);  in distribute_cfs_runtime()
5060 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)  in do_sched_cfs_period_timer()  argument
5065 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
5068 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
5069 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
5072 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
5078 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
5083 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
5088 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
5093 	while (throttled && cfs_b->runtime > 0) {  in do_sched_cfs_period_timer()
5094 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
5096 		distribute_cfs_runtime(cfs_b);  in do_sched_cfs_period_timer()
5097 		raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
5099 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
5108 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
5130 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
5132 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
5147 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
5152 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
5156 	if (cfs_b->slack_started)  in start_cfs_slack_bandwidth()
5158 	cfs_b->slack_started = true;  in start_cfs_slack_bandwidth()
5160 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
5168 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
5174 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
5175 	if (cfs_b->quota != RUNTIME_INF) {  in __return_cfs_rq_runtime()
5176 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
5179 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
5180 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
5181 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
5183 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
5204 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
5210 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5211 	cfs_b->slack_started = false;  in do_sched_cfs_slack_timer()
5213 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
5214 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5218 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
5219 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
5221 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
5226 	distribute_cfs_runtime(cfs_b);  in do_sched_cfs_slack_timer()
5291 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
5294 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
5303 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
5310 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in sched_cfs_period_timer()
5312 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
5316 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);  in sched_cfs_period_timer()
5319 			u64 new, old = ktime_to_ns(cfs_b->period);  in sched_cfs_period_timer()
5328 				cfs_b->period = ns_to_ktime(new);  in sched_cfs_period_timer()
5329 				cfs_b->quota *= 2;  in sched_cfs_period_timer()
5330 				cfs_b->burst *= 2;  in sched_cfs_period_timer()
5336 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
5342 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
5350 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
5351 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in sched_cfs_period_timer()
5356 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in init_cfs_bandwidth()  argument
5358 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
5359 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
5360 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
5361 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
5362 	cfs_b->burst = 0;  in init_cfs_bandwidth()
5364 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
5365 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);  in init_cfs_bandwidth()
5366 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
5367 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
5368 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
5369 	cfs_b->slack_started = false;  in init_cfs_bandwidth()
5378 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
5380 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
5382 	if (cfs_b->period_active)  in start_cfs_bandwidth()
5385 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
5386 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
5387 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
5390 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
5393 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
5396 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
5397 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
5416 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
5419 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
5420 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
5421 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
5486 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in init_cfs_bandwidth()  argument
5496 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument