4362 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
4364 	if (cfs_b->quota != RUNTIME_INF)  in __refill_cfs_bandwidth_runtime()
4365 		cfs_b->runtime = cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
4377 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);  in assign_cfs_rq_runtime()  local
4383 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4384 	if (cfs_b->quota == RUNTIME_INF)  in assign_cfs_rq_runtime()
4387 		start_cfs_bandwidth(cfs_b);  in assign_cfs_rq_runtime()
4389 		if (cfs_b->runtime > 0) {  in assign_cfs_rq_runtime()
4390 			amount = min(cfs_b->runtime, min_amount);  in assign_cfs_rq_runtime()
4391 			cfs_b->runtime -= amount;  in assign_cfs_rq_runtime()
4392 			cfs_b->idle = 0;  in assign_cfs_rq_runtime()
4395 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4493 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
4527 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
4528 	empty = list_empty(&cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4535 	if (cfs_b->distribute_running)  in throttle_cfs_rq()
4536 		list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4538 		list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4545 		start_cfs_bandwidth(cfs_b);  in throttle_cfs_rq()
4547 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
4553 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
4564 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
4565 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
4567 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
4601 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, u64 remaining)  in distribute_cfs_runtime()  argument
4608 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
4648 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)  in do_sched_cfs_period_timer()  argument
4654 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
4657 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4658 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
4664 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
4667 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
4671 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
4676 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
4685 	while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) {  in do_sched_cfs_period_timer()
4686 		runtime = cfs_b->runtime;  in do_sched_cfs_period_timer()
4687 		cfs_b->distribute_running = 1;  in do_sched_cfs_period_timer()
4688 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
4690 		runtime = distribute_cfs_runtime(cfs_b, runtime);  in do_sched_cfs_period_timer()
4691 		raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
4693 		cfs_b->distribute_running = 0;  in do_sched_cfs_period_timer()
4694 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4696 		lsub_positive(&cfs_b->runtime, runtime);  in do_sched_cfs_period_timer()
4705 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
4727 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
4729 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
4744 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
4749 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
4753 	if (cfs_b->slack_started)  in start_cfs_slack_bandwidth()
4755 	cfs_b->slack_started = true;  in start_cfs_slack_bandwidth()
4757 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
4765 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
4771 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
4772 	if (cfs_b->quota != RUNTIME_INF) {  in __return_cfs_rq_runtime()
4773 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
4776 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
4777 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
4778 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
4780 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
4801 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
4807 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4808 	cfs_b->slack_started = false;  in do_sched_cfs_slack_timer()
4809 	if (cfs_b->distribute_running) {  in do_sched_cfs_slack_timer()
4810 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4814 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
4815 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4819 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
4820 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
4823 		cfs_b->distribute_running = 1;  in do_sched_cfs_slack_timer()
4825 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4830 	runtime = distribute_cfs_runtime(cfs_b, runtime);  in do_sched_cfs_slack_timer()
4832 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4833 	lsub_positive(&cfs_b->runtime, runtime);  in do_sched_cfs_slack_timer()
4834 	cfs_b->distribute_running = 0;  in do_sched_cfs_slack_timer()
4835 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
4901 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
4904 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
4913 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
4920 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in sched_cfs_period_timer()
4922 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
4927 			u64 new, old = ktime_to_ns(cfs_b->period);  in sched_cfs_period_timer()
4936 				cfs_b->period = ns_to_ktime(new);  in sched_cfs_period_timer()
4937 				cfs_b->quota *= 2;  in sched_cfs_period_timer()
4943 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
4949 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
4956 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);  in sched_cfs_period_timer()
4959 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
4960 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in sched_cfs_period_timer()
4965 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in init_cfs_bandwidth()  argument
4967 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
4968 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
4969 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
4970 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
4972 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
4973 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);  in init_cfs_bandwidth()
4974 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
4975 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
4976 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
4977 	cfs_b->distribute_running = 0;  in init_cfs_bandwidth()
4978 	cfs_b->slack_started = false;  in init_cfs_bandwidth()
4987 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
4989 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
4991 	if (cfs_b->period_active)  in start_cfs_bandwidth()
4994 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
4995 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
4996 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
4999 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
5002 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
5005 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
5006 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
5025 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
5028 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
5029 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
5030 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
5095 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in init_cfs_bandwidth()  argument
5105 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument