4257 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
4261 	if (cfs_b->quota == RUNTIME_INF)  in __refill_cfs_bandwidth_runtime()
4265 	cfs_b->runtime = cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
4266 	cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);  in __refill_cfs_bandwidth_runtime()
4267 	cfs_b->expires_seq++;  in __refill_cfs_bandwidth_runtime()
4288 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);  in assign_cfs_rq_runtime()  local
4295 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4296 	if (cfs_b->quota == RUNTIME_INF)  in assign_cfs_rq_runtime()
4299 		start_cfs_bandwidth(cfs_b);  in assign_cfs_rq_runtime()
4301 		if (cfs_b->runtime > 0) {  in assign_cfs_rq_runtime()
4302 			amount = min(cfs_b->runtime, min_amount);  in assign_cfs_rq_runtime()
4303 			cfs_b->runtime -= amount;  in assign_cfs_rq_runtime()
4304 			cfs_b->idle = 0;  in assign_cfs_rq_runtime()
4307 	expires_seq = cfs_b->expires_seq;  in assign_cfs_rq_runtime()
4308 	expires = cfs_b->runtime_expires;  in assign_cfs_rq_runtime()
4309 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
4331 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in expire_cfs_rq_runtime()  local
4348 	if (cfs_rq->expires_seq == cfs_b->expires_seq) {  in expire_cfs_rq_runtime()
4442 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
4474 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
4475 	empty = list_empty(&cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4482 	if (cfs_b->distribute_running)  in throttle_cfs_rq()
4483 		list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4485 		list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
4492 		start_cfs_bandwidth(cfs_b);  in throttle_cfs_rq()
4494 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
4500 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
4511 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
4512 	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
4514 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
4544 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,  in distribute_cfs_runtime()  argument
4552 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
4590 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)  in do_sched_cfs_period_timer()  argument
4596 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
4599 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4600 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
4606 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
4609 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
4613 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
4618 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
4620 	runtime_expires = cfs_b->runtime_expires;  in do_sched_cfs_period_timer()
4629 	while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) {  in do_sched_cfs_period_timer()
4630 		runtime = cfs_b->runtime;  in do_sched_cfs_period_timer()
4631 		cfs_b->distribute_running = 1;  in do_sched_cfs_period_timer()
4632 		raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_period_timer()
4634 		runtime = distribute_cfs_runtime(cfs_b, runtime,  in do_sched_cfs_period_timer()
4636 		raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_period_timer()
4638 		cfs_b->distribute_running = 0;  in do_sched_cfs_period_timer()
4639 		throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
4641 		cfs_b->runtime -= min(runtime, cfs_b->runtime);  in do_sched_cfs_period_timer()
4650 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
4672 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
4674 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
4689 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
4694 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
4697 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
4705 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
4711 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
4712 	if (cfs_b->quota != RUNTIME_INF &&  in __return_cfs_rq_runtime()
4713 	    cfs_rq->runtime_expires == cfs_b->runtime_expires) {  in __return_cfs_rq_runtime()
4714 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
4717 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
4718 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
4719 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
4721 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
4742 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
4748 	raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4749 	if (cfs_b->distribute_running) {  in do_sched_cfs_slack_timer()
4750 		raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4754 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
4755 		raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4759 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
4760 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
4762 	expires = cfs_b->runtime_expires;  in do_sched_cfs_slack_timer()
4764 		cfs_b->distribute_running = 1;  in do_sched_cfs_slack_timer()
4766 	raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4771 	runtime = distribute_cfs_runtime(cfs_b, runtime, expires);  in do_sched_cfs_slack_timer()
4773 	raw_spin_lock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4774 	if (expires == cfs_b->runtime_expires)  in do_sched_cfs_slack_timer()
4775 		cfs_b->runtime -= min(runtime, cfs_b->runtime);  in do_sched_cfs_slack_timer()
4776 	cfs_b->distribute_running = 0;  in do_sched_cfs_slack_timer()
4777 	raw_spin_unlock(&cfs_b->lock);  in do_sched_cfs_slack_timer()
4843 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
4846 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
4853 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
4858 	raw_spin_lock(&cfs_b->lock);  in sched_cfs_period_timer()
4860 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
4864 		idle = do_sched_cfs_period_timer(cfs_b, overrun);  in sched_cfs_period_timer()
4867 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
4868 	raw_spin_unlock(&cfs_b->lock);  in sched_cfs_period_timer()
4873 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in init_cfs_bandwidth()  argument
4875 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
4876 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
4877 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
4878 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
4880 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
4881 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);  in init_cfs_bandwidth()
4882 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
4883 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
4884 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
4885 	cfs_b->distribute_running = 0;  in init_cfs_bandwidth()
4894 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
4898 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
4900 	if (cfs_b->period_active)  in start_cfs_bandwidth()
4903 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
4904 	overrun = hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
4905 	cfs_b->runtime_expires += (overrun + 1) * ktime_to_ns(cfs_b->period);  in start_cfs_bandwidth()
4906 	cfs_b->expires_seq++;  in start_cfs_bandwidth()
4907 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
4910 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
4913 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
4916 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
4917 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
4936 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
4939 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
4940 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
4941 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
5005 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in init_cfs_bandwidth()  argument
5015 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument