5393 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
5397 	if (unlikely(cfs_b->quota == RUNTIME_INF))  in __refill_cfs_bandwidth_runtime()
5400 	cfs_b->runtime += cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
5401 	runtime = cfs_b->runtime_snap - cfs_b->runtime;  in __refill_cfs_bandwidth_runtime()
5403 		cfs_b->burst_time += runtime;  in __refill_cfs_bandwidth_runtime()
5404 		cfs_b->nr_burst++;  in __refill_cfs_bandwidth_runtime()
5407 	cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);  in __refill_cfs_bandwidth_runtime()
5408 	cfs_b->runtime_snap = cfs_b->runtime;  in __refill_cfs_bandwidth_runtime()
5417 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,  in __assign_cfs_rq_runtime()  argument
5422 	lockdep_assert_held(&cfs_b->lock);  in __assign_cfs_rq_runtime()
5427 	if (cfs_b->quota == RUNTIME_INF)  in __assign_cfs_rq_runtime()
5430 		start_cfs_bandwidth(cfs_b);  in __assign_cfs_rq_runtime()
5432 		if (cfs_b->runtime > 0) {  in __assign_cfs_rq_runtime()
5433 			amount = min(cfs_b->runtime, min_amount);  in __assign_cfs_rq_runtime()
5434 			cfs_b->runtime -= amount;  in __assign_cfs_rq_runtime()
5435 			cfs_b->idle = 0;  in __assign_cfs_rq_runtime()
5447 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in assign_cfs_rq_runtime()  local
5450 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
5451 	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());  in assign_cfs_rq_runtime()
5452 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
5563 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
5567 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
5569 	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {  in throttle_cfs_rq()
5581 				  &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
5583 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
5652 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
5662 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
5664 		cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
5668 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
5812 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)  in distribute_cfs_runtime()  argument
5823 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
5845 		raw_spin_lock(&cfs_b->lock);  in distribute_cfs_runtime()
5847 		if (runtime > cfs_b->runtime)  in distribute_cfs_runtime()
5848 			runtime = cfs_b->runtime;  in distribute_cfs_runtime()
5849 		cfs_b->runtime -= runtime;  in distribute_cfs_runtime()
5850 		remaining = cfs_b->runtime;  in distribute_cfs_runtime()
5851 		raw_spin_unlock(&cfs_b->lock);  in distribute_cfs_runtime()
5888 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)  in do_sched_cfs_period_timer()  argument
5893 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
5896 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
5897 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
5900 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
5906 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
5911 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
5916 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
5921 	while (throttled && cfs_b->runtime > 0) {  in do_sched_cfs_period_timer()
5922 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
5924 		throttled = distribute_cfs_runtime(cfs_b);  in do_sched_cfs_period_timer()
5925 		raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
5934 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
5956 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
5958 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
5973 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
5978 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
5982 	if (cfs_b->slack_started)  in start_cfs_slack_bandwidth()
5984 	cfs_b->slack_started = true;  in start_cfs_slack_bandwidth()
5986 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
5994 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
6000 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
6001 	if (cfs_b->quota != RUNTIME_INF) {  in __return_cfs_rq_runtime()
6002 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
6005 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
6006 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
6007 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
6009 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
6030 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
6036 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6037 	cfs_b->slack_started = false;  in do_sched_cfs_slack_timer()
6039 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
6040 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6044 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
6045 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
6047 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6052 	distribute_cfs_runtime(cfs_b);  in do_sched_cfs_slack_timer()
6117 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
6120 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
6129 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
6136 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in sched_cfs_period_timer()
6138 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
6142 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);  in sched_cfs_period_timer()
6145 			u64 new, old = ktime_to_ns(cfs_b->period);  in sched_cfs_period_timer()
6154 				cfs_b->period = ns_to_ktime(new);  in sched_cfs_period_timer()
6155 				cfs_b->quota *= 2;  in sched_cfs_period_timer()
6156 				cfs_b->burst *= 2;  in sched_cfs_period_timer()
6162 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
6168 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
6176 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
6177 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in sched_cfs_period_timer()
6182 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent)  in init_cfs_bandwidth()  argument
6184 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
6185 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
6186 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
6187 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
6188 	cfs_b->burst = 0;  in init_cfs_bandwidth()
6189 	cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF;  in init_cfs_bandwidth()
6191 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
6192 	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);  in init_cfs_bandwidth()
6193 	cfs_b->period_timer.function = sched_cfs_period_timer;  in init_cfs_bandwidth()
6196 	hrtimer_set_expires(&cfs_b->period_timer,  in init_cfs_bandwidth()
6197 			    get_random_u32_below(cfs_b->period));  in init_cfs_bandwidth()
6198 	hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in init_cfs_bandwidth()
6199 	cfs_b->slack_timer.function = sched_cfs_slack_timer;  in init_cfs_bandwidth()
6200 	cfs_b->slack_started = false;  in init_cfs_bandwidth()
6212 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
6214 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
6216 	if (cfs_b->period_active)  in start_cfs_bandwidth()
6219 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
6220 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
6221 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
6224 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
6229 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
6232 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
6233 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
6276 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
6279 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
6280 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
6281 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
6396 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {}  in init_cfs_bandwidth()  argument
6404 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument