3618 static int idle_balance(struct rq *this_rq, struct rq_flags *rf);
5314 static void cpu_load_update(struct rq *this_rq, unsigned long this_load,  in cpu_load_update()  argument
5317 	unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0];  in cpu_load_update()
5320 	this_rq->nr_load_updates++;  in cpu_load_update()
5323 	this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */  in cpu_load_update()
5329 		old_load = this_rq->cpu_load[i];  in cpu_load_update()
5351 		this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;  in cpu_load_update()
5376 static void cpu_load_update_nohz(struct rq *this_rq,  in cpu_load_update_nohz()  argument
5382 	pending_updates = curr_jiffies - this_rq->last_load_update_tick;  in cpu_load_update_nohz()
5384 		this_rq->last_load_update_tick = curr_jiffies;  in cpu_load_update_nohz()
5390 		cpu_load_update(this_rq, load, pending_updates);  in cpu_load_update_nohz()
5398 static void cpu_load_update_idle(struct rq *this_rq)  in cpu_load_update_idle()  argument
5403 	if (weighted_cpuload(this_rq))  in cpu_load_update_idle()
5406 	cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0);  in cpu_load_update_idle()
5417 	struct rq *this_rq = this_rq();  in cpu_load_update_nohz_start()  local
5424 	this_rq->cpu_load[0] = weighted_cpuload(this_rq);  in cpu_load_update_nohz_start()
5433 	struct rq *this_rq = this_rq();  in cpu_load_update_nohz_stop()  local
5437 	if (curr_jiffies == this_rq->last_load_update_tick)  in cpu_load_update_nohz_stop()
5440 	load = weighted_cpuload(this_rq);  in cpu_load_update_nohz_stop()
5441 	rq_lock(this_rq, &rf);  in cpu_load_update_nohz_stop()
5442 	update_rq_clock(this_rq);  in cpu_load_update_nohz_stop()
5443 	cpu_load_update_nohz(this_rq, curr_jiffies, load);  in cpu_load_update_nohz_stop()
5444 	rq_unlock(this_rq, &rf);  in cpu_load_update_nohz_stop()
5447 static inline void cpu_load_update_nohz(struct rq *this_rq,  in cpu_load_update_nohz()  argument
5452 static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load)  in cpu_load_update_periodic()  argument
5456 	this_rq->last_load_update_tick = READ_ONCE(jiffies);  in cpu_load_update_periodic()
5458 	cpu_load_update(this_rq, load, 1);  in cpu_load_update_periodic()
5464 void cpu_load_update_active(struct rq *this_rq)  in cpu_load_update_active()  argument
5466 	unsigned long load = weighted_cpuload(this_rq);  in cpu_load_update_active()
5469 		cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load);  in cpu_load_update_active()
5471 		cpu_load_update_periodic(this_rq, load);  in cpu_load_update_active()
6080 	avg_idle = this_rq()->avg_idle / 512;  in select_idle_cpu()
8535 static int load_balance(int this_cpu, struct rq *this_rq,  in load_balance()  argument
8549 		.dst_rq		= this_rq,  in load_balance()
9197 	SCHED_WARN_ON(rq != this_rq());  in nohz_balance_exit_idle()
9292 static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags,  in _nohz_idle_balance()  argument
9300 	int this_cpu = this_rq->cpu;  in _nohz_idle_balance()
9366 		has_blocked_load |= this_rq->has_blocked_load;  in _nohz_idle_balance()
9370 		rebalance_domains(this_rq, CPU_IDLE);  in _nohz_idle_balance()
9398 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)  in nohz_idle_balance()  argument
9400 	int this_cpu = this_rq->cpu;  in nohz_idle_balance()
9418 	_nohz_idle_balance(this_rq, flags, idle);  in nohz_idle_balance()
9423 static void nohz_newidle_balance(struct rq *this_rq)  in nohz_newidle_balance()  argument
9425 	int this_cpu = this_rq->cpu;  in nohz_newidle_balance()
9435 	if (this_rq->avg_idle < sysctl_sched_migration_cost)  in nohz_newidle_balance()
9443 	raw_spin_unlock(&this_rq->lock);  in nohz_newidle_balance()
9450 	if (!_nohz_idle_balance(this_rq, NOHZ_STATS_KICK, CPU_NEWLY_IDLE))  in nohz_newidle_balance()
9452 	raw_spin_lock(&this_rq->lock);  in nohz_newidle_balance()
9458 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)  in nohz_idle_balance()  argument
9463 static inline void nohz_newidle_balance(struct rq *this_rq) { }  in nohz_newidle_balance()  argument
9470 static int idle_balance(struct rq *this_rq, struct rq_flags *rf)  in idle_balance()  argument
9473 	int this_cpu = this_rq->cpu;  in idle_balance()
9482 	this_rq->idle_stamp = rq_clock(this_rq);  in idle_balance()
9496 	rq_unpin_lock(this_rq, rf);  in idle_balance()
9498 	if (this_rq->avg_idle < sysctl_sched_migration_cost ||  in idle_balance()
9499 	    !this_rq->rd->overload) {  in idle_balance()
9502 		sd = rcu_dereference_check_sched_domain(this_rq->sd);  in idle_balance()
9507 		nohz_newidle_balance(this_rq);  in idle_balance()
9512 	raw_spin_unlock(&this_rq->lock);  in idle_balance()
9523 		if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) {  in idle_balance()
9531 			pulled_task = load_balance(this_cpu, this_rq,  in idle_balance()
9548 		if (pulled_task || this_rq->nr_running > 0)  in idle_balance()
9553 	raw_spin_lock(&this_rq->lock);  in idle_balance()
9555 	if (curr_cost > this_rq->max_idle_balance_cost)  in idle_balance()
9556 		this_rq->max_idle_balance_cost = curr_cost;  in idle_balance()
9564 	if (this_rq->cfs.h_nr_running && !pulled_task)  in idle_balance()
9568 	if (time_after(this_rq->next_balance, next_balance))  in idle_balance()
9569 		this_rq->next_balance = next_balance;  in idle_balance()
9572 	if (this_rq->nr_running != this_rq->cfs.h_nr_running)  in idle_balance()
9576 		this_rq->idle_stamp = 0;  in idle_balance()
9578 	rq_repin_lock(this_rq, rf);  in idle_balance()
9589 	struct rq *this_rq = this_rq();  in run_rebalance_domains()  local
9590 	enum cpu_idle_type idle = this_rq->idle_balance ?  in run_rebalance_domains()
9601 	if (nohz_idle_balance(this_rq, idle))  in run_rebalance_domains()
9605 	update_blocked_averages(this_rq->cpu);  in run_rebalance_domains()
9606 	rebalance_domains(this_rq, idle);  in run_rebalance_domains()
9672 	struct rq *rq = this_rq();  in task_fork_fair()