84 struct rq;
96 extern void calc_global_load_tick(struct rq *this_rq);
97 extern long calc_load_fold_active(struct rq *this_rq, long adjust);
552 	struct rq		*rq;	/* CPU runqueue to which this cfs_rq is attached */  member
621 	struct rq		*rq;  member
789 extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
847 struct rq {  struct
1007 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)  in rq_of()  argument
1009 	return cfs_rq->rq;  in rq_of()
1014 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)  in rq_of()
1016 	return container_of(cfs_rq, struct rq, cfs);  in rq_of()
1020 static inline int cpu_of(struct rq *rq)  in cpu_of()  argument
1023 	return rq->cpu;  in cpu_of()
1031 extern void __update_idle_core(struct rq *rq);
1033 static inline void update_idle_core(struct rq *rq)  in update_idle_core()  argument
1036 		__update_idle_core(rq);  in update_idle_core()
1040 static inline void update_idle_core(struct rq *rq) { }  in update_idle_core()  argument
1043 DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
1051 extern void update_rq_clock(struct rq *rq);
1053 static inline u64 __rq_clock_broken(struct rq *rq)  in __rq_clock_broken()  argument
1055 	return READ_ONCE(rq->clock);  in __rq_clock_broken()
1085 static inline void assert_clock_updated(struct rq *rq)  in assert_clock_updated()  argument
1091 	SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP);  in assert_clock_updated()
1094 static inline u64 rq_clock(struct rq *rq)  in rq_clock()  argument
1096 	lockdep_assert_held(&rq->lock);  in rq_clock()
1097 	assert_clock_updated(rq);  in rq_clock()
1099 	return rq->clock;  in rq_clock()
1102 static inline u64 rq_clock_task(struct rq *rq)  in rq_clock_task()  argument
1104 	lockdep_assert_held(&rq->lock);  in rq_clock_task()
1105 	assert_clock_updated(rq);  in rq_clock_task()
1107 	return rq->clock_task;  in rq_clock_task()
1110 static inline void rq_clock_skip_update(struct rq *rq)  in rq_clock_skip_update()  argument
1112 	lockdep_assert_held(&rq->lock);  in rq_clock_skip_update()
1113 	rq->clock_update_flags |= RQCF_REQ_SKIP;  in rq_clock_skip_update()
1120 static inline void rq_clock_cancel_skipupdate(struct rq *rq)  in rq_clock_cancel_skipupdate()  argument
1122 	lockdep_assert_held(&rq->lock);  in rq_clock_cancel_skipupdate()
1123 	rq->clock_update_flags &= ~RQCF_REQ_SKIP;  in rq_clock_cancel_skipupdate()
1139 static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf)  in rq_pin_lock()  argument
1141 	rf->cookie = lockdep_pin_lock(&rq->lock);  in rq_pin_lock()
1144 	rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);  in rq_pin_lock()
1149 static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)  in rq_unpin_lock()  argument
1152 	if (rq->clock_update_flags > RQCF_ACT_SKIP)  in rq_unpin_lock()
1156 	lockdep_unpin_lock(&rq->lock, rf->cookie);  in rq_unpin_lock()
1159 static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf)  in rq_repin_lock()  argument
1161 	lockdep_repin_lock(&rq->lock, rf->cookie);  in rq_repin_lock()
1167 	rq->clock_update_flags |= rf->clock_update_flags;  in rq_repin_lock()
1171 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
1172 	__acquires(rq->lock);
1174 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
1176 	__acquires(rq->lock);
1178 static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)  in __task_rq_unlock()  argument
1179 	__releases(rq->lock)  in __task_rq_unlock()
1181 	rq_unpin_lock(rq, rf);  in __task_rq_unlock()
1182 	raw_spin_unlock(&rq->lock);  in __task_rq_unlock()
1186 task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)  in task_rq_unlock()  argument
1187 	__releases(rq->lock)  in task_rq_unlock()
1190 	rq_unpin_lock(rq, rf);  in task_rq_unlock()
1191 	raw_spin_unlock(&rq->lock);  in task_rq_unlock()
1196 rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)  in rq_lock_irqsave()  argument
1197 	__acquires(rq->lock)  in rq_lock_irqsave()
1199 	raw_spin_lock_irqsave(&rq->lock, rf->flags);  in rq_lock_irqsave()
1200 	rq_pin_lock(rq, rf);  in rq_lock_irqsave()
1204 rq_lock_irq(struct rq *rq, struct rq_flags *rf)  in rq_lock_irq()  argument
1205 	__acquires(rq->lock)  in rq_lock_irq()
1207 	raw_spin_lock_irq(&rq->lock);  in rq_lock_irq()
1208 	rq_pin_lock(rq, rf);  in rq_lock_irq()
1212 rq_lock(struct rq *rq, struct rq_flags *rf)  in rq_lock()  argument
1213 	__acquires(rq->lock)  in rq_lock()
1215 	raw_spin_lock(&rq->lock);  in rq_lock()
1216 	rq_pin_lock(rq, rf);  in rq_lock()
1220 rq_relock(struct rq *rq, struct rq_flags *rf)  in rq_relock()  argument
1221 	__acquires(rq->lock)  in rq_relock()
1223 	raw_spin_lock(&rq->lock);  in rq_relock()
1224 	rq_repin_lock(rq, rf);  in rq_relock()
1228 rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf)  in rq_unlock_irqrestore()  argument
1229 	__releases(rq->lock)  in rq_unlock_irqrestore()
1231 	rq_unpin_lock(rq, rf);  in rq_unlock_irqrestore()
1232 	raw_spin_unlock_irqrestore(&rq->lock, rf->flags);  in rq_unlock_irqrestore()
1236 rq_unlock_irq(struct rq *rq, struct rq_flags *rf)  in rq_unlock_irq()  argument
1237 	__releases(rq->lock)  in rq_unlock_irq()
1239 	rq_unpin_lock(rq, rf);  in rq_unlock_irq()
1240 	raw_spin_unlock_irq(&rq->lock);  in rq_unlock_irq()
1244 rq_unlock(struct rq *rq, struct rq_flags *rf)  in rq_unlock()  argument
1245 	__releases(rq->lock)  in rq_unlock()
1247 	rq_unpin_lock(rq, rf);  in rq_unlock()
1248 	raw_spin_unlock(&rq->lock);  in rq_unlock()
1251 static inline struct rq *
1253 	__acquires(rq->lock)  in this_rq_lock_irq()
1255 	struct rq *rq;  in this_rq_lock_irq()  local
1258 	rq = this_rq();  in this_rq_lock_irq()
1259 	rq_lock(rq, rf);  in this_rq_lock_irq()
1260 	return rq;  in this_rq_lock_irq()
1309 queue_balance_callback(struct rq *rq,  in queue_balance_callback()  argument
1311 		       void (*func)(struct rq *rq))  in queue_balance_callback()  argument
1313 	lockdep_assert_held(&rq->lock);  in queue_balance_callback()
1319 	head->next = rq->balance_callback;  in queue_balance_callback()
1320 	rq->balance_callback = head;  in queue_balance_callback()
1462 extern int newidle_balance(struct rq *this_rq, struct rq_flags *rf);
1468 static inline int newidle_balance(struct rq *this_rq, struct rq_flags *rf) { return 0; }  in newidle_balance()
1617 static inline int task_current(struct rq *rq, struct task_struct *p)  in task_current()  argument
1619 	return rq->curr == p;  in task_current()
1622 static inline int task_running(struct rq *rq, struct task_struct *p)  in task_running()  argument
1627 	return task_current(rq, p);  in task_running()
1709 	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);
1710 	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);
1711 	void (*yield_task)   (struct rq *rq);
1712 	bool (*yield_to_task)(struct rq *rq, struct task_struct *p, bool preempt);
1714 	void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags);
1727 	struct task_struct * (*pick_next_task)(struct rq *rq,
1730 	void (*put_prev_task)(struct rq *rq, struct task_struct *p);
1731 	void (*set_next_task)(struct rq *rq, struct task_struct *p);
1734 	int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
1738 	void (*task_woken)(struct rq *this_rq, struct task_struct *task);
1743 	void (*rq_online)(struct rq *rq);
1744 	void (*rq_offline)(struct rq *rq);
1747 	void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);
1756 	void (*switched_from)(struct rq *this_rq, struct task_struct *task);
1757 	void (*switched_to)  (struct rq *this_rq, struct task_struct *task);
1758 	void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
1761 	unsigned int (*get_rr_interval)(struct rq *rq,
1764 	void (*update_curr)(struct rq *rq);
1774 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)  in put_prev_task()  argument
1776 	WARN_ON_ONCE(rq->curr != prev);  in put_prev_task()
1777 	prev->sched_class->put_prev_task(rq, prev);  in put_prev_task()
1780 static inline void set_next_task(struct rq *rq, struct task_struct *next)  in set_next_task()  argument
1782 	WARN_ON_ONCE(rq->curr != next);  in set_next_task()
1783 	next->sched_class->set_next_task(rq, next);  in set_next_task()
1804 static inline bool sched_stop_runnable(struct rq *rq)  in sched_stop_runnable()  argument
1806 	return rq->stop && task_on_rq_queued(rq->stop);  in sched_stop_runnable()
1809 static inline bool sched_dl_runnable(struct rq *rq)  in sched_dl_runnable()  argument
1811 	return rq->dl.dl_nr_running > 0;  in sched_dl_runnable()
1814 static inline bool sched_rt_runnable(struct rq *rq)  in sched_rt_runnable()  argument
1816 	return rq->rt.rt_queued > 0;  in sched_rt_runnable()
1819 static inline bool sched_fair_runnable(struct rq *rq)  in sched_fair_runnable()  argument
1821 	return rq->cfs.nr_running > 0;  in sched_fair_runnable()
1828 extern void trigger_load_balance(struct rq *rq);
1835 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
1838 	rq->idle_state = idle_state;  in idle_set_state()
1841 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
1845 	return rq->idle_state;  in idle_get_state()
1848 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
1853 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
1871 extern void resched_curr(struct rq *rq);
1892 extern bool sched_can_stop_tick(struct rq *rq);
1900 static inline void sched_update_tick_dependency(struct rq *rq)  in sched_update_tick_dependency()  argument
1907 	cpu = cpu_of(rq);  in sched_update_tick_dependency()
1912 	if (sched_can_stop_tick(rq))  in sched_update_tick_dependency()
1919 static inline void sched_update_tick_dependency(struct rq *rq) { }  in sched_update_tick_dependency()  argument
1922 static inline void add_nr_running(struct rq *rq, unsigned count)  in add_nr_running()  argument
1924 	unsigned prev_nr = rq->nr_running;  in add_nr_running()
1926 	rq->nr_running = prev_nr + count;  in add_nr_running()
1929 	if (prev_nr < 2 && rq->nr_running >= 2) {  in add_nr_running()
1930 		if (!READ_ONCE(rq->rd->overload))  in add_nr_running()
1931 			WRITE_ONCE(rq->rd->overload, 1);  in add_nr_running()
1935 	sched_update_tick_dependency(rq);  in add_nr_running()
1938 static inline void sub_nr_running(struct rq *rq, unsigned count)  in sub_nr_running()  argument
1940 	rq->nr_running -= count;  in sub_nr_running()
1942 	sched_update_tick_dependency(rq);  in sub_nr_running()
1945 extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
1946 extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
1948 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
1960 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
1964 	if (!cpu_active(cpu_of(rq)))  in hrtick_enabled()
1966 	return hrtimer_is_hres_active(&rq->hrtick_timer);  in hrtick_enabled()
1969 void hrtick_start(struct rq *rq, u64 delay);
1973 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
1991 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2);
2001 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
2020 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
2046 static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)  in double_lock_balance()
2057 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)  in double_unlock_balance()
2097 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)  in double_rq_lock()
2122 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()
2133 extern void set_rq_online (struct rq *rq);
2134 extern void set_rq_offline(struct rq *rq);
2145 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)  in double_rq_lock()
2161 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()
2211 extern void nohz_balance_exit_idle(struct rq *rq);
2213 static inline void nohz_balance_exit_idle(struct rq *rq) { }  in nohz_balance_exit_idle()  argument
2227 		struct rq *rq = cpu_rq(i);  in __dl_update()  local
2229 		rq->dl.extra_bw += bw;  in __dl_update()
2298 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)  in cpufreq_update_util()  argument
2303 						  cpu_of(rq)));  in cpufreq_update_util()
2305 		data->func(data, rq_clock(rq), flags);  in cpufreq_update_util()
2308 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}  in cpufreq_update_util()  argument
2315 unsigned int uclamp_util_with(struct rq *rq, unsigned int util,  in uclamp_util_with()  argument
2318 	unsigned int min_util = READ_ONCE(rq->uclamp[UCLAMP_MIN].value);  in uclamp_util_with()
2319 	unsigned int max_util = READ_ONCE(rq->uclamp[UCLAMP_MAX].value);  in uclamp_util_with()
2337 static inline unsigned int uclamp_util(struct rq *rq, unsigned int util)  in uclamp_util()  argument
2339 	return uclamp_util_with(rq, util, NULL);  in uclamp_util()
2342 static inline unsigned int uclamp_util_with(struct rq *rq, unsigned int util,  in uclamp_util_with()  argument
2347 static inline unsigned int uclamp_util(struct rq *rq, unsigned int util)  in uclamp_util()  argument
2389 static inline unsigned long cpu_bw_dl(struct rq *rq)  in cpu_bw_dl()  argument
2391 	return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT;  in cpu_bw_dl()
2394 static inline unsigned long cpu_util_dl(struct rq *rq)  in cpu_util_dl()  argument
2396 	return READ_ONCE(rq->avg_dl.util_avg);  in cpu_util_dl()
2399 static inline unsigned long cpu_util_cfs(struct rq *rq)  in cpu_util_cfs()  argument
2401 	unsigned long util = READ_ONCE(rq->cfs.avg.util_avg);  in cpu_util_cfs()
2405 			     READ_ONCE(rq->cfs.avg.util_est.enqueued));  in cpu_util_cfs()
2411 static inline unsigned long cpu_util_rt(struct rq *rq)  in cpu_util_rt()  argument
2413 	return READ_ONCE(rq->avg_rt.util_avg);  in cpu_util_rt()
2425 static inline unsigned long cpu_util_irq(struct rq *rq)  in cpu_util_irq()  argument
2427 	return rq->avg_irq.util_avg;  in cpu_util_irq()
2440 static inline unsigned long cpu_util_irq(struct rq *rq)  in cpu_util_irq()  argument
2477 static inline void membarrier_switch_mm(struct rq *rq,  in membarrier_switch_mm()  argument
2487 	if (READ_ONCE(rq->membarrier_state) == membarrier_state)  in membarrier_switch_mm()
2490 	WRITE_ONCE(rq->membarrier_state, membarrier_state);  in membarrier_switch_mm()
2493 static inline void membarrier_switch_mm(struct rq *rq,  in membarrier_switch_mm()  argument