88 struct rq;
100 extern void calc_global_load_tick(struct rq *this_rq);
101 extern long calc_load_fold_active(struct rq *this_rq, long adjust);
103 extern void call_trace_sched_update_nr_running(struct rq *rq, int count);
595 	struct rq		*rq;	/* CPU runqueue to which this cfs_rq is attached */  member
667 	struct rq		*rq;  member
862 extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
922 struct rq {  struct
1100 	struct rq		*core;  argument
1118 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)  in rq_of()  argument
1120 	return cfs_rq->rq;  in rq_of()
1125 static inline struct rq *rq_of(struct cfs_rq *cfs_rq)  in rq_of()
1127 	return container_of(cfs_rq, struct rq, cfs);  in rq_of()
1131 static inline int cpu_of(struct rq *rq)  in cpu_of()  argument
1134 	return rq->cpu;  in cpu_of()
1157 static inline bool sched_core_enabled(struct rq *rq)  in sched_core_enabled()  argument
1159 	return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled;  in sched_core_enabled()
1171 static inline raw_spinlock_t *rq_lockp(struct rq *rq)  in rq_lockp()  argument
1173 	if (sched_core_enabled(rq))  in rq_lockp()
1174 		return &rq->core->__lock;  in rq_lockp()
1176 	return &rq->__lock;  in rq_lockp()
1179 static inline raw_spinlock_t *__rq_lockp(struct rq *rq)  in __rq_lockp()  argument
1181 	if (rq->core_enabled)  in __rq_lockp()
1182 		return &rq->core->__lock;  in __rq_lockp()
1184 	return &rq->__lock;  in __rq_lockp()
1195 static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p)  in sched_cpu_cookie_match()  argument
1198 	if (!sched_core_enabled(rq))  in sched_cpu_cookie_match()
1201 	return rq->core->core_cookie == p->core_cookie;  in sched_cpu_cookie_match()
1204 static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)  in sched_core_cookie_match()  argument
1210 	if (!sched_core_enabled(rq))  in sched_core_cookie_match()
1213 	for_each_cpu(cpu, cpu_smt_mask(cpu_of(rq))) {  in sched_core_cookie_match()
1224 	return idle_core || rq->core->core_cookie == p->core_cookie;  in sched_core_cookie_match()
1227 static inline bool sched_group_cookie_match(struct rq *rq,  in sched_group_cookie_match()  argument
1234 	if (!sched_core_enabled(rq))  in sched_group_cookie_match()
1238 		if (sched_core_cookie_match(rq, p))  in sched_group_cookie_match()
1244 extern void queue_core_balance(struct rq *rq);
1251 extern void sched_core_enqueue(struct rq *rq, struct task_struct *p);
1252 extern void sched_core_dequeue(struct rq *rq, struct task_struct *p);
1264 static inline bool sched_core_enabled(struct rq *rq)  in sched_core_enabled()  argument
1274 static inline raw_spinlock_t *rq_lockp(struct rq *rq)  in rq_lockp()  argument
1276 	return &rq->__lock;  in rq_lockp()
1279 static inline raw_spinlock_t *__rq_lockp(struct rq *rq)  in __rq_lockp()  argument
1281 	return &rq->__lock;  in __rq_lockp()
1284 static inline void queue_core_balance(struct rq *rq)  in queue_core_balance()  argument
1288 static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p)  in sched_cpu_cookie_match()  argument
1293 static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)  in sched_core_cookie_match()  argument
1298 static inline bool sched_group_cookie_match(struct rq *rq,  in sched_group_cookie_match()  argument
1306 static inline void lockdep_assert_rq_held(struct rq *rq)  in lockdep_assert_rq_held()  argument
1308 	lockdep_assert_held(__rq_lockp(rq));  in lockdep_assert_rq_held()
1311 extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass);
1312 extern bool raw_spin_rq_trylock(struct rq *rq);
1313 extern void raw_spin_rq_unlock(struct rq *rq);
1315 static inline void raw_spin_rq_lock(struct rq *rq)  in raw_spin_rq_lock()  argument
1317 	raw_spin_rq_lock_nested(rq, 0);  in raw_spin_rq_lock()
1320 static inline void raw_spin_rq_lock_irq(struct rq *rq)  in raw_spin_rq_lock_irq()  argument
1323 	raw_spin_rq_lock(rq);  in raw_spin_rq_lock_irq()
1326 static inline void raw_spin_rq_unlock_irq(struct rq *rq)  in raw_spin_rq_unlock_irq()  argument
1328 	raw_spin_rq_unlock(rq);  in raw_spin_rq_unlock_irq()
1332 static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq)  in _raw_spin_rq_lock_irqsave()  argument
1336 	raw_spin_rq_lock(rq);  in _raw_spin_rq_lock_irqsave()
1340 static inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags)  in raw_spin_rq_unlock_irqrestore()  argument
1342 	raw_spin_rq_unlock(rq);  in raw_spin_rq_unlock_irqrestore()
1346 #define raw_spin_rq_lock_irqsave(rq, flags)	\  argument
1348 	flags = _raw_spin_rq_lock_irqsave(rq);	\
1352 extern void __update_idle_core(struct rq *rq);
1354 static inline void update_idle_core(struct rq *rq)  in update_idle_core()  argument
1357 		__update_idle_core(rq);  in update_idle_core()
1361 static inline void update_idle_core(struct rq *rq) { }  in update_idle_core()  argument
1364 DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
1411 	struct rq *rq = task_rq(p);  in cfs_rq_of()  local
1413 	return &rq->cfs;  in cfs_rq_of()
1423 extern void update_rq_clock(struct rq *rq);
1425 static inline u64 __rq_clock_broken(struct rq *rq)  in __rq_clock_broken()  argument
1427 	return READ_ONCE(rq->clock);  in __rq_clock_broken()
1457 static inline void assert_clock_updated(struct rq *rq)  in assert_clock_updated()  argument
1463 	SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP);  in assert_clock_updated()
1466 static inline u64 rq_clock(struct rq *rq)  in rq_clock()  argument
1468 	lockdep_assert_rq_held(rq);  in rq_clock()
1469 	assert_clock_updated(rq);  in rq_clock()
1471 	return rq->clock;  in rq_clock()
1474 static inline u64 rq_clock_task(struct rq *rq)  in rq_clock_task()  argument
1476 	lockdep_assert_rq_held(rq);  in rq_clock_task()
1477 	assert_clock_updated(rq);  in rq_clock_task()
1479 	return rq->clock_task;  in rq_clock_task()
1495 static inline u64 rq_clock_thermal(struct rq *rq)  in rq_clock_thermal()  argument
1497 	return rq_clock_task(rq) >> sched_thermal_decay_shift;  in rq_clock_thermal()
1500 static inline void rq_clock_skip_update(struct rq *rq)  in rq_clock_skip_update()  argument
1502 	lockdep_assert_rq_held(rq);  in rq_clock_skip_update()
1503 	rq->clock_update_flags |= RQCF_REQ_SKIP;  in rq_clock_skip_update()
1510 static inline void rq_clock_cancel_skipupdate(struct rq *rq)  in rq_clock_cancel_skipupdate()  argument
1512 	lockdep_assert_rq_held(rq);  in rq_clock_cancel_skipupdate()
1513 	rq->clock_update_flags &= ~RQCF_REQ_SKIP;  in rq_clock_cancel_skipupdate()
1541 static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf)  in rq_pin_lock()  argument
1543 	rf->cookie = lockdep_pin_lock(__rq_lockp(rq));  in rq_pin_lock()
1546 	rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);  in rq_pin_lock()
1549 	SCHED_WARN_ON(rq->balance_callback && rq->balance_callback != &balance_push_callback);  in rq_pin_lock()
1554 static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)  in rq_unpin_lock()  argument
1557 	if (rq->clock_update_flags > RQCF_ACT_SKIP)  in rq_unpin_lock()
1561 	lockdep_unpin_lock(__rq_lockp(rq), rf->cookie);  in rq_unpin_lock()
1564 static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf)  in rq_repin_lock()  argument
1566 	lockdep_repin_lock(__rq_lockp(rq), rf->cookie);  in rq_repin_lock()
1572 	rq->clock_update_flags |= rf->clock_update_flags;  in rq_repin_lock()
1576 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
1577 	__acquires(rq->lock);
1579 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
1581 	__acquires(rq->lock);
1583 static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)  in __task_rq_unlock()  argument
1584 	__releases(rq->lock)  in __task_rq_unlock()
1586 	rq_unpin_lock(rq, rf);  in __task_rq_unlock()
1587 	raw_spin_rq_unlock(rq);  in __task_rq_unlock()
1591 task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)  in task_rq_unlock()  argument
1592 	__releases(rq->lock)  in task_rq_unlock()
1595 	rq_unpin_lock(rq, rf);  in task_rq_unlock()
1596 	raw_spin_rq_unlock(rq);  in task_rq_unlock()
1601 rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)  in rq_lock_irqsave()  argument
1602 	__acquires(rq->lock)  in rq_lock_irqsave()
1604 	raw_spin_rq_lock_irqsave(rq, rf->flags);  in rq_lock_irqsave()
1605 	rq_pin_lock(rq, rf);  in rq_lock_irqsave()
1609 rq_lock_irq(struct rq *rq, struct rq_flags *rf)  in rq_lock_irq()  argument
1610 	__acquires(rq->lock)  in rq_lock_irq()
1612 	raw_spin_rq_lock_irq(rq);  in rq_lock_irq()
1613 	rq_pin_lock(rq, rf);  in rq_lock_irq()
1617 rq_lock(struct rq *rq, struct rq_flags *rf)  in rq_lock()  argument
1618 	__acquires(rq->lock)  in rq_lock()
1620 	raw_spin_rq_lock(rq);  in rq_lock()
1621 	rq_pin_lock(rq, rf);  in rq_lock()
1625 rq_relock(struct rq *rq, struct rq_flags *rf)  in rq_relock()  argument
1626 	__acquires(rq->lock)  in rq_relock()
1628 	raw_spin_rq_lock(rq);  in rq_relock()
1629 	rq_repin_lock(rq, rf);  in rq_relock()
1633 rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf)  in rq_unlock_irqrestore()  argument
1634 	__releases(rq->lock)  in rq_unlock_irqrestore()
1636 	rq_unpin_lock(rq, rf);  in rq_unlock_irqrestore()
1637 	raw_spin_rq_unlock_irqrestore(rq, rf->flags);  in rq_unlock_irqrestore()
1641 rq_unlock_irq(struct rq *rq, struct rq_flags *rf)  in rq_unlock_irq()  argument
1642 	__releases(rq->lock)  in rq_unlock_irq()
1644 	rq_unpin_lock(rq, rf);  in rq_unlock_irq()
1645 	raw_spin_rq_unlock_irq(rq);  in rq_unlock_irq()
1649 rq_unlock(struct rq *rq, struct rq_flags *rf)  in rq_unlock()  argument
1650 	__releases(rq->lock)  in rq_unlock()
1652 	rq_unpin_lock(rq, rf);  in rq_unlock()
1653 	raw_spin_rq_unlock(rq);  in rq_unlock()
1656 static inline struct rq *
1658 	__acquires(rq->lock)  in this_rq_lock_irq()
1660 	struct rq *rq;  in this_rq_lock_irq()  local
1663 	rq = this_rq();  in this_rq_lock_irq()
1664 	rq_lock(rq, rf);  in this_rq_lock_irq()
1665 	return rq;  in this_rq_lock_irq()
1714 queue_balance_callback(struct rq *rq,  in queue_balance_callback()  argument
1716 		       void (*func)(struct rq *rq))  in queue_balance_callback()  argument
1718 	lockdep_assert_rq_held(rq);  in queue_balance_callback()
1720 	if (unlikely(head->next || rq->balance_callback == &balance_push_callback))  in queue_balance_callback()
1724 	head->next = rq->balance_callback;  in queue_balance_callback()
1725 	rq->balance_callback = head;  in queue_balance_callback()
2019 static inline int task_current(struct rq *rq, struct task_struct *p)  in task_current()  argument
2021 	return rq->curr == p;  in task_current()
2024 static inline int task_running(struct rq *rq, struct task_struct *p)  in task_running()  argument
2029 	return task_current(rq, p);  in task_running()
2118 	void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);
2119 	void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);
2120 	void (*yield_task)   (struct rq *rq);
2121 	bool (*yield_to_task)(struct rq *rq, struct task_struct *p);
2123 	void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags);
2125 	struct task_struct *(*pick_next_task)(struct rq *rq);
2127 	void (*put_prev_task)(struct rq *rq, struct task_struct *p);
2128 	void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
2131 	int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
2134 	struct task_struct * (*pick_task)(struct rq *rq);
2138 	void (*task_woken)(struct rq *this_rq, struct task_struct *task);
2144 	void (*rq_online)(struct rq *rq);
2145 	void (*rq_offline)(struct rq *rq);
2147 	struct rq *(*find_lock_rq)(struct task_struct *p, struct rq *rq);
2150 	void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);
2159 	void (*switched_from)(struct rq *this_rq, struct task_struct *task);
2160 	void (*switched_to)  (struct rq *this_rq, struct task_struct *task);
2161 	void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
2164 	unsigned int (*get_rr_interval)(struct rq *rq,
2167 	void (*update_curr)(struct rq *rq);
2177 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)  in put_prev_task()  argument
2179 	WARN_ON_ONCE(rq->curr != prev);  in put_prev_task()
2180 	prev->sched_class->put_prev_task(rq, prev);  in put_prev_task()
2183 static inline void set_next_task(struct rq *rq, struct task_struct *next)  in set_next_task()  argument
2185 	next->sched_class->set_next_task(rq, next, false);  in set_next_task()
2221 static inline bool sched_stop_runnable(struct rq *rq)  in sched_stop_runnable()  argument
2223 	return rq->stop && task_on_rq_queued(rq->stop);  in sched_stop_runnable()
2226 static inline bool sched_dl_runnable(struct rq *rq)  in sched_dl_runnable()  argument
2228 	return rq->dl.dl_nr_running > 0;  in sched_dl_runnable()
2231 static inline bool sched_rt_runnable(struct rq *rq)  in sched_rt_runnable()  argument
2233 	return rq->rt.rt_queued > 0;  in sched_rt_runnable()
2236 static inline bool sched_fair_runnable(struct rq *rq)  in sched_fair_runnable()  argument
2238 	return rq->cfs.nr_running > 0;  in sched_fair_runnable()
2241 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_f…
2242 extern struct task_struct *pick_next_task_idle(struct rq *rq);
2253 extern void trigger_load_balance(struct rq *rq);
2257 static inline struct task_struct *get_push_task(struct rq *rq)  in get_push_task()  argument
2259 	struct task_struct *p = rq->curr;  in get_push_task()
2261 	lockdep_assert_rq_held(rq);  in get_push_task()
2263 	if (rq->push_busy)  in get_push_task()
2272 	rq->push_busy = true;  in get_push_task()
2281 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
2284 	rq->idle_state = idle_state;  in idle_set_state()
2287 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
2291 	return rq->idle_state;  in idle_get_state()
2294 static inline void idle_set_state(struct rq *rq,  in idle_set_state()  argument
2299 static inline struct cpuidle_state *idle_get_state(struct rq *rq)  in idle_get_state()  argument
2317 extern void resched_curr(struct rq *rq);
2339 extern bool sched_can_stop_tick(struct rq *rq);
2347 static inline void sched_update_tick_dependency(struct rq *rq)  in sched_update_tick_dependency()  argument
2349 	int cpu = cpu_of(rq);  in sched_update_tick_dependency()
2354 	if (sched_can_stop_tick(rq))  in sched_update_tick_dependency()
2361 static inline void sched_update_tick_dependency(struct rq *rq) { }  in sched_update_tick_dependency()  argument
2364 static inline void add_nr_running(struct rq *rq, unsigned count)  in add_nr_running()  argument
2366 	unsigned prev_nr = rq->nr_running;  in add_nr_running()
2368 	rq->nr_running = prev_nr + count;  in add_nr_running()
2370 		call_trace_sched_update_nr_running(rq, count);  in add_nr_running()
2374 	if (prev_nr < 2 && rq->nr_running >= 2) {  in add_nr_running()
2375 		if (!READ_ONCE(rq->rd->overload))  in add_nr_running()
2376 			WRITE_ONCE(rq->rd->overload, 1);  in add_nr_running()
2380 	sched_update_tick_dependency(rq);  in add_nr_running()
2383 static inline void sub_nr_running(struct rq *rq, unsigned count)  in sub_nr_running()  argument
2385 	rq->nr_running -= count;  in sub_nr_running()
2387 		call_trace_sched_update_nr_running(rq, -count);  in sub_nr_running()
2391 	sched_update_tick_dependency(rq);  in sub_nr_running()
2394 extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
2395 extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
2397 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
2424 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
2426 	if (!cpu_active(cpu_of(rq)))  in hrtick_enabled()
2428 	return hrtimer_is_hres_active(&rq->hrtick_timer);  in hrtick_enabled()
2431 static inline int hrtick_enabled_fair(struct rq *rq)  in hrtick_enabled_fair()  argument
2435 	return hrtick_enabled(rq);  in hrtick_enabled_fair()
2438 static inline int hrtick_enabled_dl(struct rq *rq)  in hrtick_enabled_dl()  argument
2442 	return hrtick_enabled(rq);  in hrtick_enabled_dl()
2445 void hrtick_start(struct rq *rq, u64 delay);
2449 static inline int hrtick_enabled_fair(struct rq *rq)  in hrtick_enabled_fair()  argument
2454 static inline int hrtick_enabled_dl(struct rq *rq)  in hrtick_enabled_dl()  argument
2459 static inline int hrtick_enabled(struct rq *rq)  in hrtick_enabled()  argument
2494 static inline bool rq_order_less(struct rq *rq1, struct rq *rq2)  in rq_order_less()
2520 extern void double_rq_lock(struct rq *rq1, struct rq *rq2);
2532 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
2551 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)  in _double_lock_balance()
2578 static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)  in double_lock_balance()
2585 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)  in double_unlock_balance()
2626 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()
2637 extern void set_rq_online (struct rq *rq);
2638 extern void set_rq_offline(struct rq *rq);
2649 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)  in double_rq_lock()
2665 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)  in double_rq_unlock()
2721 extern void nohz_balance_exit_idle(struct rq *rq);
2723 static inline void nohz_balance_exit_idle(struct rq *rq) { }  in nohz_balance_exit_idle()  argument
2742 		struct rq *rq = cpu_rq(i);  in __dl_update()  local
2744 		rq->dl.extra_bw += bw;  in __dl_update()
2813 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)  in cpufreq_update_util()  argument
2818 						  cpu_of(rq)));  in cpufreq_update_util()
2820 		data->func(data, rq_clock(rq), flags);  in cpufreq_update_util()
2823 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}  in cpufreq_update_util()  argument
2847 unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,  in uclamp_rq_util_with()  argument
2864 		if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)  in uclamp_rq_util_with()
2868 	min_util = max_t(unsigned long, min_util, READ_ONCE(rq->uclamp[UCLAMP_MIN].value));  in uclamp_rq_util_with()
2869 	max_util = max_t(unsigned long, max_util, READ_ONCE(rq->uclamp[UCLAMP_MAX].value));  in uclamp_rq_util_with()
2896 unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,  in uclamp_rq_util_with()  argument
2941 static inline unsigned long cpu_bw_dl(struct rq *rq)  in cpu_bw_dl()  argument
2943 	return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT;  in cpu_bw_dl()
2946 static inline unsigned long cpu_util_dl(struct rq *rq)  in cpu_util_dl()  argument
2948 	return READ_ONCE(rq->avg_dl.util_avg);  in cpu_util_dl()
2951 static inline unsigned long cpu_util_cfs(struct rq *rq)  in cpu_util_cfs()  argument
2953 	unsigned long util = READ_ONCE(rq->cfs.avg.util_avg);  in cpu_util_cfs()
2957 			     READ_ONCE(rq->cfs.avg.util_est.enqueued));  in cpu_util_cfs()
2963 static inline unsigned long cpu_util_rt(struct rq *rq)  in cpu_util_rt()  argument
2965 	return READ_ONCE(rq->avg_rt.util_avg);  in cpu_util_rt()
2970 static inline unsigned long cpu_util_irq(struct rq *rq)  in cpu_util_irq()  argument
2972 	return rq->avg_irq.util_avg;  in cpu_util_irq()
2985 static inline unsigned long cpu_util_irq(struct rq *rq)  in cpu_util_irq()  argument
3022 static inline void membarrier_switch_mm(struct rq *rq,  in membarrier_switch_mm()  argument
3032 	if (READ_ONCE(rq->membarrier_state) == membarrier_state)  in membarrier_switch_mm()
3035 	WRITE_ONCE(rq->membarrier_state, membarrier_state);  in membarrier_switch_mm()
3038 static inline void membarrier_switch_mm(struct rq *rq,  in membarrier_switch_mm()  argument