Lines Matching refs:rq
118 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
239 void sched_core_enqueue(struct rq *rq, struct task_struct *p) in sched_core_enqueue() argument
241 rq->core->core_task_seq++; in sched_core_enqueue()
246 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); in sched_core_enqueue()
249 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) in sched_core_dequeue() argument
251 rq->core->core_task_seq++; in sched_core_dequeue()
254 rb_erase(&p->core_node, &rq->core_tree); in sched_core_dequeue()
263 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && in sched_core_dequeue()
264 rq->core->core_forceidle_count && rq->curr == rq->idle) in sched_core_dequeue()
265 resched_curr(rq); in sched_core_dequeue()
299 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) in sched_core_find() argument
304 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); in sched_core_find()
309 if (!sched_task_is_throttled(p, rq->cpu)) in sched_core_find()
453 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } in sched_core_enqueue() argument
455 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } in sched_core_dequeue() argument
551 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) in raw_spin_rq_lock_nested() argument
558 raw_spin_lock_nested(&rq->__lock, subclass); in raw_spin_rq_lock_nested()
565 lock = __rq_lockp(rq); in raw_spin_rq_lock_nested()
567 if (likely(lock == __rq_lockp(rq))) { in raw_spin_rq_lock_nested()
576 bool raw_spin_rq_trylock(struct rq *rq) in raw_spin_rq_trylock() argument
584 ret = raw_spin_trylock(&rq->__lock); in raw_spin_rq_trylock()
590 lock = __rq_lockp(rq); in raw_spin_rq_trylock()
592 if (!ret || (likely(lock == __rq_lockp(rq)))) { in raw_spin_rq_trylock()
600 void raw_spin_rq_unlock(struct rq *rq) in raw_spin_rq_unlock() argument
602 raw_spin_unlock(rq_lockp(rq)); in raw_spin_rq_unlock()
609 void double_rq_lock(struct rq *rq1, struct rq *rq2) in double_rq_lock()
627 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) in __task_rq_lock()
628 __acquires(rq->lock) in __task_rq_lock()
630 struct rq *rq; in __task_rq_lock() local
635 rq = task_rq(p); in __task_rq_lock()
636 raw_spin_rq_lock(rq); in __task_rq_lock()
637 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { in __task_rq_lock()
638 rq_pin_lock(rq, rf); in __task_rq_lock()
639 return rq; in __task_rq_lock()
641 raw_spin_rq_unlock(rq); in __task_rq_lock()
651 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) in task_rq_lock()
653 __acquires(rq->lock) in task_rq_lock()
655 struct rq *rq; in task_rq_lock() local
659 rq = task_rq(p); in task_rq_lock()
660 raw_spin_rq_lock(rq); in task_rq_lock()
678 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { in task_rq_lock()
679 rq_pin_lock(rq, rf); in task_rq_lock()
680 return rq; in task_rq_lock()
682 raw_spin_rq_unlock(rq); in task_rq_lock()
694 static void update_rq_clock_task(struct rq *rq, s64 delta) in update_rq_clock_task() argument
703 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; in update_rq_clock_task()
723 rq->prev_irq_time += irq_delta; in update_rq_clock_task()
725 psi_account_irqtime(rq->curr, irq_delta); in update_rq_clock_task()
726 delayacct_irq(rq->curr, irq_delta); in update_rq_clock_task()
730 steal = paravirt_steal_clock(cpu_of(rq)); in update_rq_clock_task()
731 steal -= rq->prev_steal_time_rq; in update_rq_clock_task()
736 rq->prev_steal_time_rq += steal; in update_rq_clock_task()
741 rq->clock_task += delta; in update_rq_clock_task()
745 update_irq_load_avg(rq, irq_delta + steal); in update_rq_clock_task()
747 update_rq_clock_pelt(rq, delta); in update_rq_clock_task()
750 void update_rq_clock(struct rq *rq) in update_rq_clock() argument
754 lockdep_assert_rq_held(rq); in update_rq_clock()
756 if (rq->clock_update_flags & RQCF_ACT_SKIP) in update_rq_clock()
761 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); in update_rq_clock()
762 rq->clock_update_flags |= RQCF_UPDATED; in update_rq_clock()
765 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; in update_rq_clock()
768 rq->clock += delta; in update_rq_clock()
769 update_rq_clock_task(rq, delta); in update_rq_clock()
777 static void hrtick_clear(struct rq *rq) in hrtick_clear() argument
779 if (hrtimer_active(&rq->hrtick_timer)) in hrtick_clear()
780 hrtimer_cancel(&rq->hrtick_timer); in hrtick_clear()
789 struct rq *rq = container_of(timer, struct rq, hrtick_timer); in hrtick() local
792 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); in hrtick()
794 rq_lock(rq, &rf); in hrtick()
795 update_rq_clock(rq); in hrtick()
796 rq->curr->sched_class->task_tick(rq, rq->curr, 1); in hrtick()
797 rq_unlock(rq, &rf); in hrtick()
804 static void __hrtick_restart(struct rq *rq) in __hrtick_restart() argument
806 struct hrtimer *timer = &rq->hrtick_timer; in __hrtick_restart()
807 ktime_t time = rq->hrtick_time; in __hrtick_restart()
817 struct rq *rq = arg; in __hrtick_start() local
820 rq_lock(rq, &rf); in __hrtick_start()
821 __hrtick_restart(rq); in __hrtick_start()
822 rq_unlock(rq, &rf); in __hrtick_start()
830 void hrtick_start(struct rq *rq, u64 delay) in hrtick_start() argument
832 struct hrtimer *timer = &rq->hrtick_timer; in hrtick_start()
840 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); in hrtick_start()
842 if (rq == this_rq()) in hrtick_start()
843 __hrtick_restart(rq); in hrtick_start()
845 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); in hrtick_start()
854 void hrtick_start(struct rq *rq, u64 delay) in hrtick_start() argument
861 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), in hrtick_start()
867 static void hrtick_rq_init(struct rq *rq) in hrtick_rq_init() argument
870 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); in hrtick_rq_init()
872 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in hrtick_rq_init()
873 rq->hrtick_timer.function = hrtick; in hrtick_rq_init()
876 static inline void hrtick_clear(struct rq *rq) in hrtick_clear() argument
880 static inline void hrtick_rq_init(struct rq *rq) in hrtick_rq_init() argument
1041 void resched_curr(struct rq *rq) in resched_curr() argument
1043 struct task_struct *curr = rq->curr; in resched_curr()
1046 lockdep_assert_rq_held(rq); in resched_curr()
1051 cpu = cpu_of(rq); in resched_curr()
1067 struct rq *rq = cpu_rq(cpu); in resched_cpu() local
1070 raw_spin_rq_lock_irqsave(rq, flags); in resched_cpu()
1072 resched_curr(rq); in resched_cpu()
1073 raw_spin_rq_unlock_irqrestore(rq, flags); in resched_cpu()
1130 struct rq *rq = cpu_rq(cpu); in wake_up_idle_cpu() local
1135 if (set_nr_and_not_polling(rq->idle)) in wake_up_idle_cpu()
1174 struct rq *rq = info; in nohz_csd_func() local
1175 int cpu = cpu_of(rq); in nohz_csd_func()
1184 rq->idle_balance = idle_cpu(cpu); in nohz_csd_func()
1185 if (rq->idle_balance && !need_resched()) { in nohz_csd_func()
1186 rq->nohz_idle_balance = flags; in nohz_csd_func()
1194 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) in __need_bw_check() argument
1196 if (rq->nr_running != 1) in __need_bw_check()
1208 bool sched_can_stop_tick(struct rq *rq) in sched_can_stop_tick() argument
1213 if (rq->dl.dl_nr_running) in sched_can_stop_tick()
1220 if (rq->rt.rr_nr_running) { in sched_can_stop_tick()
1221 if (rq->rt.rr_nr_running == 1) in sched_can_stop_tick()
1231 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; in sched_can_stop_tick()
1240 if (rq->nr_running > 1) in sched_can_stop_tick()
1250 if (sched_feat(HZ_BW) && __need_bw_check(rq, rq->curr)) { in sched_can_stop_tick()
1251 if (cfs_task_bw_constrained(rq->curr)) in sched_can_stop_tick()
1417 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, in uclamp_idle_value() argument
1426 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; in uclamp_idle_value()
1433 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, in uclamp_idle_reset() argument
1437 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) in uclamp_idle_reset()
1440 uclamp_rq_set(rq, clamp_id, clamp_value); in uclamp_idle_reset()
1444 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, in uclamp_rq_max_value() argument
1447 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; in uclamp_rq_max_value()
1461 return uclamp_idle_value(rq, clamp_id, clamp_value); in uclamp_rq_max_value()
1484 struct rq *rq; in uclamp_update_util_min_rt_default() local
1490 rq = task_rq_lock(p, &rf); in uclamp_update_util_min_rt_default()
1492 task_rq_unlock(rq, p, &rf); in uclamp_update_util_min_rt_default()
1566 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, in uclamp_rq_inc_id() argument
1569 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; in uclamp_rq_inc_id()
1573 lockdep_assert_rq_held(rq); in uclamp_rq_inc_id()
1582 uclamp_idle_reset(rq, clamp_id, uc_se->value); in uclamp_rq_inc_id()
1591 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) in uclamp_rq_inc_id()
1592 uclamp_rq_set(rq, clamp_id, uc_se->value); in uclamp_rq_inc_id()
1604 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, in uclamp_rq_dec_id() argument
1607 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; in uclamp_rq_dec_id()
1613 lockdep_assert_rq_held(rq); in uclamp_rq_dec_id()
1658 rq_clamp = uclamp_rq_get(rq, clamp_id); in uclamp_rq_dec_id()
1665 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); in uclamp_rq_dec_id()
1666 uclamp_rq_set(rq, clamp_id, bkt_clamp); in uclamp_rq_dec_id()
1670 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) in uclamp_rq_inc() argument
1687 uclamp_rq_inc_id(rq, p, clamp_id); in uclamp_rq_inc()
1690 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) in uclamp_rq_inc()
1691 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; in uclamp_rq_inc()
1694 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) in uclamp_rq_dec() argument
1711 uclamp_rq_dec_id(rq, p, clamp_id); in uclamp_rq_dec()
1714 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, in uclamp_rq_reinc_id() argument
1720 uclamp_rq_dec_id(rq, p, clamp_id); in uclamp_rq_reinc_id()
1721 uclamp_rq_inc_id(rq, p, clamp_id); in uclamp_rq_reinc_id()
1727 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) in uclamp_rq_reinc_id()
1728 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; in uclamp_rq_reinc_id()
1736 struct rq *rq; in uclamp_update_active() local
1746 rq = task_rq_lock(p, &rf); in uclamp_update_active()
1755 uclamp_rq_reinc_id(rq, p, clamp_id); in uclamp_update_active()
1757 task_rq_unlock(rq, p, &rf); in uclamp_update_active()
2013 static void __init init_uclamp_rq(struct rq *rq) in init_uclamp_rq() argument
2016 struct uclamp_rq *uc_rq = rq->uclamp; in init_uclamp_rq()
2024 rq->uclamp_flags = UCLAMP_FLAG_IDLE; in init_uclamp_rq()
2053 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } in uclamp_rq_inc() argument
2054 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } in uclamp_rq_dec() argument
2091 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) in enqueue_task() argument
2094 update_rq_clock(rq); in enqueue_task()
2097 sched_info_enqueue(rq, p); in enqueue_task()
2101 uclamp_rq_inc(rq, p); in enqueue_task()
2102 p->sched_class->enqueue_task(rq, p, flags); in enqueue_task()
2104 if (sched_core_enabled(rq)) in enqueue_task()
2105 sched_core_enqueue(rq, p); in enqueue_task()
2108 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) in dequeue_task() argument
2110 if (sched_core_enabled(rq)) in dequeue_task()
2111 sched_core_dequeue(rq, p, flags); in dequeue_task()
2114 update_rq_clock(rq); in dequeue_task()
2117 sched_info_dequeue(rq, p); in dequeue_task()
2121 uclamp_rq_dec(rq, p); in dequeue_task()
2122 p->sched_class->dequeue_task(rq, p, flags); in dequeue_task()
2125 void activate_task(struct rq *rq, struct task_struct *p, int flags) in activate_task() argument
2130 sched_mm_cid_migrate_to(rq, p); in activate_task()
2132 enqueue_task(rq, p, flags); in activate_task()
2137 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) in deactivate_task() argument
2141 dequeue_task(rq, p, flags); in deactivate_task()
2208 static inline void check_class_changed(struct rq *rq, struct task_struct *p, in check_class_changed() argument
2214 prev_class->switched_from(rq, p); in check_class_changed()
2216 p->sched_class->switched_to(rq, p); in check_class_changed()
2218 p->sched_class->prio_changed(rq, p, oldprio); in check_class_changed()
2221 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) in check_preempt_curr() argument
2223 if (p->sched_class == rq->curr->sched_class) in check_preempt_curr()
2224 rq->curr->sched_class->check_preempt_curr(rq, p, flags); in check_preempt_curr()
2225 else if (sched_class_above(p->sched_class, rq->curr->sched_class)) in check_preempt_curr()
2226 resched_curr(rq); in check_preempt_curr()
2232 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) in check_preempt_curr()
2233 rq_clock_skip_update(rq); in check_preempt_curr()
2290 struct rq *rq; in wait_task_inactive() local
2299 rq = task_rq(p); in wait_task_inactive()
2312 while (task_on_cpu(rq, p)) { in wait_task_inactive()
2323 rq = task_rq_lock(p, &rf); in wait_task_inactive()
2325 running = task_on_cpu(rq, p); in wait_task_inactive()
2337 task_rq_unlock(rq, p, &rf); in wait_task_inactive()
2392 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) in migrate_disable_switch() argument
2395 .new_mask = cpumask_of(rq->cpu), in migrate_disable_switch()
2462 static inline bool rq_has_pinned_tasks(struct rq *rq) in rq_has_pinned_tasks() argument
2464 return rq->nr_pinned; in rq_has_pinned_tasks()
2516 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, in move_queued_task() argument
2519 lockdep_assert_rq_held(rq); in move_queued_task()
2521 deactivate_task(rq, p, DEQUEUE_NOCLOCK); in move_queued_task()
2523 rq_unlock(rq, rf); in move_queued_task()
2525 rq = cpu_rq(new_cpu); in move_queued_task()
2527 rq_lock(rq, rf); in move_queued_task()
2529 activate_task(rq, p, 0); in move_queued_task()
2530 check_preempt_curr(rq, p, 0); in move_queued_task()
2532 return rq; in move_queued_task()
2562 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, in __migrate_task() argument
2567 return rq; in __migrate_task()
2569 rq = move_queued_task(rq, rf, p, dest_cpu); in __migrate_task()
2571 return rq; in __migrate_task()
2584 struct rq *rq = this_rq(); in migration_cpu_stop() local
2601 rq_lock(rq, &rf); in migration_cpu_stop()
2614 if (task_rq(p) == rq) { in migration_cpu_stop()
2627 update_rq_clock(rq); in migration_cpu_stop()
2628 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); in migration_cpu_stop()
2667 task_rq_unlock(rq, p, &rf); in migration_cpu_stop()
2675 task_rq_unlock(rq, p, &rf); in migration_cpu_stop()
2685 struct rq *lowest_rq = NULL, *rq = this_rq(); in push_cpu_stop() local
2689 raw_spin_rq_lock(rq); in push_cpu_stop()
2691 if (task_rq(p) != rq) in push_cpu_stop()
2702 lowest_rq = p->sched_class->find_lock_rq(p, rq); in push_cpu_stop()
2708 if (task_rq(p) == rq) { in push_cpu_stop()
2709 deactivate_task(rq, p, 0); in push_cpu_stop()
2715 double_unlock_balance(rq, lowest_rq); in push_cpu_stop()
2718 rq->push_busy = false; in push_cpu_stop()
2719 raw_spin_rq_unlock(rq); in push_cpu_stop()
2750 struct rq *rq = task_rq(p); in __do_set_cpus_allowed() local
2771 running = task_current(rq, p); in __do_set_cpus_allowed()
2778 lockdep_assert_rq_held(rq); in __do_set_cpus_allowed()
2779 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); in __do_set_cpus_allowed()
2782 put_prev_task(rq, p); in __do_set_cpus_allowed()
2787 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); in __do_set_cpus_allowed()
2789 set_next_task(rq, p); in __do_set_cpus_allowed()
2961 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, in affine_move_task() argument
2963 __releases(rq->lock) in affine_move_task()
2974 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { in affine_move_task()
2975 rq->push_busy = true; in affine_move_task()
2989 task_rq_unlock(rq, p, rf); in affine_move_task()
2992 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, in affine_move_task()
2993 p, &rq->push_work); in affine_move_task()
3043 task_rq_unlock(rq, p, rf); in affine_move_task()
3047 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { in affine_move_task()
3060 task_rq_unlock(rq, p, rf); in affine_move_task()
3063 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, in affine_move_task()
3073 rq = move_queued_task(rq, rf, p, dest_cpu); in affine_move_task()
3080 task_rq_unlock(rq, p, rf); in affine_move_task()
3108 struct rq *rq, in __set_cpus_allowed_ptr_locked() argument
3110 __releases(rq->lock) in __set_cpus_allowed_ptr_locked()
3119 update_rq_clock(rq); in __set_cpus_allowed_ptr_locked()
3177 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); in __set_cpus_allowed_ptr_locked()
3180 task_rq_unlock(rq, p, rf); in __set_cpus_allowed_ptr_locked()
3198 struct rq *rq; in __set_cpus_allowed_ptr() local
3200 rq = task_rq_lock(p, &rf); in __set_cpus_allowed_ptr()
3207 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) in __set_cpus_allowed_ptr()
3208 ctx->new_mask = rq->scratch_mask; in __set_cpus_allowed_ptr()
3210 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); in __set_cpus_allowed_ptr()
3242 struct rq *rq; in restrict_cpus_allowed_ptr() local
3245 rq = task_rq_lock(p, &rf); in restrict_cpus_allowed_ptr()
3262 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); in restrict_cpus_allowed_ptr()
3265 task_rq_unlock(rq, p, &rf); in restrict_cpus_allowed_ptr()
3400 struct rq *src_rq, *dst_rq; in __migrate_swap_task()
3435 struct rq *src_rq, *dst_rq; in migrate_swap_stop()
3704 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } in migrate_disable_switch() argument
3706 static inline bool rq_has_pinned_tasks(struct rq *rq) in rq_has_pinned_tasks() argument
3721 struct rq *rq; in ttwu_stat() local
3726 rq = this_rq(); in ttwu_stat()
3729 if (cpu == rq->cpu) { in ttwu_stat()
3730 __schedstat_inc(rq->ttwu_local); in ttwu_stat()
3738 for_each_domain(rq->cpu, sd) { in ttwu_stat()
3750 __schedstat_inc(rq->ttwu_count); in ttwu_stat()
3767 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, in ttwu_do_activate() argument
3772 lockdep_assert_rq_held(rq); in ttwu_do_activate()
3775 rq->nr_uninterruptible--; in ttwu_do_activate()
3787 activate_task(rq, p, en_flags); in ttwu_do_activate()
3788 check_preempt_curr(rq, p, wake_flags); in ttwu_do_activate()
3798 rq_unpin_lock(rq, rf); in ttwu_do_activate()
3799 p->sched_class->task_woken(rq, p); in ttwu_do_activate()
3800 rq_repin_lock(rq, rf); in ttwu_do_activate()
3803 if (rq->idle_stamp) { in ttwu_do_activate()
3804 u64 delta = rq_clock(rq) - rq->idle_stamp; in ttwu_do_activate()
3805 u64 max = 2*rq->max_idle_balance_cost; in ttwu_do_activate()
3807 update_avg(&rq->avg_idle, delta); in ttwu_do_activate()
3809 if (rq->avg_idle > max) in ttwu_do_activate()
3810 rq->avg_idle = max; in ttwu_do_activate()
3812 rq->wake_stamp = jiffies; in ttwu_do_activate()
3813 rq->wake_avg_idle = rq->avg_idle / 2; in ttwu_do_activate()
3815 rq->idle_stamp = 0; in ttwu_do_activate()
3848 struct rq *rq; in ttwu_runnable() local
3851 rq = __task_rq_lock(p, &rf); in ttwu_runnable()
3853 if (!task_on_cpu(rq, p)) { in ttwu_runnable()
3858 update_rq_clock(rq); in ttwu_runnable()
3859 check_preempt_curr(rq, p, wake_flags); in ttwu_runnable()
3864 __task_rq_unlock(rq, &rf); in ttwu_runnable()
3873 struct rq *rq = this_rq(); in sched_ttwu_pending() local
3880 rq_lock_irqsave(rq, &rf); in sched_ttwu_pending()
3881 update_rq_clock(rq); in sched_ttwu_pending()
3887 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) in sched_ttwu_pending()
3888 set_task_cpu(p, cpu_of(rq)); in sched_ttwu_pending()
3890 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); in sched_ttwu_pending()
3903 WRITE_ONCE(rq->ttwu_pending, 0); in sched_ttwu_pending()
3904 rq_unlock_irqrestore(rq, &rf); in sched_ttwu_pending()
3931 struct rq *rq = cpu_rq(cpu); in __ttwu_queue_wakelist() local
3935 WRITE_ONCE(rq->ttwu_pending, 1); in __ttwu_queue_wakelist()
3941 struct rq *rq = cpu_rq(cpu); in wake_up_if_idle() local
3944 if (is_idle_task(rcu_dereference(rq->curr))) { in wake_up_if_idle()
3945 guard(rq_lock_irqsave)(rq); in wake_up_if_idle()
3946 if (is_idle_task(rq->curr)) in wake_up_if_idle()
3947 resched_curr(rq); in wake_up_if_idle()
4021 struct rq *rq = cpu_rq(cpu); in ttwu_queue() local
4027 rq_lock(rq, &rf); in ttwu_queue()
4028 update_rq_clock(rq); in ttwu_queue()
4029 ttwu_do_activate(rq, p, wake_flags, &rf); in ttwu_queue()
4030 rq_unlock(rq, &rf); in ttwu_queue()
4405 struct rq *rq = NULL; in task_call_func() local
4412 rq = __task_rq_lock(p, &rf); in task_call_func()
4426 if (rq) in task_call_func()
4427 rq_unlock(rq, &rf); in task_call_func()
4851 struct rq *rq; in wake_up_new_task() local
4868 rq = __task_rq_lock(p, &rf); in wake_up_new_task()
4869 update_rq_clock(rq); in wake_up_new_task()
4872 activate_task(rq, p, ENQUEUE_NOCLOCK); in wake_up_new_task()
4874 check_preempt_curr(rq, p, WF_FORK); in wake_up_new_task()
4881 rq_unpin_lock(rq, &rf); in wake_up_new_task()
4882 p->sched_class->task_woken(rq, p); in wake_up_new_task()
4883 rq_repin_lock(rq, &rf); in wake_up_new_task()
4886 task_rq_unlock(rq, p, &rf); in wake_up_new_task()
5010 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) in do_balance_callbacks() argument
5012 void (*func)(struct rq *rq); in do_balance_callbacks()
5015 lockdep_assert_rq_held(rq); in do_balance_callbacks()
5018 func = (void (*)(struct rq *))head->func; in do_balance_callbacks()
5023 func(rq); in do_balance_callbacks()
5027 static void balance_push(struct rq *rq);
5046 __splice_balance_callbacks(struct rq *rq, bool split) in __splice_balance_callbacks() argument
5048 struct balance_callback *head = rq->balance_callback; in __splice_balance_callbacks()
5053 lockdep_assert_rq_held(rq); in __splice_balance_callbacks()
5065 rq->balance_callback = NULL; in __splice_balance_callbacks()
5070 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) in splice_balance_callbacks() argument
5072 return __splice_balance_callbacks(rq, true); in splice_balance_callbacks()
5075 static void __balance_callbacks(struct rq *rq) in __balance_callbacks() argument
5077 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); in __balance_callbacks()
5080 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) in balance_callbacks() argument
5085 raw_spin_rq_lock_irqsave(rq, flags); in balance_callbacks()
5086 do_balance_callbacks(rq, head); in balance_callbacks()
5087 raw_spin_rq_unlock_irqrestore(rq, flags); in balance_callbacks()
5093 static inline void __balance_callbacks(struct rq *rq) in __balance_callbacks() argument
5097 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) in splice_balance_callbacks() argument
5102 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) in balance_callbacks() argument
5109 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) in prepare_lock_switch() argument
5117 rq_unpin_lock(rq, rf); in prepare_lock_switch()
5118 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); in prepare_lock_switch()
5121 rq_lockp(rq)->owner = next; in prepare_lock_switch()
5125 static inline void finish_lock_switch(struct rq *rq) in finish_lock_switch() argument
5132 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); in finish_lock_switch()
5133 __balance_callbacks(rq); in finish_lock_switch()
5134 raw_spin_rq_unlock_irq(rq); in finish_lock_switch()
5179 prepare_task_switch(struct rq *rq, struct task_struct *prev, in prepare_task_switch() argument
5183 sched_info_switch(rq, prev, next); in prepare_task_switch()
5211 static struct rq *finish_task_switch(struct task_struct *prev) in finish_task_switch()
5212 __releases(rq->lock) in finish_task_switch()
5214 struct rq *rq = this_rq(); in finish_task_switch() local
5215 struct mm_struct *mm = rq->prev_mm; in finish_task_switch()
5234 rq->prev_mm = NULL; in finish_task_switch()
5252 finish_lock_switch(rq); in finish_task_switch()
5292 return rq; in finish_task_switch()
5300 __releases(rq->lock) in schedule_tail()
5323 static __always_inline struct rq *
5324 context_switch(struct rq *rq, struct task_struct *prev, in context_switch() argument
5327 prepare_task_switch(rq, prev, next); in context_switch()
5355 membarrier_switch_mm(rq, prev->active_mm, next->mm); in context_switch()
5369 rq->prev_mm = prev->active_mm; in context_switch()
5375 switch_mm_cid(rq, prev, next); in context_switch()
5377 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); in context_switch()
5379 prepare_lock_switch(rq, next, rf); in context_switch()
5549 struct rq *rq; in task_sched_runtime() local
5568 rq = task_rq_lock(p, &rf); in task_sched_runtime()
5574 if (task_current(rq, p) && task_on_rq_queued(p)) { in task_sched_runtime()
5576 update_rq_clock(rq); in task_sched_runtime()
5577 p->sched_class->update_curr(rq); in task_sched_runtime()
5580 task_rq_unlock(rq, p, &rf); in task_sched_runtime()
5586 static u64 cpu_resched_latency(struct rq *rq) in cpu_resched_latency() argument
5589 u64 resched_latency, now = rq_clock(rq); in cpu_resched_latency()
5601 if (!rq->last_seen_need_resched_ns) { in cpu_resched_latency()
5602 rq->last_seen_need_resched_ns = now; in cpu_resched_latency()
5603 rq->ticks_without_resched = 0; in cpu_resched_latency()
5607 rq->ticks_without_resched++; in cpu_resched_latency()
5608 resched_latency = now - rq->last_seen_need_resched_ns; in cpu_resched_latency()
5631 static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } in cpu_resched_latency() argument
5641 struct rq *rq = cpu_rq(cpu); in scheduler_tick() local
5642 struct task_struct *curr = rq->curr; in scheduler_tick()
5652 rq_lock(rq, &rf); in scheduler_tick()
5654 update_rq_clock(rq); in scheduler_tick()
5655 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); in scheduler_tick()
5656 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); in scheduler_tick()
5657 curr->sched_class->task_tick(rq, curr, 0); in scheduler_tick()
5659 resched_latency = cpu_resched_latency(rq); in scheduler_tick()
5660 calc_global_load_tick(rq); in scheduler_tick()
5661 sched_core_tick(rq); in scheduler_tick()
5662 task_tick_mm_cid(rq, curr); in scheduler_tick()
5664 rq_unlock(rq, &rf); in scheduler_tick()
5675 rq->idle_balance = idle_cpu(cpu); in scheduler_tick()
5676 trigger_load_balance(rq); in scheduler_tick()
5722 struct rq *rq = cpu_rq(cpu); in sched_tick_remote() local
5733 guard(rq_lock_irq)(rq); in sched_tick_remote()
5734 struct task_struct *curr = rq->curr; in sched_tick_remote()
5737 update_rq_clock(rq); in sched_tick_remote()
5744 u64 delta = rq_clock_task(rq) - curr->se.exec_start; in sched_tick_remote()
5747 curr->sched_class->task_tick(rq, curr, 0); in sched_tick_remote()
5749 calc_load_nohz_remote(rq); in sched_tick_remote()
5964 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, in put_prev_task_balance() argument
5978 if (class->balance(rq, prev, rf)) in put_prev_task_balance()
5983 put_prev_task(rq, prev); in put_prev_task_balance()
5990 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in __pick_next_task() argument
6002 rq->nr_running == rq->cfs.h_nr_running)) { in __pick_next_task()
6004 p = pick_next_task_fair(rq, prev, rf); in __pick_next_task()
6010 put_prev_task(rq, prev); in __pick_next_task()
6011 p = pick_next_task_idle(rq); in __pick_next_task()
6018 put_prev_task_balance(rq, prev, rf); in __pick_next_task()
6021 p = class->pick_next_task(rq); in __pick_next_task()
6048 static inline struct task_struct *pick_task(struct rq *rq) in pick_task() argument
6054 p = class->pick_task(rq); in pick_task()
6062 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
6064 static void queue_core_balance(struct rq *rq);
6067 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in pick_next_task() argument
6072 bool core_clock_updated = (rq == rq->core); in pick_next_task()
6075 struct rq *rq_i; in pick_next_task()
6078 if (!sched_core_enabled(rq)) in pick_next_task()
6079 return __pick_next_task(rq, prev, rf); in pick_next_task()
6081 cpu = cpu_of(rq); in pick_next_task()
6090 rq->core_pick = NULL; in pick_next_task()
6091 return __pick_next_task(rq, prev, rf); in pick_next_task()
6103 if (rq->core->core_pick_seq == rq->core->core_task_seq && in pick_next_task()
6104 rq->core->core_pick_seq != rq->core_sched_seq && in pick_next_task()
6105 rq->core_pick) { in pick_next_task()
6106 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); in pick_next_task()
6108 next = rq->core_pick; in pick_next_task()
6110 put_prev_task(rq, prev); in pick_next_task()
6111 set_next_task(rq, next); in pick_next_task()
6114 rq->core_pick = NULL; in pick_next_task()
6118 put_prev_task_balance(rq, prev, rf); in pick_next_task()
6121 need_sync = !!rq->core->core_cookie; in pick_next_task()
6124 rq->core->core_cookie = 0UL; in pick_next_task()
6125 if (rq->core->core_forceidle_count) { in pick_next_task()
6127 update_rq_clock(rq->core); in pick_next_task()
6130 sched_core_account_forceidle(rq); in pick_next_task()
6132 rq->core->core_forceidle_start = 0; in pick_next_task()
6133 rq->core->core_forceidle_count = 0; in pick_next_task()
6134 rq->core->core_forceidle_occupation = 0; in pick_next_task()
6149 rq->core->core_task_seq++; in pick_next_task()
6156 next = pick_task(rq); in pick_next_task()
6158 rq->core_pick = NULL; in pick_next_task()
6164 task_vruntime_update(rq, next, false); in pick_next_task()
6183 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) in pick_next_task()
6191 cookie = rq->core->core_cookie = max->core_cookie; in pick_next_task()
6213 rq->core->core_forceidle_count++; in pick_next_task()
6215 rq->core->core_forceidle_seq++; in pick_next_task()
6222 if (schedstat_enabled() && rq->core->core_forceidle_count) { in pick_next_task()
6223 rq->core->core_forceidle_start = rq_clock(rq->core); in pick_next_task()
6224 rq->core->core_forceidle_occupation = occ; in pick_next_task()
6227 rq->core->core_pick_seq = rq->core->core_task_seq; in pick_next_task()
6228 next = rq->core_pick; in pick_next_task()
6229 rq->core_sched_seq = rq->core->core_pick_seq; in pick_next_task()
6263 if (!(fi_before && rq->core->core_forceidle_count)) in pick_next_task()
6264 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); in pick_next_task()
6285 set_next_task(rq, next); in pick_next_task()
6287 if (rq->core->core_forceidle_count && next == rq->idle) in pick_next_task()
6288 queue_core_balance(rq); in pick_next_task()
6295 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); in try_steal_cookie()
6366 static void sched_core_balance(struct rq *rq) in sched_core_balance() argument
6369 int cpu = cpu_of(rq); in sched_core_balance()
6373 raw_spin_rq_unlock_irq(rq); in sched_core_balance()
6381 raw_spin_rq_lock_irq(rq); in sched_core_balance()
6388 static void queue_core_balance(struct rq *rq) in queue_core_balance() argument
6390 if (!sched_core_enabled(rq)) in queue_core_balance()
6393 if (!rq->core->core_cookie) in queue_core_balance()
6396 if (!rq->nr_running) /* not forced idle */ in queue_core_balance()
6399 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); in queue_core_balance()
6410 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; in sched_core_cpu_starting() local
6415 WARN_ON_ONCE(rq->core != rq); in sched_core_cpu_starting()
6425 rq = cpu_rq(t); in sched_core_cpu_starting()
6426 if (rq->core == rq) { in sched_core_cpu_starting()
6427 core_rq = rq; in sched_core_cpu_starting()
6437 rq = cpu_rq(t); in sched_core_cpu_starting()
6440 rq->core = core_rq; in sched_core_cpu_starting()
6442 WARN_ON_ONCE(rq->core != core_rq); in sched_core_cpu_starting()
6449 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; in sched_core_cpu_deactivate() local
6456 WARN_ON_ONCE(rq->core != rq); in sched_core_cpu_deactivate()
6461 if (rq->core != rq) in sched_core_cpu_deactivate()
6476 core_rq->core_task_seq = rq->core_task_seq; in sched_core_cpu_deactivate()
6477 core_rq->core_pick_seq = rq->core_pick_seq; in sched_core_cpu_deactivate()
6478 core_rq->core_cookie = rq->core_cookie; in sched_core_cpu_deactivate()
6479 core_rq->core_forceidle_count = rq->core_forceidle_count; in sched_core_cpu_deactivate()
6480 core_rq->core_forceidle_seq = rq->core_forceidle_seq; in sched_core_cpu_deactivate()
6481 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; in sched_core_cpu_deactivate()
6492 rq = cpu_rq(t); in sched_core_cpu_deactivate()
6493 rq->core = core_rq; in sched_core_cpu_deactivate()
6499 struct rq *rq = cpu_rq(cpu); in sched_core_cpu_dying() local
6501 if (rq->core != rq) in sched_core_cpu_dying()
6502 rq->core = rq; in sched_core_cpu_dying()
6512 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in pick_next_task() argument
6514 return __pick_next_task(rq, prev, rf); in pick_next_task()
6582 struct rq *rq; in __schedule() local
6586 rq = cpu_rq(cpu); in __schedule()
6587 prev = rq->curr; in __schedule()
6592 hrtick_clear(rq); in __schedule()
6612 rq_lock(rq, &rf); in __schedule()
6616 rq->clock_update_flags <<= 1; in __schedule()
6617 update_rq_clock(rq); in __schedule()
6636 rq->nr_uninterruptible++; in __schedule()
6649 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); in __schedule()
6652 atomic_inc(&rq->nr_iowait); in __schedule()
6659 next = pick_next_task(rq, prev, &rf); in __schedule()
6663 rq->last_seen_need_resched_ns = 0; in __schedule()
6667 rq->nr_switches++; in __schedule()
6672 RCU_INIT_POINTER(rq->curr, next); in __schedule()
6689 migrate_disable_switch(rq, prev); in __schedule()
6695 rq = context_switch(rq, prev, next, &rf); in __schedule()
6697 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); in __schedule()
6699 rq_unpin_lock(rq, &rf); in __schedule()
6700 __balance_callbacks(rq); in __schedule()
6701 raw_spin_rq_unlock_irq(rq); in __schedule()
7069 struct rq *rq; in rt_mutex_setprio() local
7080 rq = __task_rq_lock(p, &rf); in rt_mutex_setprio()
7081 update_rq_clock(rq); in rt_mutex_setprio()
7112 if (unlikely(p == rq->idle)) { in rt_mutex_setprio()
7113 WARN_ON(p != rq->curr); in rt_mutex_setprio()
7126 running = task_current(rq, p); in rt_mutex_setprio()
7128 dequeue_task(rq, p, queue_flag); in rt_mutex_setprio()
7130 put_prev_task(rq, p); in rt_mutex_setprio()
7165 enqueue_task(rq, p, queue_flag); in rt_mutex_setprio()
7167 set_next_task(rq, p); in rt_mutex_setprio()
7169 check_class_changed(rq, p, prev_class, oldprio); in rt_mutex_setprio()
7174 rq_unpin_lock(rq, &rf); in rt_mutex_setprio()
7175 __balance_callbacks(rq); in rt_mutex_setprio()
7176 raw_spin_rq_unlock(rq); in rt_mutex_setprio()
7192 struct rq *rq; in set_user_nice() local
7200 rq = task_rq_lock(p, &rf); in set_user_nice()
7201 update_rq_clock(rq); in set_user_nice()
7214 running = task_current(rq, p); in set_user_nice()
7216 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); in set_user_nice()
7218 put_prev_task(rq, p); in set_user_nice()
7226 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); in set_user_nice()
7228 set_next_task(rq, p); in set_user_nice()
7234 p->sched_class->prio_changed(rq, p, old_prio); in set_user_nice()
7237 task_rq_unlock(rq, p, &rf); in set_user_nice()
7327 struct rq *rq = cpu_rq(cpu); in idle_cpu() local
7329 if (rq->curr != rq->idle) in idle_cpu()
7332 if (rq->nr_running) in idle_cpu()
7336 if (rq->ttwu_pending) in idle_cpu()
7374 struct rq *rq = cpu_rq(cpu); in sched_core_idle_cpu() local
7376 if (sched_core_enabled(rq) && rq->curr == rq->idle) in sched_core_idle_cpu()
7410 struct rq *rq = cpu_rq(cpu); in effective_cpu_util() local
7415 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { in effective_cpu_util()
7424 irq = cpu_util_irq(rq); in effective_cpu_util()
7440 util = util_cfs + cpu_util_rt(rq); in effective_cpu_util()
7442 util = uclamp_rq_util_with(rq, util, p); in effective_cpu_util()
7444 dl_util = cpu_util_dl(rq); in effective_cpu_util()
7488 util += cpu_bw_dl(rq); in effective_cpu_util()
7631 struct rq *rq; in __sched_setscheduler() local
7698 rq = task_rq_lock(p, &rf); in __sched_setscheduler()
7699 update_rq_clock(rq); in __sched_setscheduler()
7704 if (p == rq->stop) { in __sched_setscheduler()
7745 cpumask_t *span = rq->rd->span; in __sched_setscheduler()
7753 rq->rd->dl_bw.bw == 0) { in __sched_setscheduler()
7764 task_rq_unlock(rq, p, &rf); in __sched_setscheduler()
7798 running = task_current(rq, p); in __sched_setscheduler()
7800 dequeue_task(rq, p, queue_flags); in __sched_setscheduler()
7802 put_prev_task(rq, p); in __sched_setscheduler()
7820 enqueue_task(rq, p, queue_flags); in __sched_setscheduler()
7823 set_next_task(rq, p); in __sched_setscheduler()
7825 check_class_changed(rq, p, prev_class, oldprio); in __sched_setscheduler()
7829 head = splice_balance_callbacks(rq); in __sched_setscheduler()
7830 task_rq_unlock(rq, p, &rf); in __sched_setscheduler()
7839 balance_callbacks(rq, head); in __sched_setscheduler()
7845 task_rq_unlock(rq, p, &rf); in __sched_setscheduler()
8531 struct rq *rq; in do_sched_yield() local
8533 rq = this_rq_lock_irq(&rf); in do_sched_yield()
8535 schedstat_inc(rq->yld_count); in do_sched_yield()
8536 current->sched_class->yield_task(rq); in do_sched_yield()
8539 rq_unlock_irq(rq, &rf); in do_sched_yield()
8934 struct rq *rq, *p_rq; in yield_to() local
8939 rq = this_rq(); in yield_to()
8947 if (rq->nr_running == 1 && p_rq->nr_running == 1) { in yield_to()
8952 double_rq_lock(rq, p_rq); in yield_to()
8954 double_rq_unlock(rq, p_rq); in yield_to()
8967 yielded = curr->sched_class->yield_to_task(rq, p); in yield_to()
8969 schedstat_inc(rq->yld_count); in yield_to()
8974 if (preempt && rq != p_rq) in yield_to()
8979 double_rq_unlock(rq, p_rq); in yield_to()
9089 struct rq *rq; in sched_rr_get_interval() local
9105 rq = task_rq_lock(p, &rf); in sched_rr_get_interval()
9108 time_slice = p->sched_class->get_rr_interval(rq, p); in sched_rr_get_interval()
9109 task_rq_unlock(rq, p, &rf); in sched_rr_get_interval()
9258 struct rq *rq = cpu_rq(cpu); in init_idle() local
9264 raw_spin_rq_lock(rq); in init_idle()
9298 rq->idle = idle; in init_idle()
9299 rcu_assign_pointer(rq->curr, idle); in init_idle()
9304 raw_spin_rq_unlock(rq); in init_idle()
9384 struct rq *rq; in sched_setnuma() local
9386 rq = task_rq_lock(p, &rf); in sched_setnuma()
9388 running = task_current(rq, p); in sched_setnuma()
9391 dequeue_task(rq, p, DEQUEUE_SAVE); in sched_setnuma()
9393 put_prev_task(rq, p); in sched_setnuma()
9398 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); in sched_setnuma()
9400 set_next_task(rq, p); in sched_setnuma()
9401 task_rq_unlock(rq, p, &rf); in sched_setnuma()
9428 struct rq *rq = this_rq(); in __balance_push_cpu_stop() local
9433 rq_lock(rq, &rf); in __balance_push_cpu_stop()
9435 update_rq_clock(rq); in __balance_push_cpu_stop()
9437 if (task_rq(p) == rq && task_on_rq_queued(p)) { in __balance_push_cpu_stop()
9438 cpu = select_fallback_rq(rq->cpu, p); in __balance_push_cpu_stop()
9439 rq = __migrate_task(rq, &rf, p, cpu); in __balance_push_cpu_stop()
9442 rq_unlock(rq, &rf); in __balance_push_cpu_stop()
9458 static void balance_push(struct rq *rq) in balance_push() argument
9460 struct task_struct *push_task = rq->curr; in balance_push()
9462 lockdep_assert_rq_held(rq); in balance_push()
9467 rq->balance_callback = &balance_push_callback; in balance_push()
9473 if (!cpu_dying(rq->cpu) || rq != this_rq()) in balance_push()
9494 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && in balance_push()
9495 rcuwait_active(&rq->hotplug_wait)) { in balance_push()
9496 raw_spin_rq_unlock(rq); in balance_push()
9497 rcuwait_wake_up(&rq->hotplug_wait); in balance_push()
9498 raw_spin_rq_lock(rq); in balance_push()
9508 raw_spin_rq_unlock(rq); in balance_push()
9509 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, in balance_push()
9516 raw_spin_rq_lock(rq); in balance_push()
9521 struct rq *rq = cpu_rq(cpu); in balance_push_set() local
9524 rq_lock_irqsave(rq, &rf); in balance_push_set()
9526 WARN_ON_ONCE(rq->balance_callback); in balance_push_set()
9527 rq->balance_callback = &balance_push_callback; in balance_push_set()
9528 } else if (rq->balance_callback == &balance_push_callback) { in balance_push_set()
9529 rq->balance_callback = NULL; in balance_push_set()
9531 rq_unlock_irqrestore(rq, &rf); in balance_push_set()
9542 struct rq *rq = this_rq(); in balance_hotplug_wait() local
9544 rcuwait_wait_event(&rq->hotplug_wait, in balance_hotplug_wait()
9545 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), in balance_hotplug_wait()
9551 static inline void balance_push(struct rq *rq) in balance_push() argument
9565 void set_rq_online(struct rq *rq) in set_rq_online() argument
9567 if (!rq->online) { in set_rq_online()
9570 cpumask_set_cpu(rq->cpu, rq->rd->online); in set_rq_online()
9571 rq->online = 1; in set_rq_online()
9575 class->rq_online(rq); in set_rq_online()
9580 void set_rq_offline(struct rq *rq) in set_rq_offline() argument
9582 if (rq->online) { in set_rq_offline()
9585 update_rq_clock(rq); in set_rq_offline()
9588 class->rq_offline(rq); in set_rq_offline()
9591 cpumask_clear_cpu(rq->cpu, rq->rd->online); in set_rq_offline()
9592 rq->online = 0; in set_rq_offline()
9648 struct rq *rq = cpu_rq(cpu); in sched_cpu_activate() local
9681 rq_lock_irqsave(rq, &rf); in sched_cpu_activate()
9682 if (rq->rd) { in sched_cpu_activate()
9683 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); in sched_cpu_activate()
9684 set_rq_online(rq); in sched_cpu_activate()
9686 rq_unlock_irqrestore(rq, &rf); in sched_cpu_activate()
9693 struct rq *rq = cpu_rq(cpu); in sched_cpu_deactivate() local
9701 nohz_balance_exit_idle(rq); in sched_cpu_deactivate()
9725 rq_lock_irqsave(rq, &rf); in sched_cpu_deactivate()
9726 if (rq->rd) { in sched_cpu_deactivate()
9727 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); in sched_cpu_deactivate()
9728 set_rq_offline(rq); in sched_cpu_deactivate()
9730 rq_unlock_irqrestore(rq, &rf); in sched_cpu_deactivate()
9759 struct rq *rq = cpu_rq(cpu); in sched_rq_cpu_starting() local
9761 rq->calc_load_update = calc_load_update; in sched_rq_cpu_starting()
9801 static void calc_load_migrate(struct rq *rq) in calc_load_migrate() argument
9803 long delta = calc_load_fold_active(rq, 1); in calc_load_migrate()
9809 static void dump_rq_tasks(struct rq *rq, const char *loglvl) in dump_rq_tasks() argument
9812 int cpu = cpu_of(rq); in dump_rq_tasks()
9814 lockdep_assert_rq_held(rq); in dump_rq_tasks()
9816 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); in dump_rq_tasks()
9830 struct rq *rq = cpu_rq(cpu); in sched_cpu_dying() local
9836 rq_lock_irqsave(rq, &rf); in sched_cpu_dying()
9837 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { in sched_cpu_dying()
9839 dump_rq_tasks(rq, KERN_WARNING); in sched_cpu_dying()
9841 rq_unlock_irqrestore(rq, &rf); in sched_cpu_dying()
9843 calc_load_migrate(rq); in sched_cpu_dying()
9845 hrtick_clear(rq); in sched_cpu_dying()
9974 struct rq *rq; in sched_init() local
9976 rq = cpu_rq(i); in sched_init()
9977 raw_spin_lock_init(&rq->__lock); in sched_init()
9978 rq->nr_running = 0; in sched_init()
9979 rq->calc_load_active = 0; in sched_init()
9980 rq->calc_load_update = jiffies + LOAD_FREQ; in sched_init()
9981 init_cfs_rq(&rq->cfs); in sched_init()
9982 init_rt_rq(&rq->rt); in sched_init()
9983 init_dl_rq(&rq->dl); in sched_init()
9985 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); in sched_init()
9986 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; in sched_init()
10006 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); in sched_init()
10009 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; in sched_init()
10011 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); in sched_init()
10014 rq->sd = NULL; in sched_init()
10015 rq->rd = NULL; in sched_init()
10016 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; in sched_init()
10017 rq->balance_callback = &balance_push_callback; in sched_init()
10018 rq->active_balance = 0; in sched_init()
10019 rq->next_balance = jiffies; in sched_init()
10020 rq->push_cpu = 0; in sched_init()
10021 rq->cpu = i; in sched_init()
10022 rq->online = 0; in sched_init()
10023 rq->idle_stamp = 0; in sched_init()
10024 rq->avg_idle = 2*sysctl_sched_migration_cost; in sched_init()
10025 rq->wake_stamp = jiffies; in sched_init()
10026 rq->wake_avg_idle = rq->avg_idle; in sched_init()
10027 rq->max_idle_balance_cost = sysctl_sched_migration_cost; in sched_init()
10029 INIT_LIST_HEAD(&rq->cfs_tasks); in sched_init()
10031 rq_attach_root(rq, &def_root_domain); in sched_init()
10033 rq->last_blocked_load_update_tick = jiffies; in sched_init()
10034 atomic_set(&rq->nohz_flags, 0); in sched_init()
10036 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); in sched_init()
10039 rcuwait_init(&rq->hotplug_wait); in sched_init()
10042 hrtick_rq_init(rq); in sched_init()
10043 atomic_set(&rq->nr_iowait, 0); in sched_init()
10046 rq->core = rq; in sched_init()
10047 rq->core_pick = NULL; in sched_init()
10048 rq->core_enabled = 0; in sched_init()
10049 rq->core_tree = RB_ROOT; in sched_init()
10050 rq->core_forceidle_count = 0; in sched_init()
10051 rq->core_forceidle_occupation = 0; in sched_init()
10052 rq->core_forceidle_start = 0; in sched_init()
10054 rq->core_cookie = 0UL; in sched_init()
10056 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); in sched_init()
10502 struct rq *rq; in sched_move_task() local
10504 rq = task_rq_lock(tsk, &rf); in sched_move_task()
10513 update_rq_clock(rq); in sched_move_task()
10515 running = task_current(rq, tsk); in sched_move_task()
10519 dequeue_task(rq, tsk, queue_flags); in sched_move_task()
10521 put_prev_task(rq, tsk); in sched_move_task()
10526 enqueue_task(rq, tsk, queue_flags); in sched_move_task()
10528 set_next_task(rq, tsk); in sched_move_task()
10534 resched_curr(rq); in sched_move_task()
10538 task_rq_unlock(rq, tsk, &rf); in sched_move_task()
10897 struct rq *rq = cfs_rq->rq; in tg_set_cfs_bandwidth() local
10900 rq_lock_irq(rq, &rf); in tg_set_cfs_bandwidth()
10906 rq_unlock_irq(rq, &rf); in tg_set_cfs_bandwidth()
11571 void call_trace_sched_update_nr_running(struct rq *rq, int count) in call_trace_sched_update_nr_running() argument
11573 trace_sched_update_nr_running_tp(rq, count); in call_trace_sched_update_nr_running()
11692 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, in __sched_mm_cid_migrate_from_fetch_cid()
11733 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, in __sched_mm_cid_migrate_from_try_steal_cid()
11797 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) in sched_mm_cid_migrate_to()
11802 struct rq *src_rq; in sched_mm_cid_migrate_to()
11853 struct rq *rq = cpu_rq(cpu); in sched_mm_cid_remote_clear() local
11891 t = rcu_dereference(rq->curr); in sched_mm_cid_remote_clear()
11911 struct rq *rq = cpu_rq(cpu); in sched_mm_cid_remote_clear_old() local
11920 rq_clock = READ_ONCE(rq->clock); in sched_mm_cid_remote_clear_old()
11929 curr = rcu_dereference(rq->curr); in sched_mm_cid_remote_clear_old()
12013 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) in task_tick_mm_cid() argument
12030 struct rq *rq; in sched_mm_cid_exit_signals() local
12036 rq = this_rq(); in sched_mm_cid_exit_signals()
12037 rq_lock_irqsave(rq, &rf); in sched_mm_cid_exit_signals()
12047 rq_unlock_irqrestore(rq, &rf); in sched_mm_cid_exit_signals()
12054 struct rq *rq; in sched_mm_cid_before_execve() local
12060 rq = this_rq(); in sched_mm_cid_before_execve()
12061 rq_lock_irqsave(rq, &rf); in sched_mm_cid_before_execve()
12071 rq_unlock_irqrestore(rq, &rf); in sched_mm_cid_before_execve()
12078 struct rq *rq; in sched_mm_cid_after_execve() local
12084 rq = this_rq(); in sched_mm_cid_after_execve()
12085 rq_lock_irqsave(rq, &rf); in sched_mm_cid_after_execve()
12093 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, mm); in sched_mm_cid_after_execve()
12094 rq_unlock_irqrestore(rq, &rf); in sched_mm_cid_after_execve()