60 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()
62 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
68 	struct rq *rq = task_rq(p);  in dl_rq_of_se()  local
70 	return &rq->dl;  in dl_rq_of_se()
175 		struct rq *rq = cpu_rq(i);  in __dl_update()  local
177 		rq->dl.extra_bw += bw;  in __dl_update()
311 	struct rq *rq;  in dl_change_utilization()  local
318 	rq = task_rq(p);  in dl_change_utilization()
320 		sub_running_bw(&p->dl, &rq->dl);  in dl_change_utilization()
332 	__sub_rq_bw(p->dl.dl_bw, &rq->dl);  in dl_change_utilization()
333 	__add_rq_bw(new_bw, &rq->dl);  in dl_change_utilization()
395 	struct rq *rq = rq_of_dl_rq(dl_rq);  in task_non_contending()  local
418 	zerolag_time -= rq_clock(rq);  in task_non_contending()
431 				sub_rq_bw(&p->dl, &rq->dl);  in task_non_contending()
531 static inline int dl_overloaded(struct rq *rq)  in dl_overloaded()  argument
533 	return atomic_read(&rq->rd->dlo_count);  in dl_overloaded()
536 static inline void dl_set_overload(struct rq *rq)  in dl_set_overload()  argument
538 	if (!rq->online)  in dl_set_overload()
541 	cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_set_overload()
549 	atomic_inc(&rq->rd->dlo_count);  in dl_set_overload()
552 static inline void dl_clear_overload(struct rq *rq)  in dl_clear_overload()  argument
554 	if (!rq->online)  in dl_clear_overload()
557 	atomic_dec(&rq->rd->dlo_count);  in dl_clear_overload()
558 	cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask);  in dl_clear_overload()
606 static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
613 				 &rq->dl.pushable_dl_tasks_root,  in enqueue_pushable_dl_task()
616 		rq->dl.earliest_dl.next = p->dl.deadline;  in enqueue_pushable_dl_task()
619 static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
621 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()
635 static inline int has_pushable_dl_tasks(struct rq *rq)  in has_pushable_dl_tasks()  argument
637 	return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root.rb_root);  in has_pushable_dl_tasks()
640 static int push_dl_task(struct rq *rq);
642 static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev)  in need_pull_dl_task()  argument
644 	return rq->online && dl_task(prev);  in need_pull_dl_task()
650 static void push_dl_tasks(struct rq *);
651 static void pull_dl_task(struct rq *);
653 static inline void deadline_queue_push_tasks(struct rq *rq)  in deadline_queue_push_tasks()  argument
655 	if (!has_pushable_dl_tasks(rq))  in deadline_queue_push_tasks()
658 	queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks);  in deadline_queue_push_tasks()
661 static inline void deadline_queue_pull_task(struct rq *rq)  in deadline_queue_pull_task()  argument
663 	queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task);  in deadline_queue_pull_task()
666 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq);
668 static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p)  in dl_task_offline_migration()  argument
670 	struct rq *later_rq = NULL;  in dl_task_offline_migration()
673 	later_rq = find_lock_later_rq(p, rq);  in dl_task_offline_migration()
697 		double_lock_balance(rq, later_rq);  in dl_task_offline_migration()
707 		sub_running_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
708 		sub_rq_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
713 		sub_rq_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
722 	dl_b = &rq->rd->dl_bw;  in dl_task_offline_migration()
724 	__dl_sub(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));  in dl_task_offline_migration()
733 	double_unlock_balance(later_rq, rq);  in dl_task_offline_migration()
741 void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_dl_task()  argument
746 void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_dl_task()  argument
760 static inline void deadline_queue_push_tasks(struct rq *rq)  in deadline_queue_push_tasks()  argument
764 static inline void deadline_queue_pull_task(struct rq *rq)  in deadline_queue_pull_task()  argument
769 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
770 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags);
771 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, int flags);
774 					    struct rq *rq)  in replenish_dl_new_period()  argument
777 	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;  in replenish_dl_new_period()
796 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()  local
799 	WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline));  in setup_new_dl_entity()
814 	replenish_dl_new_period(dl_se, rq);  in setup_new_dl_entity()
838 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()  local
847 		replenish_dl_new_period(dl_se, rq);  in replenish_dl_entity()
872 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
874 		replenish_dl_new_period(dl_se, rq);  in replenish_dl_entity()
955 update_dl_revised_wakeup(struct sched_dl_entity *dl_se, struct rq *rq)  in update_dl_revised_wakeup()  argument
957 	u64 laxity = dl_se->deadline - rq_clock(rq);  in update_dl_revised_wakeup()
965 	WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq)));  in update_dl_revised_wakeup()
1019 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()  local
1021 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
1022 	    dl_entity_overflow(dl_se, rq_clock(rq))) {  in update_dl_entity()
1025 			     !dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in update_dl_entity()
1027 			update_dl_revised_wakeup(dl_se, rq);  in update_dl_entity()
1031 		replenish_dl_new_period(dl_se, rq);  in update_dl_entity()
1054 	struct rq *rq = task_rq(p);  in start_dl_timer()  local
1058 	lockdep_assert_rq_held(rq);  in start_dl_timer()
1067 	delta = ktime_to_ns(now) - rq_clock(rq);  in start_dl_timer()
1115 	struct rq *rq;  in dl_task_timer()  local
1117 	rq = task_rq_lock(p, &rf);  in dl_task_timer()
1141 	update_rq_clock(rq);  in dl_task_timer()
1163 	if (unlikely(!rq->online)) {  in dl_task_timer()
1168 		lockdep_unpin_lock(__rq_lockp(rq), rf.cookie);  in dl_task_timer()
1169 		rq = dl_task_offline_migration(rq, p);  in dl_task_timer()
1170 		rf.cookie = lockdep_pin_lock(__rq_lockp(rq));  in dl_task_timer()
1171 		update_rq_clock(rq);  in dl_task_timer()
1181 	enqueue_task_dl(rq, p, ENQUEUE_REPLENISH);  in dl_task_timer()
1182 	if (dl_task(rq->curr))  in dl_task_timer()
1183 		check_preempt_curr_dl(rq, p, 0);  in dl_task_timer()
1185 		resched_curr(rq);  in dl_task_timer()
1192 	if (has_pushable_dl_tasks(rq)) {  in dl_task_timer()
1197 		rq_unpin_lock(rq, &rf);  in dl_task_timer()
1198 		push_dl_task(rq);  in dl_task_timer()
1199 		rq_repin_lock(rq, &rf);  in dl_task_timer()
1204 	task_rq_unlock(rq, p, &rf);  in dl_task_timer()
1244 	struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se));  in dl_check_constrained_dl()  local
1246 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in dl_check_constrained_dl()
1247 	    dl_time_before(rq_clock(rq), dl_next_period(dl_se))) {  in dl_check_constrained_dl()
1281 static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se)  in grub_reclaim()  argument
1283 	u64 u_inact = rq->dl.this_bw - rq->dl.running_bw; /* Utot - Uact */  in grub_reclaim()
1285 	u64 u_act_min = (dl_se->dl_bw * rq->dl.bw_ratio) >> RATIO_SHIFT;  in grub_reclaim()
1295 	if (u_inact + rq->dl.extra_bw > BW_UNIT - u_act_min)  in grub_reclaim()
1298 		u_act = BW_UNIT - u_inact - rq->dl.extra_bw;  in grub_reclaim()
1307 static void update_curr_dl(struct rq *rq)  in update_curr_dl()  argument
1309 	struct task_struct *curr = rq->curr;  in update_curr_dl()
1312 	int cpu = cpu_of(rq);  in update_curr_dl()
1326 	now = rq_clock_task(rq);  in update_curr_dl()
1353 						 rq,  in update_curr_dl()
1374 		__dequeue_task_dl(rq, curr, 0);  in update_curr_dl()
1376 			enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH);  in update_curr_dl()
1378 		if (!is_leftmost(curr, &rq->dl))  in update_curr_dl()
1379 			resched_curr(rq);  in update_curr_dl()
1394 		struct rt_rq *rt_rq = &rq->rt;  in update_curr_dl()
1415 	struct rq *rq;  in inactive_task_timer()  local
1417 	rq = task_rq_lock(p, &rf);  in inactive_task_timer()
1420 	update_rq_clock(rq);  in inactive_task_timer()
1441 	sub_running_bw(dl_se, &rq->dl);  in inactive_task_timer()
1444 	task_rq_unlock(rq, p, &rf);  in inactive_task_timer()
1465 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()  local
1470 			cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_HIGHER);  in inc_dl_deadline()
1472 		cpudl_set(&rq->rd->cpudl, rq->cpu, deadline);  in inc_dl_deadline()
1478 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()  local
1487 		cpudl_clear(&rq->rd->cpudl, rq->cpu);  in dec_dl_deadline()
1488 		cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);  in dec_dl_deadline()
1494 		cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline);  in dec_dl_deadline()
1670 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in enqueue_task_dl()  argument
1722 		add_rq_bw(&p->dl, &rq->dl);  in enqueue_task_dl()
1723 		add_running_bw(&p->dl, &rq->dl);  in enqueue_task_dl()
1750 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)  in enqueue_task_dl()
1751 		enqueue_pushable_dl_task(rq, p);  in enqueue_task_dl()
1754 static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in __dequeue_task_dl()  argument
1756 	update_stats_dequeue_dl(&rq->dl, &p->dl, flags);  in __dequeue_task_dl()
1758 	dequeue_pushable_dl_task(rq, p);  in __dequeue_task_dl()
1761 static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)  in dequeue_task_dl()  argument
1763 	update_curr_dl(rq);  in dequeue_task_dl()
1764 	__dequeue_task_dl(rq, p, flags);  in dequeue_task_dl()
1767 		sub_running_bw(&p->dl, &rq->dl);  in dequeue_task_dl()
1768 		sub_rq_bw(&p->dl, &rq->dl);  in dequeue_task_dl()
1794 static void yield_task_dl(struct rq *rq)  in yield_task_dl()  argument
1802 	rq->curr->dl.dl_yielded = 1;  in yield_task_dl()
1804 	update_rq_clock(rq);  in yield_task_dl()
1805 	update_curr_dl(rq);  in yield_task_dl()
1811 	rq_clock_skip_update(rq);  in yield_task_dl()
1817 						 struct rq *rq)  in dl_task_is_earliest_deadline()  argument
1819 	return (!rq->dl.dl_nr_running ||  in dl_task_is_earliest_deadline()
1821 			       rq->dl.earliest_dl.curr));  in dl_task_is_earliest_deadline()
1831 	struct rq *rq;  in select_task_rq_dl()  local
1836 	rq = cpu_rq(cpu);  in select_task_rq_dl()
1839 	curr = READ_ONCE(rq->curr); /* unlocked access */  in select_task_rq_dl()
1878 	struct rq *rq;  in migrate_task_rq_dl()  local
1883 	rq = task_rq(p);  in migrate_task_rq_dl()
1889 	rq_lock(rq, &rf);  in migrate_task_rq_dl()
1891 		update_rq_clock(rq);  in migrate_task_rq_dl()
1892 		sub_running_bw(&p->dl, &rq->dl);  in migrate_task_rq_dl()
1904 	sub_rq_bw(&p->dl, &rq->dl);  in migrate_task_rq_dl()
1905 	rq_unlock(rq, &rf);  in migrate_task_rq_dl()
1908 static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)  in check_preempt_equal_dl()  argument
1914 	if (rq->curr->nr_cpus_allowed == 1 ||  in check_preempt_equal_dl()
1915 	    !cpudl_find(&rq->rd->cpudl, rq->curr, NULL))  in check_preempt_equal_dl()
1923 	    cpudl_find(&rq->rd->cpudl, p, NULL))  in check_preempt_equal_dl()
1926 	resched_curr(rq);  in check_preempt_equal_dl()
1929 static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf)  in balance_dl()  argument
1931 	if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) {  in balance_dl()
1938 		rq_unpin_lock(rq, rf);  in balance_dl()
1939 		pull_dl_task(rq);  in balance_dl()
1940 		rq_repin_lock(rq, rf);  in balance_dl()
1943 	return sched_stop_runnable(rq) || sched_dl_runnable(rq);  in balance_dl()
1951 static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,  in check_preempt_curr_dl()  argument
1954 	if (dl_entity_preempt(&p->dl, &rq->curr->dl)) {  in check_preempt_curr_dl()
1955 		resched_curr(rq);  in check_preempt_curr_dl()
1964 	if ((p->dl.deadline == rq->curr->dl.deadline) &&  in check_preempt_curr_dl()
1965 	    !test_tsk_need_resched(rq->curr))  in check_preempt_curr_dl()
1966 		check_preempt_equal_dl(rq, p);  in check_preempt_curr_dl()
1971 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1973 	hrtick_start(rq, p->dl.runtime);  in start_hrtick_dl()
1976 static void start_hrtick_dl(struct rq *rq, struct task_struct *p)  in start_hrtick_dl()  argument
1981 static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)  in set_next_task_dl()  argument
1984 	struct dl_rq *dl_rq = &rq->dl;  in set_next_task_dl()
1986 	p->se.exec_start = rq_clock_task(rq);  in set_next_task_dl()
1991 	dequeue_pushable_dl_task(rq, p);  in set_next_task_dl()
1996 	if (hrtick_enabled_dl(rq))  in set_next_task_dl()
1997 		start_hrtick_dl(rq, p);  in set_next_task_dl()
1999 	if (rq->curr->sched_class != &dl_sched_class)  in set_next_task_dl()
2000 		update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);  in set_next_task_dl()
2002 	deadline_queue_push_tasks(rq);  in set_next_task_dl()
2015 static struct task_struct *pick_task_dl(struct rq *rq)  in pick_task_dl()  argument
2018 	struct dl_rq *dl_rq = &rq->dl;  in pick_task_dl()
2021 	if (!sched_dl_runnable(rq))  in pick_task_dl()
2031 static struct task_struct *pick_next_task_dl(struct rq *rq)  in pick_next_task_dl()  argument
2035 	p = pick_task_dl(rq);  in pick_next_task_dl()
2037 		set_next_task_dl(rq, p, true);  in pick_next_task_dl()
2042 static void put_prev_task_dl(struct rq *rq, struct task_struct *p)  in put_prev_task_dl()  argument
2045 	struct dl_rq *dl_rq = &rq->dl;  in put_prev_task_dl()
2050 	update_curr_dl(rq);  in put_prev_task_dl()
2052 	update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1);  in put_prev_task_dl()
2054 		enqueue_pushable_dl_task(rq, p);  in put_prev_task_dl()
2065 static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued)  in task_tick_dl()  argument
2067 	update_curr_dl(rq);  in task_tick_dl()
2069 	update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1);  in task_tick_dl()
2075 	if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 &&  in task_tick_dl()
2076 	    is_leftmost(p, &rq->dl))  in task_tick_dl()
2077 		start_hrtick_dl(rq, p);  in task_tick_dl()
2093 static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)  in pick_dl_task()  argument
2095 	if (!task_on_cpu(rq, p) &&  in pick_dl_task()
2105 static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu)  in pick_earliest_pushable_dl_task()  argument
2110 	if (!has_pushable_dl_tasks(rq))  in pick_earliest_pushable_dl_task()
2113 	next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root);  in pick_earliest_pushable_dl_task()
2119 		if (pick_dl_task(rq, p, cpu))  in pick_earliest_pushable_dl_task()
2219 static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)  in find_lock_later_rq()  argument
2221 	struct rq *later_rq = NULL;  in find_lock_later_rq()
2228 		if ((cpu == -1) || (cpu == rq->cpu))  in find_lock_later_rq()
2244 		if (double_lock_balance(rq, later_rq)) {  in find_lock_later_rq()
2245 			if (unlikely(task_rq(task) != rq ||  in find_lock_later_rq()
2247 				     task_on_cpu(rq, task) ||  in find_lock_later_rq()
2250 				double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
2265 		double_unlock_balance(rq, later_rq);  in find_lock_later_rq()
2272 static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)  in pick_next_pushable_dl_task()  argument
2276 	if (!has_pushable_dl_tasks(rq))  in pick_next_pushable_dl_task()
2279 	p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root));  in pick_next_pushable_dl_task()
2281 	WARN_ON_ONCE(rq->cpu != task_cpu(p));  in pick_next_pushable_dl_task()
2282 	WARN_ON_ONCE(task_current(rq, p));  in pick_next_pushable_dl_task()
2296 static int push_dl_task(struct rq *rq)  in push_dl_task()  argument
2299 	struct rq *later_rq;  in push_dl_task()
2302 	if (!rq->dl.overloaded)  in push_dl_task()
2305 	next_task = pick_next_pushable_dl_task(rq);  in push_dl_task()
2315 	if (dl_task(rq->curr) &&  in push_dl_task()
2316 	    dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&  in push_dl_task()
2317 	    rq->curr->nr_cpus_allowed > 1) {  in push_dl_task()
2318 		resched_curr(rq);  in push_dl_task()
2325 	if (WARN_ON(next_task == rq->curr))  in push_dl_task()
2332 	later_rq = find_lock_later_rq(next_task, rq);  in push_dl_task()
2341 		task = pick_next_pushable_dl_task(rq);  in push_dl_task()
2359 	deactivate_task(rq, next_task, 0);  in push_dl_task()
2366 	double_unlock_balance(rq, later_rq);  in push_dl_task()
2374 static void push_dl_tasks(struct rq *rq)  in push_dl_tasks()  argument
2377 	while (push_dl_task(rq))  in push_dl_tasks()
2381 static void pull_dl_task(struct rq *this_rq)  in pull_dl_task()
2386 	struct rq *src_rq;  in pull_dl_task()
2475 static void task_woken_dl(struct rq *rq, struct task_struct *p)  in task_woken_dl()  argument
2477 	if (!task_on_cpu(rq, p) &&  in task_woken_dl()
2478 	    !test_tsk_need_resched(rq->curr) &&  in task_woken_dl()
2480 	    dl_task(rq->curr) &&  in task_woken_dl()
2481 	    (rq->curr->nr_cpus_allowed < 2 ||  in task_woken_dl()
2482 	     !dl_entity_preempt(&p->dl, &rq->curr->dl))) {  in task_woken_dl()
2483 		push_dl_tasks(rq);  in task_woken_dl()
2492 	struct rq *rq;  in set_cpus_allowed_dl()  local
2496 	rq = task_rq(p);  in set_cpus_allowed_dl()
2497 	src_rd = rq->rd;  in set_cpus_allowed_dl()
2507 		src_dl_b = dl_bw_of(cpu_of(rq));  in set_cpus_allowed_dl()
2522 static void rq_online_dl(struct rq *rq)  in rq_online_dl()  argument
2524 	if (rq->dl.overloaded)  in rq_online_dl()
2525 		dl_set_overload(rq);  in rq_online_dl()
2527 	cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_online_dl()
2528 	if (rq->dl.dl_nr_running > 0)  in rq_online_dl()
2529 		cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr);  in rq_online_dl()
2533 static void rq_offline_dl(struct rq *rq)  in rq_offline_dl()  argument
2535 	if (rq->dl.overloaded)  in rq_offline_dl()
2536 		dl_clear_overload(rq);  in rq_offline_dl()
2538 	cpudl_clear(&rq->rd->cpudl, rq->cpu);  in rq_offline_dl()
2539 	cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu);  in rq_offline_dl()
2554 	struct rq *rq;  in dl_add_task_root_domain()  local
2563 	rq = __task_rq_lock(p, &rf);  in dl_add_task_root_domain()
2565 	dl_b = &rq->rd->dl_bw;  in dl_add_task_root_domain()
2568 	__dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));  in dl_add_task_root_domain()
2572 	task_rq_unlock(rq, p, &rf);  in dl_add_task_root_domain()
2586 static void switched_from_dl(struct rq *rq, struct task_struct *p)  in switched_from_dl()  argument
2607 			sub_running_bw(&p->dl, &rq->dl);  in switched_from_dl()
2608 		sub_rq_bw(&p->dl, &rq->dl);  in switched_from_dl()
2624 	if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)  in switched_from_dl()
2627 	deadline_queue_pull_task(rq);  in switched_from_dl()
2634 static void switched_to_dl(struct rq *rq, struct task_struct *p)  in switched_to_dl()  argument
2641 		add_rq_bw(&p->dl, &rq->dl);  in switched_to_dl()
2646 	if (rq->curr != p) {  in switched_to_dl()
2648 		if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)  in switched_to_dl()
2649 			deadline_queue_push_tasks(rq);  in switched_to_dl()
2651 		if (dl_task(rq->curr))  in switched_to_dl()
2652 			check_preempt_curr_dl(rq, p, 0);  in switched_to_dl()
2654 			resched_curr(rq);  in switched_to_dl()
2656 		update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);  in switched_to_dl()
2664 static void prio_changed_dl(struct rq *rq, struct task_struct *p,  in prio_changed_dl()  argument
2667 	if (task_on_rq_queued(p) || task_current(rq, p)) {  in prio_changed_dl()
2675 		if (!rq->dl.overloaded)  in prio_changed_dl()
2676 			deadline_queue_pull_task(rq);  in prio_changed_dl()
2683 		if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline))  in prio_changed_dl()
2684 			resched_curr(rq);  in prio_changed_dl()
2691 		resched_curr(rq);  in prio_changed_dl()