57 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)  in dl_task_of()  argument
59 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
67 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()  argument
69 	struct task_struct *p = dl_task_of(dl_se);  in dl_rq_of_se()
75 static inline int on_dl_rq(struct sched_dl_entity *dl_se)  in on_dl_rq()  argument
77 	return !RB_EMPTY_NODE(&dl_se->rb_node);  in on_dl_rq()
81 static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se)  in pi_of()  argument
83 	return dl_se->pi_se;  in pi_of()
86 static inline bool is_dl_boosted(struct sched_dl_entity *dl_se)  in is_dl_boosted()  argument
88 	return pi_of(dl_se) != dl_se;  in is_dl_boosted()
91 static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se)  in pi_of()  argument
93 	return dl_se;  in pi_of()
96 static inline bool is_dl_boosted(struct sched_dl_entity *dl_se)  in is_dl_boosted()  argument
284 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_rq_bw()  argument
286 	if (!dl_entity_is_special(dl_se))  in add_rq_bw()
287 		__add_rq_bw(dl_se->dl_bw, dl_rq);  in add_rq_bw()
291 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_rq_bw()  argument
293 	if (!dl_entity_is_special(dl_se))  in sub_rq_bw()
294 		__sub_rq_bw(dl_se->dl_bw, dl_rq);  in sub_rq_bw()
298 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_running_bw()  argument
300 	if (!dl_entity_is_special(dl_se))  in add_running_bw()
301 		__add_running_bw(dl_se->dl_bw, dl_rq);  in add_running_bw()
305 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_running_bw()  argument
307 	if (!dl_entity_is_special(dl_se))  in sub_running_bw()
308 		__sub_running_bw(dl_se->dl_bw, dl_rq);  in sub_running_bw()
394 	struct sched_dl_entity *dl_se = &p->dl;  in task_non_contending()  local
395 	struct hrtimer *timer = &dl_se->inactive_timer;  in task_non_contending()
396 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_non_contending()
404 	if (dl_se->dl_runtime == 0)  in task_non_contending()
407 	if (dl_entity_is_special(dl_se))  in task_non_contending()
410 	WARN_ON(dl_se->dl_non_contending);  in task_non_contending()
412 	zerolag_time = dl_se->deadline -  in task_non_contending()
413 		 div64_long((dl_se->runtime * dl_se->dl_period),  in task_non_contending()
414 			dl_se->dl_runtime);  in task_non_contending()
426 	if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) {  in task_non_contending()
428 			sub_running_bw(dl_se, dl_rq);  in task_non_contending()
443 	dl_se->dl_non_contending = 1;  in task_non_contending()
448 static void task_contending(struct sched_dl_entity *dl_se, int flags)  in task_contending()  argument
450 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_contending()
456 	if (dl_se->dl_runtime == 0)  in task_contending()
460 		add_rq_bw(dl_se, dl_rq);  in task_contending()
462 	if (dl_se->dl_non_contending) {  in task_contending()
463 		dl_se->dl_non_contending = 0;  in task_contending()
471 		if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1)  in task_contending()
472 			put_task_struct(dl_task_of(dl_se));  in task_contending()
481 		add_running_bw(dl_se, dl_rq);  in task_contending()
487 	struct sched_dl_entity *dl_se = &p->dl;  in is_leftmost()  local
489 	return rb_first_cached(&dl_rq->root) == &dl_se->rb_node;  in is_leftmost()
569 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
571 	struct task_struct *p = dl_task_of(dl_se);  in inc_dl_migration()
579 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
581 	struct task_struct *p = dl_task_of(dl_se);  in dec_dl_migration()
746 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
751 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
768 static inline void replenish_dl_new_period(struct sched_dl_entity *dl_se,  in replenish_dl_new_period()  argument
772 	dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline;  in replenish_dl_new_period()
773 	dl_se->runtime = pi_of(dl_se)->dl_runtime;  in replenish_dl_new_period()
788 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se)  in setup_new_dl_entity()  argument
790 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()
793 	WARN_ON(is_dl_boosted(dl_se));  in setup_new_dl_entity()
794 	WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline));  in setup_new_dl_entity()
801 	if (dl_se->dl_throttled)  in setup_new_dl_entity()
809 	replenish_dl_new_period(dl_se, rq);  in setup_new_dl_entity()
830 static void replenish_dl_entity(struct sched_dl_entity *dl_se)  in replenish_dl_entity()  argument
832 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()
835 	WARN_ON_ONCE(pi_of(dl_se)->dl_runtime <= 0);  in replenish_dl_entity()
841 	if (dl_se->dl_deadline == 0)  in replenish_dl_entity()
842 		replenish_dl_new_period(dl_se, rq);  in replenish_dl_entity()
844 	if (dl_se->dl_yielded && dl_se->runtime > 0)  in replenish_dl_entity()
845 		dl_se->runtime = 0;  in replenish_dl_entity()
853 	while (dl_se->runtime <= 0) {  in replenish_dl_entity()
854 		dl_se->deadline += pi_of(dl_se)->dl_period;  in replenish_dl_entity()
855 		dl_se->runtime += pi_of(dl_se)->dl_runtime;  in replenish_dl_entity()
867 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
869 		replenish_dl_new_period(dl_se, rq);  in replenish_dl_entity()
872 	if (dl_se->dl_yielded)  in replenish_dl_entity()
873 		dl_se->dl_yielded = 0;  in replenish_dl_entity()
874 	if (dl_se->dl_throttled)  in replenish_dl_entity()
875 		dl_se->dl_throttled = 0;  in replenish_dl_entity()
902 static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t)  in dl_entity_overflow()  argument
924 	left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);  in dl_entity_overflow()
925 	right = ((dl_se->deadline - t) >> DL_SCALE) *  in dl_entity_overflow()
926 		(pi_of(dl_se)->dl_runtime >> DL_SCALE);  in dl_entity_overflow()
950 update_dl_revised_wakeup(struct sched_dl_entity *dl_se, struct rq *rq)  in update_dl_revised_wakeup()  argument
952 	u64 laxity = dl_se->deadline - rq_clock(rq);  in update_dl_revised_wakeup()
960 	WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq)));  in update_dl_revised_wakeup()
962 	dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT;  in update_dl_revised_wakeup()
976 static inline bool dl_is_implicit(struct sched_dl_entity *dl_se)  in dl_is_implicit()  argument
978 	return dl_se->dl_deadline == dl_se->dl_period;  in dl_is_implicit()
1011 static void update_dl_entity(struct sched_dl_entity *dl_se)  in update_dl_entity()  argument
1013 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()
1016 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
1017 	    dl_entity_overflow(dl_se, rq_clock(rq))) {  in update_dl_entity()
1019 		if (unlikely(!dl_is_implicit(dl_se) &&  in update_dl_entity()
1020 			     !dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in update_dl_entity()
1021 			     !is_dl_boosted(dl_se))) {  in update_dl_entity()
1022 			update_dl_revised_wakeup(dl_se, rq);  in update_dl_entity()
1026 		replenish_dl_new_period(dl_se, rq);  in update_dl_entity()
1030 static inline u64 dl_next_period(struct sched_dl_entity *dl_se)  in dl_next_period()  argument
1032 	return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period;  in dl_next_period()
1047 	struct sched_dl_entity *dl_se = &p->dl;  in start_dl_timer()  local
1048 	struct hrtimer *timer = &dl_se->dl_timer;  in start_dl_timer()
1060 	act = ns_to_ktime(dl_next_period(dl_se));  in start_dl_timer()
1105 	struct sched_dl_entity *dl_se = container_of(timer,  in dl_task_timer()  local
1108 	struct task_struct *p = dl_task_of(dl_se);  in dl_task_timer()
1125 	if (is_dl_boosted(dl_se))  in dl_task_timer()
1132 	if (!dl_se->dl_throttled)  in dl_task_timer()
1153 		replenish_dl_entity(dl_se);  in dl_task_timer()
1210 void init_dl_task_timer(struct sched_dl_entity *dl_se)  in init_dl_task_timer()  argument
1212 	struct hrtimer *timer = &dl_se->dl_timer;  in init_dl_task_timer()
1236 static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se)  in dl_check_constrained_dl()  argument
1238 	struct task_struct *p = dl_task_of(dl_se);  in dl_check_constrained_dl()
1239 	struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se));  in dl_check_constrained_dl()
1241 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in dl_check_constrained_dl()
1242 	    dl_time_before(rq_clock(rq), dl_next_period(dl_se))) {  in dl_check_constrained_dl()
1243 		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(p)))  in dl_check_constrained_dl()
1245 		dl_se->dl_throttled = 1;  in dl_check_constrained_dl()
1246 		if (dl_se->runtime > 0)  in dl_check_constrained_dl()
1247 			dl_se->runtime = 0;  in dl_check_constrained_dl()
1252 int dl_runtime_exceeded(struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
1254 	return (dl_se->runtime <= 0);  in dl_runtime_exceeded()
1274 static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se)  in grub_reclaim()  argument
1285 	if (u_inact + rq->dl.extra_bw > rq->dl.max_bw - dl_se->dl_bw)  in grub_reclaim()
1286 		u_act = dl_se->dl_bw;  in grub_reclaim()
1301 	struct sched_dl_entity *dl_se = &curr->dl;  in update_curr_dl()  local
1306 	if (!dl_task(curr) || !on_dl_rq(dl_se))  in update_curr_dl()
1320 		if (unlikely(dl_se->dl_yielded))  in update_curr_dl()
1332 	if (dl_entity_is_special(dl_se))  in update_curr_dl()
1342 	if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) {  in update_curr_dl()
1354 	dl_se->runtime -= scaled_delta_exec;  in update_curr_dl()
1357 	if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) {  in update_curr_dl()
1358 		dl_se->dl_throttled = 1;  in update_curr_dl()
1361 		if (dl_runtime_exceeded(dl_se) &&  in update_curr_dl()
1362 		    (dl_se->flags & SCHED_FLAG_DL_OVERRUN))  in update_curr_dl()
1363 			dl_se->dl_overrun = 1;  in update_curr_dl()
1366 		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(curr)))  in update_curr_dl()
1401 	struct sched_dl_entity *dl_se = container_of(timer,  in inactive_task_timer()  local
1404 	struct task_struct *p = dl_task_of(dl_se);  in inactive_task_timer()
1416 		if (READ_ONCE(p->__state) == TASK_DEAD && dl_se->dl_non_contending) {  in inactive_task_timer()
1419 			dl_se->dl_non_contending = 0;  in inactive_task_timer()
1429 	if (dl_se->dl_non_contending == 0)  in inactive_task_timer()
1432 	sub_running_bw(dl_se, &rq->dl);  in inactive_task_timer()
1433 	dl_se->dl_non_contending = 0;  in inactive_task_timer()
1441 void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)  in init_dl_inactive_task_timer()  argument
1443 	struct hrtimer *timer = &dl_se->inactive_timer;  in init_dl_inactive_task_timer()
1497 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
1499 	int prio = dl_task_of(dl_se)->prio;  in inc_dl_tasks()
1500 	u64 deadline = dl_se->deadline;  in inc_dl_tasks()
1507 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
1511 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
1513 	int prio = dl_task_of(dl_se)->prio;  in dec_dl_tasks()
1520 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
1521 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
1530 __schedstats_from_dl_se(struct sched_dl_entity *dl_se)  in __schedstats_from_dl_se()  argument
1532 	return &dl_task_of(dl_se)->stats;  in __schedstats_from_dl_se()
1536 update_stats_wait_start_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_wait_start_dl()  argument
1543 	stats = __schedstats_from_dl_se(dl_se);  in update_stats_wait_start_dl()
1544 	__update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_wait_start_dl()
1548 update_stats_wait_end_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_wait_end_dl()  argument
1555 	stats = __schedstats_from_dl_se(dl_se);  in update_stats_wait_end_dl()
1556 	__update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_wait_end_dl()
1560 update_stats_enqueue_sleeper_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_enqueue_sleeper_dl()  argument
1567 	stats = __schedstats_from_dl_se(dl_se);  in update_stats_enqueue_sleeper_dl()
1568 	__update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_enqueue_sleeper_dl()
1572 update_stats_enqueue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,  in update_stats_enqueue_dl()  argument
1579 		update_stats_enqueue_sleeper_dl(dl_rq, dl_se);  in update_stats_enqueue_dl()
1583 update_stats_dequeue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,  in update_stats_dequeue_dl()  argument
1586 	struct task_struct *p = dl_task_of(dl_se);  in update_stats_dequeue_dl()
1605 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se)  in __enqueue_dl_entity()  argument
1607 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()
1609 	WARN_ON_ONCE(!RB_EMPTY_NODE(&dl_se->rb_node));  in __enqueue_dl_entity()
1611 	rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less);  in __enqueue_dl_entity()
1613 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
1616 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se)  in __dequeue_dl_entity()  argument
1618 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()
1620 	if (RB_EMPTY_NODE(&dl_se->rb_node))  in __dequeue_dl_entity()
1623 	rb_erase_cached(&dl_se->rb_node, &dl_rq->root);  in __dequeue_dl_entity()
1625 	RB_CLEAR_NODE(&dl_se->rb_node);  in __dequeue_dl_entity()
1627 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1631 enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags)  in enqueue_dl_entity()  argument
1633 	WARN_ON_ONCE(on_dl_rq(dl_se));  in enqueue_dl_entity()
1635 	update_stats_enqueue_dl(dl_rq_of_se(dl_se), dl_se, flags);  in enqueue_dl_entity()
1643 		task_contending(dl_se, flags);  in enqueue_dl_entity()
1644 		update_dl_entity(dl_se);  in enqueue_dl_entity()
1646 		replenish_dl_entity(dl_se);  in enqueue_dl_entity()
1648 		  dl_time_before(dl_se->deadline,  in enqueue_dl_entity()
1649 				 rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) {  in enqueue_dl_entity()
1650 		setup_new_dl_entity(dl_se);  in enqueue_dl_entity()
1653 	__enqueue_dl_entity(dl_se);  in enqueue_dl_entity()
1656 static void dequeue_dl_entity(struct sched_dl_entity *dl_se)  in dequeue_dl_entity()  argument
1658 	__dequeue_dl_entity(dl_se);  in dequeue_dl_entity()
1974 	struct sched_dl_entity *dl_se = &p->dl;  in set_next_task_dl()  local
1979 		update_stats_wait_end_dl(dl_rq, dl_se);  in set_next_task_dl()
2008 	struct sched_dl_entity *dl_se;  in pick_task_dl()  local
2015 	dl_se = pick_next_dl_entity(dl_rq);  in pick_task_dl()
2016 	WARN_ON_ONCE(!dl_se);  in pick_task_dl()
2017 	p = dl_task_of(dl_se);  in pick_task_dl()
2035 	struct sched_dl_entity *dl_se = &p->dl;  in put_prev_task_dl()  local
2039 		update_stats_wait_start_dl(dl_rq, dl_se);  in put_prev_task_dl()
2916 	struct sched_dl_entity *dl_se = &p->dl;  in __setparam_dl()  local
2918 	dl_se->dl_runtime = attr->sched_runtime;  in __setparam_dl()
2919 	dl_se->dl_deadline = attr->sched_deadline;  in __setparam_dl()
2920 	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;  in __setparam_dl()
2921 	dl_se->flags = attr->sched_flags & SCHED_DL_FLAGS;  in __setparam_dl()
2922 	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);  in __setparam_dl()
2923 	dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);  in __setparam_dl()
2928 	struct sched_dl_entity *dl_se = &p->dl;  in __getparam_dl()  local
2931 	attr->sched_runtime = dl_se->dl_runtime;  in __getparam_dl()
2932 	attr->sched_deadline = dl_se->dl_deadline;  in __getparam_dl()
2933 	attr->sched_period = dl_se->dl_period;  in __getparam_dl()
2935 	attr->sched_flags |= dl_se->flags;  in __getparam_dl()
2998 	struct sched_dl_entity *dl_se = &p->dl;  in __dl_clear_params()  local
3000 	dl_se->dl_runtime		= 0;  in __dl_clear_params()
3001 	dl_se->dl_deadline		= 0;  in __dl_clear_params()
3002 	dl_se->dl_period		= 0;  in __dl_clear_params()
3003 	dl_se->flags			= 0;  in __dl_clear_params()
3004 	dl_se->dl_bw			= 0;  in __dl_clear_params()
3005 	dl_se->dl_density		= 0;  in __dl_clear_params()
3007 	dl_se->dl_throttled		= 0;  in __dl_clear_params()
3008 	dl_se->dl_yielded		= 0;  in __dl_clear_params()
3009 	dl_se->dl_non_contending	= 0;  in __dl_clear_params()
3010 	dl_se->dl_overrun		= 0;  in __dl_clear_params()
3013 	dl_se->pi_se			= dl_se;  in __dl_clear_params()
3019 	struct sched_dl_entity *dl_se = &p->dl;  in dl_param_changed()  local
3021 	if (dl_se->dl_runtime != attr->sched_runtime ||  in dl_param_changed()
3022 	    dl_se->dl_deadline != attr->sched_deadline ||  in dl_param_changed()
3023 	    dl_se->dl_period != attr->sched_period ||  in dl_param_changed()
3024 	    dl_se->flags != (attr->sched_flags & SCHED_DL_FLAGS))  in dl_param_changed()