23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)  in dl_task_of()  argument
25 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()  argument
35 	struct task_struct *p = dl_task_of(dl_se);  in dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se)  in on_dl_rq()  argument
43 	return !RB_EMPTY_NODE(&dl_se->rb_node);  in on_dl_rq()
129 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_rq_bw()  argument
131 	if (!dl_entity_is_special(dl_se))  in add_rq_bw()
132 		__add_rq_bw(dl_se->dl_bw, dl_rq);  in add_rq_bw()
136 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_rq_bw()  argument
138 	if (!dl_entity_is_special(dl_se))  in sub_rq_bw()
139 		__sub_rq_bw(dl_se->dl_bw, dl_rq);  in sub_rq_bw()
143 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_running_bw()  argument
145 	if (!dl_entity_is_special(dl_se))  in add_running_bw()
146 		__add_running_bw(dl_se->dl_bw, dl_rq);  in add_running_bw()
150 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_running_bw()  argument
152 	if (!dl_entity_is_special(dl_se))  in sub_running_bw()
153 		__sub_running_bw(dl_se->dl_bw, dl_rq);  in sub_running_bw()
239 	struct sched_dl_entity *dl_se = &p->dl;  in task_non_contending()  local
240 	struct hrtimer *timer = &dl_se->inactive_timer;  in task_non_contending()
241 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_non_contending()
249 	if (dl_se->dl_runtime == 0)  in task_non_contending()
252 	if (dl_entity_is_special(dl_se))  in task_non_contending()
255 	WARN_ON(dl_se->dl_non_contending);  in task_non_contending()
257 	zerolag_time = dl_se->deadline -  in task_non_contending()
258 		 div64_long((dl_se->runtime * dl_se->dl_period),  in task_non_contending()
259 			dl_se->dl_runtime);  in task_non_contending()
271 	if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) {  in task_non_contending()
273 			sub_running_bw(dl_se, dl_rq);  in task_non_contending()
288 	dl_se->dl_non_contending = 1;  in task_non_contending()
293 static void task_contending(struct sched_dl_entity *dl_se, int flags)  in task_contending()  argument
295 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_contending()
301 	if (dl_se->dl_runtime == 0)  in task_contending()
305 		add_rq_bw(dl_se, dl_rq);  in task_contending()
307 	if (dl_se->dl_non_contending) {  in task_contending()
308 		dl_se->dl_non_contending = 0;  in task_contending()
316 		if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1)  in task_contending()
317 			put_task_struct(dl_task_of(dl_se));  in task_contending()
326 		add_running_bw(dl_se, dl_rq);  in task_contending()
332 	struct sched_dl_entity *dl_se = &p->dl;  in is_leftmost()  local
334 	return dl_rq->root.rb_leftmost == &dl_se->rb_node;  in is_leftmost()
421 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
423 	struct task_struct *p = dl_task_of(dl_se);  in inc_dl_migration()
431 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
433 	struct task_struct *p = dl_task_of(dl_se);  in dec_dl_migration()
612 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
617 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
655 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se)  in setup_new_dl_entity()  argument
657 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()
660 	WARN_ON(dl_se->dl_boosted);  in setup_new_dl_entity()
661 	WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline));  in setup_new_dl_entity()
668 	if (dl_se->dl_throttled)  in setup_new_dl_entity()
676 	dl_se->deadline = rq_clock(rq) + dl_se->dl_deadline;  in setup_new_dl_entity()
677 	dl_se->runtime = dl_se->dl_runtime;  in setup_new_dl_entity()
698 static void replenish_dl_entity(struct sched_dl_entity *dl_se,  in replenish_dl_entity()  argument
701 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()
710 	if (dl_se->dl_deadline == 0) {  in replenish_dl_entity()
711 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
712 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
715 	if (dl_se->dl_yielded && dl_se->runtime > 0)  in replenish_dl_entity()
716 		dl_se->runtime = 0;  in replenish_dl_entity()
724 	while (dl_se->runtime <= 0) {  in replenish_dl_entity()
725 		dl_se->deadline += pi_se->dl_period;  in replenish_dl_entity()
726 		dl_se->runtime += pi_se->dl_runtime;  in replenish_dl_entity()
738 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
740 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
741 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
744 	if (dl_se->dl_yielded)  in replenish_dl_entity()
745 		dl_se->dl_yielded = 0;  in replenish_dl_entity()
746 	if (dl_se->dl_throttled)  in replenish_dl_entity()
747 		dl_se->dl_throttled = 0;  in replenish_dl_entity()
774 static bool dl_entity_overflow(struct sched_dl_entity *dl_se,  in dl_entity_overflow()  argument
797 	left = (pi_se->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);  in dl_entity_overflow()
798 	right = ((dl_se->deadline - t) >> DL_SCALE) *  in dl_entity_overflow()
823 update_dl_revised_wakeup(struct sched_dl_entity *dl_se, struct rq *rq)  in update_dl_revised_wakeup()  argument
825 	u64 laxity = dl_se->deadline - rq_clock(rq);  in update_dl_revised_wakeup()
833 	WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq)));  in update_dl_revised_wakeup()
835 	dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT;  in update_dl_revised_wakeup()
849 static inline bool dl_is_implicit(struct sched_dl_entity *dl_se)  in dl_is_implicit()  argument
851 	return dl_se->dl_deadline == dl_se->dl_period;  in dl_is_implicit()
884 static void update_dl_entity(struct sched_dl_entity *dl_se,  in update_dl_entity()  argument
887 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()
890 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
891 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
893 		if (unlikely(!dl_is_implicit(dl_se) &&  in update_dl_entity()
894 			     !dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in update_dl_entity()
895 			     !dl_se->dl_boosted)){  in update_dl_entity()
896 			update_dl_revised_wakeup(dl_se, rq);  in update_dl_entity()
900 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
901 		dl_se->runtime = pi_se->dl_runtime;  in update_dl_entity()
905 static inline u64 dl_next_period(struct sched_dl_entity *dl_se)  in dl_next_period()  argument
907 	return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period;  in dl_next_period()
922 	struct sched_dl_entity *dl_se = &p->dl;  in start_dl_timer()  local
923 	struct hrtimer *timer = &dl_se->dl_timer;  in start_dl_timer()
935 	act = ns_to_ktime(dl_next_period(dl_se));  in start_dl_timer()
980 	struct sched_dl_entity *dl_se = container_of(timer,  in dl_task_timer()  local
983 	struct task_struct *p = dl_task_of(dl_se);  in dl_task_timer()
1000 	if (dl_se->dl_boosted)  in dl_task_timer()
1007 	if (!dl_se->dl_throttled)  in dl_task_timer()
1028 		replenish_dl_entity(dl_se, dl_se);  in dl_task_timer()
1085 void init_dl_task_timer(struct sched_dl_entity *dl_se)  in init_dl_task_timer()  argument
1087 	struct hrtimer *timer = &dl_se->dl_timer;  in init_dl_task_timer()
1111 static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se)  in dl_check_constrained_dl()  argument
1113 	struct task_struct *p = dl_task_of(dl_se);  in dl_check_constrained_dl()
1114 	struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se));  in dl_check_constrained_dl()
1116 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in dl_check_constrained_dl()
1117 	    dl_time_before(rq_clock(rq), dl_next_period(dl_se))) {  in dl_check_constrained_dl()
1118 		if (unlikely(dl_se->dl_boosted || !start_dl_timer(p)))  in dl_check_constrained_dl()
1120 		dl_se->dl_throttled = 1;  in dl_check_constrained_dl()
1121 		if (dl_se->runtime > 0)  in dl_check_constrained_dl()
1122 			dl_se->runtime = 0;  in dl_check_constrained_dl()
1127 int dl_runtime_exceeded(struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
1129 	return (dl_se->runtime <= 0);  in dl_runtime_exceeded()
1153 static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se)  in grub_reclaim()  argument
1157 	u64 u_act_min = (dl_se->dl_bw * rq->dl.bw_ratio) >> RATIO_SHIFT;  in grub_reclaim()
1182 	struct sched_dl_entity *dl_se = &curr->dl;  in update_curr_dl()  local
1187 	if (!dl_task(curr) || !on_dl_rq(dl_se))  in update_curr_dl()
1201 		if (unlikely(dl_se->dl_yielded))  in update_curr_dl()
1215 	if (dl_entity_is_special(dl_se))  in update_curr_dl()
1225 	if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) {  in update_curr_dl()
1237 	dl_se->runtime -= scaled_delta_exec;  in update_curr_dl()
1240 	if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) {  in update_curr_dl()
1241 		dl_se->dl_throttled = 1;  in update_curr_dl()
1244 		if (dl_runtime_exceeded(dl_se) &&  in update_curr_dl()
1245 		    (dl_se->flags & SCHED_FLAG_DL_OVERRUN))  in update_curr_dl()
1246 			dl_se->dl_overrun = 1;  in update_curr_dl()
1249 		if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr)))  in update_curr_dl()
1284 	struct sched_dl_entity *dl_se = container_of(timer,  in inactive_task_timer()  local
1287 	struct task_struct *p = dl_task_of(dl_se);  in inactive_task_timer()
1299 		if (p->state == TASK_DEAD && dl_se->dl_non_contending) {  in inactive_task_timer()
1302 			dl_se->dl_non_contending = 0;  in inactive_task_timer()
1312 	if (dl_se->dl_non_contending == 0)  in inactive_task_timer()
1315 	sub_running_bw(dl_se, &rq->dl);  in inactive_task_timer()
1316 	dl_se->dl_non_contending = 0;  in inactive_task_timer()
1324 void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)  in init_dl_inactive_task_timer()  argument
1326 	struct hrtimer *timer = &dl_se->inactive_timer;  in init_dl_inactive_task_timer()
1375 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
1377 	int prio = dl_task_of(dl_se)->prio;  in inc_dl_tasks()
1378 	u64 deadline = dl_se->deadline;  in inc_dl_tasks()
1385 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
1389 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
1391 	int prio = dl_task_of(dl_se)->prio;  in dec_dl_tasks()
1398 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
1399 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
1402 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se)  in __enqueue_dl_entity()  argument
1404 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()
1410 	BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node));  in __enqueue_dl_entity()
1415 		if (dl_time_before(dl_se->deadline, entry->deadline))  in __enqueue_dl_entity()
1423 	rb_link_node(&dl_se->rb_node, parent, link);  in __enqueue_dl_entity()
1424 	rb_insert_color_cached(&dl_se->rb_node, &dl_rq->root, leftmost);  in __enqueue_dl_entity()
1426 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
1429 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se)  in __dequeue_dl_entity()  argument
1431 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()
1433 	if (RB_EMPTY_NODE(&dl_se->rb_node))  in __dequeue_dl_entity()
1436 	rb_erase_cached(&dl_se->rb_node, &dl_rq->root);  in __dequeue_dl_entity()
1437 	RB_CLEAR_NODE(&dl_se->rb_node);  in __dequeue_dl_entity()
1439 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1443 enqueue_dl_entity(struct sched_dl_entity *dl_se,  in enqueue_dl_entity()  argument
1446 	BUG_ON(on_dl_rq(dl_se));  in enqueue_dl_entity()
1454 		task_contending(dl_se, flags);  in enqueue_dl_entity()
1455 		update_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
1457 		replenish_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
1459 		  dl_time_before(dl_se->deadline,  in enqueue_dl_entity()
1460 				 rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) {  in enqueue_dl_entity()
1461 		setup_new_dl_entity(dl_se);  in enqueue_dl_entity()
1464 	__enqueue_dl_entity(dl_se);  in enqueue_dl_entity()
1467 static void dequeue_dl_entity(struct sched_dl_entity *dl_se)  in dequeue_dl_entity()  argument
1469 	__dequeue_dl_entity(dl_se);  in dequeue_dl_entity()
1776 	struct sched_dl_entity *dl_se;  in pick_next_task_dl()  local
1785 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1786 	BUG_ON(!dl_se);  in pick_next_task_dl()
1787 	p = dl_task_of(dl_se);  in pick_next_task_dl()
2614 	struct sched_dl_entity *dl_se = &p->dl;  in __setparam_dl()  local
2616 	dl_se->dl_runtime = attr->sched_runtime;  in __setparam_dl()
2617 	dl_se->dl_deadline = attr->sched_deadline;  in __setparam_dl()
2618 	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;  in __setparam_dl()
2619 	dl_se->flags = attr->sched_flags;  in __setparam_dl()
2620 	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);  in __setparam_dl()
2621 	dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);  in __setparam_dl()
2626 	struct sched_dl_entity *dl_se = &p->dl;  in __getparam_dl()  local
2629 	attr->sched_runtime = dl_se->dl_runtime;  in __getparam_dl()
2630 	attr->sched_deadline = dl_se->dl_deadline;  in __getparam_dl()
2631 	attr->sched_period = dl_se->dl_period;  in __getparam_dl()
2632 	attr->sched_flags = dl_se->flags;  in __getparam_dl()
2684 	struct sched_dl_entity *dl_se = &p->dl;  in __dl_clear_params()  local
2686 	dl_se->dl_runtime		= 0;  in __dl_clear_params()
2687 	dl_se->dl_deadline		= 0;  in __dl_clear_params()
2688 	dl_se->dl_period		= 0;  in __dl_clear_params()
2689 	dl_se->flags			= 0;  in __dl_clear_params()
2690 	dl_se->dl_bw			= 0;  in __dl_clear_params()
2691 	dl_se->dl_density		= 0;  in __dl_clear_params()
2693 	dl_se->dl_throttled		= 0;  in __dl_clear_params()
2694 	dl_se->dl_yielded		= 0;  in __dl_clear_params()
2695 	dl_se->dl_non_contending	= 0;  in __dl_clear_params()
2696 	dl_se->dl_overrun		= 0;  in __dl_clear_params()
2701 	struct sched_dl_entity *dl_se = &p->dl;  in dl_param_changed()  local
2703 	if (dl_se->dl_runtime != attr->sched_runtime ||  in dl_param_changed()
2704 	    dl_se->dl_deadline != attr->sched_deadline ||  in dl_param_changed()
2705 	    dl_se->dl_period != attr->sched_period ||  in dl_param_changed()
2706 	    dl_se->flags != attr->sched_flags)  in dl_param_changed()