23 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)  in dl_task_of()  argument
25 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()  argument
35 	struct task_struct *p = dl_task_of(dl_se);  in dl_rq_of_se()
41 static inline int on_dl_rq(struct sched_dl_entity *dl_se)  in on_dl_rq()  argument
43 	return !RB_EMPTY_NODE(&dl_se->rb_node);  in on_dl_rq()
129 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_rq_bw()  argument
131 	if (!dl_entity_is_special(dl_se))  in add_rq_bw()
132 		__add_rq_bw(dl_se->dl_bw, dl_rq);  in add_rq_bw()
136 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_rq_bw()  argument
138 	if (!dl_entity_is_special(dl_se))  in sub_rq_bw()
139 		__sub_rq_bw(dl_se->dl_bw, dl_rq);  in sub_rq_bw()
143 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_running_bw()  argument
145 	if (!dl_entity_is_special(dl_se))  in add_running_bw()
146 		__add_running_bw(dl_se->dl_bw, dl_rq);  in add_running_bw()
150 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_running_bw()  argument
152 	if (!dl_entity_is_special(dl_se))  in sub_running_bw()
153 		__sub_running_bw(dl_se->dl_bw, dl_rq);  in sub_running_bw()
239 	struct sched_dl_entity *dl_se = &p->dl;  in task_non_contending()  local
240 	struct hrtimer *timer = &dl_se->inactive_timer;  in task_non_contending()
241 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_non_contending()
249 	if (dl_se->dl_runtime == 0)  in task_non_contending()
252 	if (dl_entity_is_special(dl_se))  in task_non_contending()
255 	WARN_ON(hrtimer_active(&dl_se->inactive_timer));  in task_non_contending()
256 	WARN_ON(dl_se->dl_non_contending);  in task_non_contending()
258 	zerolag_time = dl_se->deadline -  in task_non_contending()
259 		 div64_long((dl_se->runtime * dl_se->dl_period),  in task_non_contending()
260 			dl_se->dl_runtime);  in task_non_contending()
274 			sub_running_bw(dl_se, dl_rq);  in task_non_contending()
289 	dl_se->dl_non_contending = 1;  in task_non_contending()
294 static void task_contending(struct sched_dl_entity *dl_se, int flags)  in task_contending()  argument
296 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_contending()
302 	if (dl_se->dl_runtime == 0)  in task_contending()
306 		add_rq_bw(dl_se, dl_rq);  in task_contending()
308 	if (dl_se->dl_non_contending) {  in task_contending()
309 		dl_se->dl_non_contending = 0;  in task_contending()
317 		if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1)  in task_contending()
318 			put_task_struct(dl_task_of(dl_se));  in task_contending()
327 		add_running_bw(dl_se, dl_rq);  in task_contending()
333 	struct sched_dl_entity *dl_se = &p->dl;  in is_leftmost()  local
335 	return dl_rq->root.rb_leftmost == &dl_se->rb_node;  in is_leftmost()
422 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
424 	struct task_struct *p = dl_task_of(dl_se);  in inc_dl_migration()
432 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
434 	struct task_struct *p = dl_task_of(dl_se);  in dec_dl_migration()
580 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
585 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
623 static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se)  in setup_new_dl_entity()  argument
625 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()
628 	WARN_ON(dl_se->dl_boosted);  in setup_new_dl_entity()
629 	WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline));  in setup_new_dl_entity()
636 	if (dl_se->dl_throttled)  in setup_new_dl_entity()
644 	dl_se->deadline = rq_clock(rq) + dl_se->dl_deadline;  in setup_new_dl_entity()
645 	dl_se->runtime = dl_se->dl_runtime;  in setup_new_dl_entity()
666 static void replenish_dl_entity(struct sched_dl_entity *dl_se,  in replenish_dl_entity()  argument
669 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()
678 	if (dl_se->dl_deadline == 0) {  in replenish_dl_entity()
679 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
680 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
683 	if (dl_se->dl_yielded && dl_se->runtime > 0)  in replenish_dl_entity()
684 		dl_se->runtime = 0;  in replenish_dl_entity()
692 	while (dl_se->runtime <= 0) {  in replenish_dl_entity()
693 		dl_se->deadline += pi_se->dl_period;  in replenish_dl_entity()
694 		dl_se->runtime += pi_se->dl_runtime;  in replenish_dl_entity()
706 	if (dl_time_before(dl_se->deadline, rq_clock(rq))) {  in replenish_dl_entity()
708 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in replenish_dl_entity()
709 		dl_se->runtime = pi_se->dl_runtime;  in replenish_dl_entity()
712 	if (dl_se->dl_yielded)  in replenish_dl_entity()
713 		dl_se->dl_yielded = 0;  in replenish_dl_entity()
714 	if (dl_se->dl_throttled)  in replenish_dl_entity()
715 		dl_se->dl_throttled = 0;  in replenish_dl_entity()
742 static bool dl_entity_overflow(struct sched_dl_entity *dl_se,  in dl_entity_overflow()  argument
765 	left = (pi_se->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);  in dl_entity_overflow()
766 	right = ((dl_se->deadline - t) >> DL_SCALE) *  in dl_entity_overflow()
791 update_dl_revised_wakeup(struct sched_dl_entity *dl_se, struct rq *rq)  in update_dl_revised_wakeup()  argument
793 	u64 laxity = dl_se->deadline - rq_clock(rq);  in update_dl_revised_wakeup()
801 	WARN_ON(dl_time_before(dl_se->deadline, rq_clock(rq)));  in update_dl_revised_wakeup()
803 	dl_se->runtime = (dl_se->dl_density * laxity) >> BW_SHIFT;  in update_dl_revised_wakeup()
817 static inline bool dl_is_implicit(struct sched_dl_entity *dl_se)  in dl_is_implicit()  argument
819 	return dl_se->dl_deadline == dl_se->dl_period;  in dl_is_implicit()
852 static void update_dl_entity(struct sched_dl_entity *dl_se,  in update_dl_entity()  argument
855 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()
858 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||  in update_dl_entity()
859 	    dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) {  in update_dl_entity()
861 		if (unlikely(!dl_is_implicit(dl_se) &&  in update_dl_entity()
862 			     !dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in update_dl_entity()
863 			     !dl_se->dl_boosted)){  in update_dl_entity()
864 			update_dl_revised_wakeup(dl_se, rq);  in update_dl_entity()
868 		dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline;  in update_dl_entity()
869 		dl_se->runtime = pi_se->dl_runtime;  in update_dl_entity()
873 static inline u64 dl_next_period(struct sched_dl_entity *dl_se)  in dl_next_period()  argument
875 	return dl_se->deadline - dl_se->dl_deadline + dl_se->dl_period;  in dl_next_period()
890 	struct sched_dl_entity *dl_se = &p->dl;  in start_dl_timer()  local
891 	struct hrtimer *timer = &dl_se->dl_timer;  in start_dl_timer()
903 	act = ns_to_ktime(dl_next_period(dl_se));  in start_dl_timer()
948 	struct sched_dl_entity *dl_se = container_of(timer,  in dl_task_timer()  local
951 	struct task_struct *p = dl_task_of(dl_se);  in dl_task_timer()
968 	if (dl_se->dl_boosted)  in dl_task_timer()
975 	if (!dl_se->dl_throttled)  in dl_task_timer()
996 		replenish_dl_entity(dl_se, dl_se);  in dl_task_timer()
1053 void init_dl_task_timer(struct sched_dl_entity *dl_se)  in init_dl_task_timer()  argument
1055 	struct hrtimer *timer = &dl_se->dl_timer;  in init_dl_task_timer()
1079 static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se)  in dl_check_constrained_dl()  argument
1081 	struct task_struct *p = dl_task_of(dl_se);  in dl_check_constrained_dl()
1082 	struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se));  in dl_check_constrained_dl()
1084 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) &&  in dl_check_constrained_dl()
1085 	    dl_time_before(rq_clock(rq), dl_next_period(dl_se))) {  in dl_check_constrained_dl()
1086 		if (unlikely(dl_se->dl_boosted || !start_dl_timer(p)))  in dl_check_constrained_dl()
1088 		dl_se->dl_throttled = 1;  in dl_check_constrained_dl()
1089 		if (dl_se->runtime > 0)  in dl_check_constrained_dl()
1090 			dl_se->runtime = 0;  in dl_check_constrained_dl()
1095 int dl_runtime_exceeded(struct sched_dl_entity *dl_se)  in dl_runtime_exceeded()  argument
1097 	return (dl_se->runtime <= 0);  in dl_runtime_exceeded()
1121 static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se)  in grub_reclaim()  argument
1125 	u64 u_act_min = (dl_se->dl_bw * rq->dl.bw_ratio) >> RATIO_SHIFT;  in grub_reclaim()
1150 	struct sched_dl_entity *dl_se = &curr->dl;  in update_curr_dl()  local
1155 	if (!dl_task(curr) || !on_dl_rq(dl_se))  in update_curr_dl()
1169 		if (unlikely(dl_se->dl_yielded))  in update_curr_dl()
1183 	if (dl_entity_is_special(dl_se))  in update_curr_dl()
1193 	if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) {  in update_curr_dl()
1205 	dl_se->runtime -= scaled_delta_exec;  in update_curr_dl()
1208 	if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) {  in update_curr_dl()
1209 		dl_se->dl_throttled = 1;  in update_curr_dl()
1212 		if (dl_runtime_exceeded(dl_se) &&  in update_curr_dl()
1213 		    (dl_se->flags & SCHED_FLAG_DL_OVERRUN))  in update_curr_dl()
1214 			dl_se->dl_overrun = 1;  in update_curr_dl()
1217 		if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr)))  in update_curr_dl()
1252 	struct sched_dl_entity *dl_se = container_of(timer,  in inactive_task_timer()  local
1255 	struct task_struct *p = dl_task_of(dl_se);  in inactive_task_timer()
1267 		if (p->state == TASK_DEAD && dl_se->dl_non_contending) {  in inactive_task_timer()
1270 			dl_se->dl_non_contending = 0;  in inactive_task_timer()
1280 	if (dl_se->dl_non_contending == 0)  in inactive_task_timer()
1283 	sub_running_bw(dl_se, &rq->dl);  in inactive_task_timer()
1284 	dl_se->dl_non_contending = 0;  in inactive_task_timer()
1292 void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)  in init_dl_inactive_task_timer()  argument
1294 	struct hrtimer *timer = &dl_se->inactive_timer;  in init_dl_inactive_task_timer()
1343 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
1345 	int prio = dl_task_of(dl_se)->prio;  in inc_dl_tasks()
1346 	u64 deadline = dl_se->deadline;  in inc_dl_tasks()
1353 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
1357 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
1359 	int prio = dl_task_of(dl_se)->prio;  in dec_dl_tasks()
1366 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
1367 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
1370 static void __enqueue_dl_entity(struct sched_dl_entity *dl_se)  in __enqueue_dl_entity()  argument
1372 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()
1378 	BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node));  in __enqueue_dl_entity()
1383 		if (dl_time_before(dl_se->deadline, entry->deadline))  in __enqueue_dl_entity()
1391 	rb_link_node(&dl_se->rb_node, parent, link);  in __enqueue_dl_entity()
1392 	rb_insert_color_cached(&dl_se->rb_node, &dl_rq->root, leftmost);  in __enqueue_dl_entity()
1394 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
1397 static void __dequeue_dl_entity(struct sched_dl_entity *dl_se)  in __dequeue_dl_entity()  argument
1399 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()
1401 	if (RB_EMPTY_NODE(&dl_se->rb_node))  in __dequeue_dl_entity()
1404 	rb_erase_cached(&dl_se->rb_node, &dl_rq->root);  in __dequeue_dl_entity()
1405 	RB_CLEAR_NODE(&dl_se->rb_node);  in __dequeue_dl_entity()
1407 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1411 enqueue_dl_entity(struct sched_dl_entity *dl_se,  in enqueue_dl_entity()  argument
1414 	BUG_ON(on_dl_rq(dl_se));  in enqueue_dl_entity()
1422 		task_contending(dl_se, flags);  in enqueue_dl_entity()
1423 		update_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
1425 		replenish_dl_entity(dl_se, pi_se);  in enqueue_dl_entity()
1427 		  dl_time_before(dl_se->deadline,  in enqueue_dl_entity()
1428 				 rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) {  in enqueue_dl_entity()
1429 		setup_new_dl_entity(dl_se);  in enqueue_dl_entity()
1432 	__enqueue_dl_entity(dl_se);  in enqueue_dl_entity()
1435 static void dequeue_dl_entity(struct sched_dl_entity *dl_se)  in dequeue_dl_entity()  argument
1437 	__dequeue_dl_entity(dl_se);  in dequeue_dl_entity()
1712 	struct sched_dl_entity *dl_se;  in pick_next_task_dl()  local
1749 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
1750 	BUG_ON(!dl_se);  in pick_next_task_dl()
1752 	p = dl_task_of(dl_se);  in pick_next_task_dl()
2580 	struct sched_dl_entity *dl_se = &p->dl;  in __setparam_dl()  local
2582 	dl_se->dl_runtime = attr->sched_runtime;  in __setparam_dl()
2583 	dl_se->dl_deadline = attr->sched_deadline;  in __setparam_dl()
2584 	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;  in __setparam_dl()
2585 	dl_se->flags = attr->sched_flags;  in __setparam_dl()
2586 	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);  in __setparam_dl()
2587 	dl_se->dl_density = to_ratio(dl_se->dl_deadline, dl_se->dl_runtime);  in __setparam_dl()
2592 	struct sched_dl_entity *dl_se = &p->dl;  in __getparam_dl()  local
2595 	attr->sched_runtime = dl_se->dl_runtime;  in __getparam_dl()
2596 	attr->sched_deadline = dl_se->dl_deadline;  in __getparam_dl()
2597 	attr->sched_period = dl_se->dl_period;  in __getparam_dl()
2598 	attr->sched_flags = dl_se->flags;  in __getparam_dl()
2650 	struct sched_dl_entity *dl_se = &p->dl;  in __dl_clear_params()  local
2652 	dl_se->dl_runtime		= 0;  in __dl_clear_params()
2653 	dl_se->dl_deadline		= 0;  in __dl_clear_params()
2654 	dl_se->dl_period		= 0;  in __dl_clear_params()
2655 	dl_se->flags			= 0;  in __dl_clear_params()
2656 	dl_se->dl_bw			= 0;  in __dl_clear_params()
2657 	dl_se->dl_density		= 0;  in __dl_clear_params()
2659 	dl_se->dl_throttled		= 0;  in __dl_clear_params()
2660 	dl_se->dl_yielded		= 0;  in __dl_clear_params()
2661 	dl_se->dl_non_contending	= 0;  in __dl_clear_params()
2662 	dl_se->dl_overrun		= 0;  in __dl_clear_params()
2667 	struct sched_dl_entity *dl_se = &p->dl;  in dl_param_changed()  local
2669 	if (dl_se->dl_runtime != attr->sched_runtime ||  in dl_param_changed()
2670 	    dl_se->dl_deadline != attr->sched_deadline ||  in dl_param_changed()
2671 	    dl_se->dl_period != attr->sched_period ||  in dl_param_changed()
2672 	    dl_se->flags != attr->sched_flags)  in dl_param_changed()