62 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()  argument
64 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
67 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()
206 	struct dl_rq *dl = container_of(dl_b, struct dl_rq, dl_bw);  in __dl_update()
234 void __add_running_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __add_running_bw()  argument
236 	u64 old = dl_rq->running_bw;  in __add_running_bw()
238 	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));  in __add_running_bw()
239 	dl_rq->running_bw += dl_bw;  in __add_running_bw()
240 	SCHED_WARN_ON(dl_rq->running_bw < old); /* overflow */  in __add_running_bw()
241 	SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);  in __add_running_bw()
243 	cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);  in __add_running_bw()
247 void __sub_running_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __sub_running_bw()  argument
249 	u64 old = dl_rq->running_bw;  in __sub_running_bw()
251 	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));  in __sub_running_bw()
252 	dl_rq->running_bw -= dl_bw;  in __sub_running_bw()
253 	SCHED_WARN_ON(dl_rq->running_bw > old); /* underflow */  in __sub_running_bw()
254 	if (dl_rq->running_bw > old)  in __sub_running_bw()
255 		dl_rq->running_bw = 0;  in __sub_running_bw()
257 	cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);  in __sub_running_bw()
261 void __add_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __add_rq_bw()  argument
263 	u64 old = dl_rq->this_bw;  in __add_rq_bw()
265 	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));  in __add_rq_bw()
266 	dl_rq->this_bw += dl_bw;  in __add_rq_bw()
267 	SCHED_WARN_ON(dl_rq->this_bw < old); /* overflow */  in __add_rq_bw()
271 void __sub_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __sub_rq_bw()  argument
273 	u64 old = dl_rq->this_bw;  in __sub_rq_bw()
275 	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));  in __sub_rq_bw()
276 	dl_rq->this_bw -= dl_bw;  in __sub_rq_bw()
277 	SCHED_WARN_ON(dl_rq->this_bw > old); /* underflow */  in __sub_rq_bw()
278 	if (dl_rq->this_bw > old)  in __sub_rq_bw()
279 		dl_rq->this_bw = 0;  in __sub_rq_bw()
280 	SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);  in __sub_rq_bw()
284 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_rq_bw()  argument
287 		__add_rq_bw(dl_se->dl_bw, dl_rq);  in add_rq_bw()
291 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_rq_bw()  argument
294 		__sub_rq_bw(dl_se->dl_bw, dl_rq);  in sub_rq_bw()
298 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_running_bw()  argument
301 		__add_running_bw(dl_se->dl_bw, dl_rq);  in add_running_bw()
305 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_running_bw()  argument
308 		__sub_running_bw(dl_se->dl_bw, dl_rq);  in sub_running_bw()
396 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_non_contending()  local
397 	struct rq *rq = rq_of_dl_rq(dl_rq);  in task_non_contending()
428 			sub_running_bw(dl_se, dl_rq);  in task_non_contending()
450 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_contending()  local
460 		add_rq_bw(dl_se, dl_rq);  in task_contending()
481 		add_running_bw(dl_se, dl_rq);  in task_contending()
485 static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq)  in is_leftmost()  argument
489 	return rb_first_cached(&dl_rq->root) == &dl_se->rb_node;  in is_leftmost()
492 static void init_dl_rq_bw_ratio(struct dl_rq *dl_rq);
504 void init_dl_rq(struct dl_rq *dl_rq)  in init_dl_rq()  argument
506 	dl_rq->root = RB_ROOT_CACHED;  in init_dl_rq()
510 	dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0;  in init_dl_rq()
512 	dl_rq->dl_nr_migratory = 0;  in init_dl_rq()
513 	dl_rq->overloaded = 0;  in init_dl_rq()
514 	dl_rq->pushable_dl_tasks_root = RB_ROOT_CACHED;  in init_dl_rq()
516 	init_dl_bw(&dl_rq->dl_bw);  in init_dl_rq()
519 	dl_rq->running_bw = 0;  in init_dl_rq()
520 	dl_rq->this_bw = 0;  in init_dl_rq()
521 	init_dl_rq_bw_ratio(dl_rq);  in init_dl_rq()
556 static void update_dl_migration(struct dl_rq *dl_rq)  in update_dl_migration()  argument
558 	if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) {  in update_dl_migration()
559 		if (!dl_rq->overloaded) {  in update_dl_migration()
560 			dl_set_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
561 			dl_rq->overloaded = 1;  in update_dl_migration()
563 	} else if (dl_rq->overloaded) {  in update_dl_migration()
564 		dl_clear_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
565 		dl_rq->overloaded = 0;  in update_dl_migration()
569 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
574 		dl_rq->dl_nr_migratory++;  in inc_dl_migration()
576 	update_dl_migration(dl_rq);  in inc_dl_migration()
579 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
584 		dl_rq->dl_nr_migratory--;  in dec_dl_migration()
586 	update_dl_migration(dl_rq);  in dec_dl_migration()
616 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()  local
617 	struct rb_root_cached *root = &dl_rq->pushable_dl_tasks_root;  in dequeue_pushable_dl_task()
625 		dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline;  in dequeue_pushable_dl_task()
746 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
751 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
790 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()  local
791 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()
832 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()  local
833 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()
1013 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()  local
1014 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()
1454 static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in inc_dl_deadline()  argument
1456 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()
1458 	if (dl_rq->earliest_dl.curr == 0 ||  in inc_dl_deadline()
1459 	    dl_time_before(deadline, dl_rq->earliest_dl.curr)) {  in inc_dl_deadline()
1460 		if (dl_rq->earliest_dl.curr == 0)  in inc_dl_deadline()
1462 		dl_rq->earliest_dl.curr = deadline;  in inc_dl_deadline()
1467 static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in dec_dl_deadline()  argument
1469 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()
1475 	if (!dl_rq->dl_nr_running) {  in dec_dl_deadline()
1476 		dl_rq->earliest_dl.curr = 0;  in dec_dl_deadline()
1477 		dl_rq->earliest_dl.next = 0;  in dec_dl_deadline()
1481 		struct rb_node *leftmost = rb_first_cached(&dl_rq->root);  in dec_dl_deadline()
1484 		dl_rq->earliest_dl.curr = entry->deadline;  in dec_dl_deadline()
1491 static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in inc_dl_deadline()  argument
1492 static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in dec_dl_deadline()  argument
1497 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
1503 	dl_rq->dl_nr_running++;  in inc_dl_tasks()
1504 	add_nr_running(rq_of_dl_rq(dl_rq), 1);  in inc_dl_tasks()
1506 	inc_dl_deadline(dl_rq, deadline);  in inc_dl_tasks()
1507 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
1511 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
1516 	WARN_ON(!dl_rq->dl_nr_running);  in dec_dl_tasks()
1517 	dl_rq->dl_nr_running--;  in dec_dl_tasks()
1518 	sub_nr_running(rq_of_dl_rq(dl_rq), 1);  in dec_dl_tasks()
1520 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
1521 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
1536 update_stats_wait_start_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_wait_start_dl()  argument
1544 	__update_stats_wait_start(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_wait_start_dl()
1548 update_stats_wait_end_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_wait_end_dl()  argument
1556 	__update_stats_wait_end(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_wait_end_dl()
1560 update_stats_enqueue_sleeper_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se)  in update_stats_enqueue_sleeper_dl()  argument
1568 	__update_stats_enqueue_sleeper(rq_of_dl_rq(dl_rq), dl_task_of(dl_se), stats);  in update_stats_enqueue_sleeper_dl()
1572 update_stats_enqueue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,  in update_stats_enqueue_dl()  argument
1579 		update_stats_enqueue_sleeper_dl(dl_rq, dl_se);  in update_stats_enqueue_dl()
1583 update_stats_dequeue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,  in update_stats_dequeue_dl()  argument
1597 					rq_clock(rq_of_dl_rq(dl_rq)));  in update_stats_dequeue_dl()
1601 					rq_clock(rq_of_dl_rq(dl_rq)));  in update_stats_dequeue_dl()
1607 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()  local
1611 	rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less);  in __enqueue_dl_entity()
1613 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
1618 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()  local
1623 	rb_erase_cached(&dl_se->rb_node, &dl_rq->root);  in __dequeue_dl_entity()
1627 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1975 	struct dl_rq *dl_rq = &rq->dl;  in set_next_task_dl()  local
1979 		update_stats_wait_end_dl(dl_rq, dl_se);  in set_next_task_dl()
1996 static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq)  in pick_next_dl_entity()  argument
1998 	struct rb_node *left = rb_first_cached(&dl_rq->root);  in pick_next_dl_entity()
2009 	struct dl_rq *dl_rq = &rq->dl;  in pick_task_dl()  local
2015 	dl_se = pick_next_dl_entity(dl_rq);  in pick_task_dl()
2036 	struct dl_rq *dl_rq = &rq->dl;  in put_prev_task_dl()  local
2039 		update_stats_wait_start_dl(dl_rq, dl_se);  in put_prev_task_dl()
2797 static void init_dl_rq_bw_ratio(struct dl_rq *dl_rq)  in init_dl_rq_bw_ratio()  argument
2800 		dl_rq->bw_ratio = 1 << RATIO_SHIFT;  in init_dl_rq_bw_ratio()
2801 		dl_rq->max_bw = dl_rq->extra_bw = 1 << BW_SHIFT;  in init_dl_rq_bw_ratio()
2803 		dl_rq->bw_ratio = to_ratio(global_rt_runtime(),  in init_dl_rq_bw_ratio()
2805 		dl_rq->max_bw = dl_rq->extra_bw =  in init_dl_rq_bw_ratio()