28 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)  in rq_of_dl_rq()  argument
30 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
33 static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)  in dl_rq_of_se()
79 void __add_running_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __add_running_bw()  argument
81 	u64 old = dl_rq->running_bw;  in __add_running_bw()
83 	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);  in __add_running_bw()
84 	dl_rq->running_bw += dl_bw;  in __add_running_bw()
85 	SCHED_WARN_ON(dl_rq->running_bw < old); /* overflow */  in __add_running_bw()
86 	SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);  in __add_running_bw()
88 	cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);  in __add_running_bw()
92 void __sub_running_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __sub_running_bw()  argument
94 	u64 old = dl_rq->running_bw;  in __sub_running_bw()
96 	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);  in __sub_running_bw()
97 	dl_rq->running_bw -= dl_bw;  in __sub_running_bw()
98 	SCHED_WARN_ON(dl_rq->running_bw > old); /* underflow */  in __sub_running_bw()
99 	if (dl_rq->running_bw > old)  in __sub_running_bw()
100 		dl_rq->running_bw = 0;  in __sub_running_bw()
102 	cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);  in __sub_running_bw()
106 void __add_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __add_rq_bw()  argument
108 	u64 old = dl_rq->this_bw;  in __add_rq_bw()
110 	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);  in __add_rq_bw()
111 	dl_rq->this_bw += dl_bw;  in __add_rq_bw()
112 	SCHED_WARN_ON(dl_rq->this_bw < old); /* overflow */  in __add_rq_bw()
116 void __sub_rq_bw(u64 dl_bw, struct dl_rq *dl_rq)  in __sub_rq_bw()  argument
118 	u64 old = dl_rq->this_bw;  in __sub_rq_bw()
120 	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);  in __sub_rq_bw()
121 	dl_rq->this_bw -= dl_bw;  in __sub_rq_bw()
122 	SCHED_WARN_ON(dl_rq->this_bw > old); /* underflow */  in __sub_rq_bw()
123 	if (dl_rq->this_bw > old)  in __sub_rq_bw()
124 		dl_rq->this_bw = 0;  in __sub_rq_bw()
125 	SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);  in __sub_rq_bw()
129 void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_rq_bw()  argument
132 		__add_rq_bw(dl_se->dl_bw, dl_rq);  in add_rq_bw()
136 void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_rq_bw()  argument
139 		__sub_rq_bw(dl_se->dl_bw, dl_rq);  in sub_rq_bw()
143 void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in add_running_bw()  argument
146 		__add_running_bw(dl_se->dl_bw, dl_rq);  in add_running_bw()
150 void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in sub_running_bw()  argument
153 		__sub_running_bw(dl_se->dl_bw, dl_rq);  in sub_running_bw()
241 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_non_contending()  local
242 	struct rq *rq = rq_of_dl_rq(dl_rq);  in task_non_contending()
273 			sub_running_bw(dl_se, dl_rq);  in task_non_contending()
295 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in task_contending()  local
305 		add_rq_bw(dl_se, dl_rq);  in task_contending()
326 		add_running_bw(dl_se, dl_rq);  in task_contending()
330 static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq)  in is_leftmost()  argument
334 	return dl_rq->root.rb_leftmost == &dl_se->rb_node;  in is_leftmost()
356 void init_dl_rq(struct dl_rq *dl_rq)  in init_dl_rq()  argument
358 	dl_rq->root = RB_ROOT_CACHED;  in init_dl_rq()
362 	dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0;  in init_dl_rq()
364 	dl_rq->dl_nr_migratory = 0;  in init_dl_rq()
365 	dl_rq->overloaded = 0;  in init_dl_rq()
366 	dl_rq->pushable_dl_tasks_root = RB_ROOT_CACHED;  in init_dl_rq()
368 	init_dl_bw(&dl_rq->dl_bw);  in init_dl_rq()
371 	dl_rq->running_bw = 0;  in init_dl_rq()
372 	dl_rq->this_bw = 0;  in init_dl_rq()
373 	init_dl_rq_bw_ratio(dl_rq);  in init_dl_rq()
408 static void update_dl_migration(struct dl_rq *dl_rq)  in update_dl_migration()  argument
410 	if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) {  in update_dl_migration()
411 		if (!dl_rq->overloaded) {  in update_dl_migration()
412 			dl_set_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
413 			dl_rq->overloaded = 1;  in update_dl_migration()
415 	} else if (dl_rq->overloaded) {  in update_dl_migration()
416 		dl_clear_overload(rq_of_dl_rq(dl_rq));  in update_dl_migration()
417 		dl_rq->overloaded = 0;  in update_dl_migration()
421 static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
426 		dl_rq->dl_nr_migratory++;  in inc_dl_migration()
428 	update_dl_migration(dl_rq);  in inc_dl_migration()
431 static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
436 		dl_rq->dl_nr_migratory--;  in dec_dl_migration()
438 	update_dl_migration(dl_rq);  in dec_dl_migration()
447 	struct dl_rq *dl_rq = &rq->dl;  in enqueue_pushable_dl_task()  local
448 	struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_root.rb_node;  in enqueue_pushable_dl_task()
468 		dl_rq->earliest_dl.next = p->dl.deadline;  in enqueue_pushable_dl_task()
472 			       &dl_rq->pushable_dl_tasks_root, leftmost);  in enqueue_pushable_dl_task()
477 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()  local
482 	if (dl_rq->pushable_dl_tasks_root.rb_leftmost == &p->pushable_dl_tasks) {  in dequeue_pushable_dl_task()
487 			dl_rq->earliest_dl.next = rb_entry(next_node,  in dequeue_pushable_dl_task()
492 	rb_erase_cached(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root);  in dequeue_pushable_dl_task()
612 void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_migration()  argument
617 void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_migration()  argument
657 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in setup_new_dl_entity()  local
658 	struct rq *rq = rq_of_dl_rq(dl_rq);  in setup_new_dl_entity()
701 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in replenish_dl_entity()  local
702 	struct rq *rq = rq_of_dl_rq(dl_rq);  in replenish_dl_entity()
887 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in update_dl_entity()  local
888 	struct rq *rq = rq_of_dl_rq(dl_rq);  in update_dl_entity()
1334 static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in inc_dl_deadline()  argument
1336 	struct rq *rq = rq_of_dl_rq(dl_rq);  in inc_dl_deadline()
1338 	if (dl_rq->earliest_dl.curr == 0 ||  in inc_dl_deadline()
1339 	    dl_time_before(deadline, dl_rq->earliest_dl.curr)) {  in inc_dl_deadline()
1340 		dl_rq->earliest_dl.curr = deadline;  in inc_dl_deadline()
1345 static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline)  in dec_dl_deadline()  argument
1347 	struct rq *rq = rq_of_dl_rq(dl_rq);  in dec_dl_deadline()
1353 	if (!dl_rq->dl_nr_running) {  in dec_dl_deadline()
1354 		dl_rq->earliest_dl.curr = 0;  in dec_dl_deadline()
1355 		dl_rq->earliest_dl.next = 0;  in dec_dl_deadline()
1358 		struct rb_node *leftmost = dl_rq->root.rb_leftmost;  in dec_dl_deadline()
1362 		dl_rq->earliest_dl.curr = entry->deadline;  in dec_dl_deadline()
1369 static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in inc_dl_deadline()  argument
1370 static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}  in dec_dl_deadline()  argument
1375 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in inc_dl_tasks()  argument
1381 	dl_rq->dl_nr_running++;  in inc_dl_tasks()
1382 	add_nr_running(rq_of_dl_rq(dl_rq), 1);  in inc_dl_tasks()
1384 	inc_dl_deadline(dl_rq, deadline);  in inc_dl_tasks()
1385 	inc_dl_migration(dl_se, dl_rq);  in inc_dl_tasks()
1389 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)  in dec_dl_tasks()  argument
1394 	WARN_ON(!dl_rq->dl_nr_running);  in dec_dl_tasks()
1395 	dl_rq->dl_nr_running--;  in dec_dl_tasks()
1396 	sub_nr_running(rq_of_dl_rq(dl_rq), 1);  in dec_dl_tasks()
1398 	dec_dl_deadline(dl_rq, dl_se->deadline);  in dec_dl_tasks()
1399 	dec_dl_migration(dl_se, dl_rq);  in dec_dl_tasks()
1404 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __enqueue_dl_entity()  local
1405 	struct rb_node **link = &dl_rq->root.rb_root.rb_node;  in __enqueue_dl_entity()
1424 	rb_insert_color_cached(&dl_se->rb_node, &dl_rq->root, leftmost);  in __enqueue_dl_entity()
1426 	inc_dl_tasks(dl_se, dl_rq);  in __enqueue_dl_entity()
1431 	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);  in __dequeue_dl_entity()  local
1436 	rb_erase_cached(&dl_se->rb_node, &dl_rq->root);  in __dequeue_dl_entity()
1439 	dec_dl_tasks(dl_se, dl_rq);  in __dequeue_dl_entity()
1763 						   struct dl_rq *dl_rq)  in pick_next_dl_entity()  argument
1765 	struct rb_node *left = rb_first_cached(&dl_rq->root);  in pick_next_dl_entity()
1777 	struct dl_rq *dl_rq = &rq->dl;  in pick_next_task_dl()  local
1785 	dl_se = pick_next_dl_entity(rq, dl_rq);  in pick_next_task_dl()
2499 void init_dl_rq_bw_ratio(struct dl_rq *dl_rq)  in init_dl_rq_bw_ratio()  argument
2502 		dl_rq->bw_ratio = 1 << RATIO_SHIFT;  in init_dl_rq_bw_ratio()
2503 		dl_rq->extra_bw = 1 << BW_SHIFT;  in init_dl_rq_bw_ratio()
2505 		dl_rq->bw_ratio = to_ratio(global_rt_runtime(),  in init_dl_rq_bw_ratio()
2507 		dl_rq->extra_bw = to_ratio(global_rt_period(),  in init_dl_rq_bw_ratio()