165 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)  argument
167 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)  in rt_task_of()  argument
170 	WARN_ON_ONCE(!rt_entity_is_task(rt_se));  in rt_task_of()
172 	return container_of(rt_se, struct task_struct, rt);  in rt_task_of()
180 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()  argument
182 	return rt_se->rt_rq;  in rt_rq_of_se()
185 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()  argument
187 	struct rt_rq *rt_rq = rt_se->rt_rq;  in rq_of_rt_se()
194 	if (tg->rt_se)  in unregister_rt_sched_group()
206 		if (tg->rt_se)  in free_rt_sched_group()
207 			kfree(tg->rt_se[i]);  in free_rt_sched_group()
211 	kfree(tg->rt_se);  in free_rt_sched_group()
215 		struct sched_rt_entity *rt_se, int cpu,  in init_tg_rt_entry()  argument
226 	tg->rt_se[cpu] = rt_se;  in init_tg_rt_entry()
228 	if (!rt_se)  in init_tg_rt_entry()
232 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
234 		rt_se->rt_rq = parent->my_q;  in init_tg_rt_entry()
236 	rt_se->my_q = rt_rq;  in init_tg_rt_entry()
237 	rt_se->parent = parent;  in init_tg_rt_entry()
238 	INIT_LIST_HEAD(&rt_se->run_list);  in init_tg_rt_entry()
244 	struct sched_rt_entity *rt_se;  in alloc_rt_sched_group()  local
250 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);  in alloc_rt_sched_group()
251 	if (!tg->rt_se)  in alloc_rt_sched_group()
263 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),  in alloc_rt_sched_group()
265 		if (!rt_se)  in alloc_rt_sched_group()
270 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
283 #define rt_entity_is_task(rt_se) (1)  argument
285 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)  in rt_task_of()  argument
287 	return container_of(rt_se, struct task_struct, rt);  in rt_task_of()
295 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()  argument
297 	struct task_struct *p = rt_task_of(rt_se);  in rq_of_rt_se()
302 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()  argument
304 	struct rq *rq = rq_of_rt_se(rt_se);  in rt_rq_of_se()
374 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
378 	if (!rt_entity_is_task(rt_se))  in inc_rt_migration()
381 	p = rt_task_of(rt_se);  in inc_rt_migration()
391 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
395 	if (!rt_entity_is_task(rt_se))  in dec_rt_migration()
398 	p = rt_task_of(rt_se);  in dec_rt_migration()
468 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
473 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
485 static inline int on_rt_rq(struct sched_rt_entity *rt_se)  in on_rt_rq()  argument
487 	return rt_se->on_rq;  in on_rt_rq()
564 #define for_each_sched_rt_entity(rt_se) \  argument
565 	for (; rt_se; rt_se = rt_se->parent)
567 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()  argument
569 	return rt_se->my_q;  in group_rt_rq()
572 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
573 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
579 	struct sched_rt_entity *rt_se;  in sched_rt_rq_enqueue()  local
583 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
586 		if (!rt_se)  in sched_rt_rq_enqueue()
588 		else if (!on_rt_rq(rt_se))  in sched_rt_rq_enqueue()
589 			enqueue_rt_entity(rt_se, 0);  in sched_rt_rq_enqueue()
598 	struct sched_rt_entity *rt_se;  in sched_rt_rq_dequeue()  local
601 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
603 	if (!rt_se) {  in sched_rt_rq_dequeue()
608 	else if (on_rt_rq(rt_se))  in sched_rt_rq_dequeue()
609 		dequeue_rt_entity(rt_se, 0);  in sched_rt_rq_dequeue()
617 static int rt_se_boosted(struct sched_rt_entity *rt_se)  in rt_se_boosted()  argument
619 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_boosted()
625 	p = rt_task_of(rt_se);  in rt_se_boosted()
669 #define for_each_sched_rt_entity(rt_se) \  argument
670 	for (; rt_se; rt_se = NULL)
672 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()  argument
986 static inline int rt_se_prio(struct sched_rt_entity *rt_se)  in rt_se_prio()  argument
989 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_prio()
995 	return rt_task_of(rt_se)->prio;  in rt_se_prio()
1048 	struct sched_rt_entity *rt_se = &curr->rt;  in update_curr_rt()  local
1070 	for_each_sched_rt_entity(rt_se) {  in update_curr_rt()
1071 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in update_curr_rt()
1218 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1220 	if (rt_se_boosted(rt_se))  in inc_rt_group()
1228 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_group()  argument
1230 	if (rt_se_boosted(rt_se))  in dec_rt_group()
1239 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1245 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}  in dec_rt_group()  argument
1250 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)  in rt_se_nr_running()  argument
1252 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_nr_running()
1261 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)  in rt_se_rr_nr_running()  argument
1263 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_rr_nr_running()
1269 	tsk = rt_task_of(rt_se);  in rt_se_rr_nr_running()
1275 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_tasks()  argument
1277 	int prio = rt_se_prio(rt_se);  in inc_rt_tasks()
1280 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);  in inc_rt_tasks()
1281 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);  in inc_rt_tasks()
1284 	inc_rt_migration(rt_se, rt_rq);  in inc_rt_tasks()
1285 	inc_rt_group(rt_se, rt_rq);  in inc_rt_tasks()
1289 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_tasks()  argument
1291 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));  in dec_rt_tasks()
1293 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);  in dec_rt_tasks()
1294 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);  in dec_rt_tasks()
1296 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));  in dec_rt_tasks()
1297 	dec_rt_migration(rt_se, rt_rq);  in dec_rt_tasks()
1298 	dec_rt_group(rt_se, rt_rq);  in dec_rt_tasks()
1314 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)  in __delist_rt_entity()  argument
1316 	list_del_init(&rt_se->run_list);  in __delist_rt_entity()
1318 	if (list_empty(array->queue + rt_se_prio(rt_se)))  in __delist_rt_entity()
1319 		__clear_bit(rt_se_prio(rt_se), array->bitmap);  in __delist_rt_entity()
1321 	rt_se->on_list = 0;  in __delist_rt_entity()
1325 __schedstats_from_rt_se(struct sched_rt_entity *rt_se)  in __schedstats_from_rt_se()  argument
1329 	if (!rt_entity_is_task(rt_se))  in __schedstats_from_rt_se()
1333 	return &rt_task_of(rt_se)->stats;  in __schedstats_from_rt_se()
1337 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_wait_start_rt()  argument
1345 	if (rt_entity_is_task(rt_se))  in update_stats_wait_start_rt()
1346 		p = rt_task_of(rt_se);  in update_stats_wait_start_rt()
1348 	stats = __schedstats_from_rt_se(rt_se);  in update_stats_wait_start_rt()
1356 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_enqueue_sleeper_rt()  argument
1364 	if (rt_entity_is_task(rt_se))  in update_stats_enqueue_sleeper_rt()
1365 		p = rt_task_of(rt_se);  in update_stats_enqueue_sleeper_rt()
1367 	stats = __schedstats_from_rt_se(rt_se);  in update_stats_enqueue_sleeper_rt()
1375 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,  in update_stats_enqueue_rt()  argument
1382 		update_stats_enqueue_sleeper_rt(rt_rq, rt_se);  in update_stats_enqueue_rt()
1386 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_wait_end_rt()  argument
1394 	if (rt_entity_is_task(rt_se))  in update_stats_wait_end_rt()
1395 		p = rt_task_of(rt_se);  in update_stats_wait_end_rt()
1397 	stats = __schedstats_from_rt_se(rt_se);  in update_stats_wait_end_rt()
1405 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,  in update_stats_dequeue_rt()  argument
1413 	if (rt_entity_is_task(rt_se))  in update_stats_dequeue_rt()
1414 		p = rt_task_of(rt_se);  in update_stats_dequeue_rt()
1430 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in __enqueue_rt_entity()  argument
1432 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __enqueue_rt_entity()
1434 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in __enqueue_rt_entity()
1435 	struct list_head *queue = array->queue + rt_se_prio(rt_se);  in __enqueue_rt_entity()
1444 		if (rt_se->on_list)  in __enqueue_rt_entity()
1445 			__delist_rt_entity(rt_se, array);  in __enqueue_rt_entity()
1450 		WARN_ON_ONCE(rt_se->on_list);  in __enqueue_rt_entity()
1452 			list_add(&rt_se->run_list, queue);  in __enqueue_rt_entity()
1454 			list_add_tail(&rt_se->run_list, queue);  in __enqueue_rt_entity()
1456 		__set_bit(rt_se_prio(rt_se), array->bitmap);  in __enqueue_rt_entity()
1457 		rt_se->on_list = 1;  in __enqueue_rt_entity()
1459 	rt_se->on_rq = 1;  in __enqueue_rt_entity()
1461 	inc_rt_tasks(rt_se, rt_rq);  in __enqueue_rt_entity()
1464 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in __dequeue_rt_entity()  argument
1466 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __dequeue_rt_entity()
1470 		WARN_ON_ONCE(!rt_se->on_list);  in __dequeue_rt_entity()
1471 		__delist_rt_entity(rt_se, array);  in __dequeue_rt_entity()
1473 	rt_se->on_rq = 0;  in __dequeue_rt_entity()
1475 	dec_rt_tasks(rt_se, rt_rq);  in __dequeue_rt_entity()
1482 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)  in dequeue_rt_stack()  argument
1487 	for_each_sched_rt_entity(rt_se) {  in dequeue_rt_stack()
1488 		rt_se->back = back;  in dequeue_rt_stack()
1489 		back = rt_se;  in dequeue_rt_stack()
1494 	for (rt_se = back; rt_se; rt_se = rt_se->back) {  in dequeue_rt_stack()
1495 		if (on_rt_rq(rt_se))  in dequeue_rt_stack()
1496 			__dequeue_rt_entity(rt_se, flags);  in dequeue_rt_stack()
1502 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in enqueue_rt_entity()  argument
1504 	struct rq *rq = rq_of_rt_se(rt_se);  in enqueue_rt_entity()
1506 	update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);  in enqueue_rt_entity()
1508 	dequeue_rt_stack(rt_se, flags);  in enqueue_rt_entity()
1509 	for_each_sched_rt_entity(rt_se)  in enqueue_rt_entity()
1510 		__enqueue_rt_entity(rt_se, flags);  in enqueue_rt_entity()
1514 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in dequeue_rt_entity()  argument
1516 	struct rq *rq = rq_of_rt_se(rt_se);  in dequeue_rt_entity()
1518 	update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);  in dequeue_rt_entity()
1520 	dequeue_rt_stack(rt_se, flags);  in dequeue_rt_entity()
1522 	for_each_sched_rt_entity(rt_se) {  in dequeue_rt_entity()
1523 		struct rt_rq *rt_rq = group_rt_rq(rt_se);  in dequeue_rt_entity()
1526 			__enqueue_rt_entity(rt_se, flags);  in dequeue_rt_entity()
1537 	struct sched_rt_entity *rt_se = &p->rt;  in enqueue_task_rt()  local
1540 		rt_se->timeout = 0;  in enqueue_task_rt()
1543 	update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);  in enqueue_task_rt()
1545 	enqueue_rt_entity(rt_se, flags);  in enqueue_task_rt()
1553 	struct sched_rt_entity *rt_se = &p->rt;  in dequeue_task_rt()  local
1556 	dequeue_rt_entity(rt_se, flags);  in dequeue_task_rt()
1566 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)  in requeue_rt_entity()  argument
1568 	if (on_rt_rq(rt_se)) {  in requeue_rt_entity()
1570 		struct list_head *queue = array->queue + rt_se_prio(rt_se);  in requeue_rt_entity()
1573 			list_move(&rt_se->run_list, queue);  in requeue_rt_entity()
1575 			list_move_tail(&rt_se->run_list, queue);  in requeue_rt_entity()
1581 	struct sched_rt_entity *rt_se = &p->rt;  in requeue_task_rt()  local
1584 	for_each_sched_rt_entity(rt_se) {  in requeue_task_rt()
1585 		rt_rq = rt_rq_of_se(rt_se);  in requeue_task_rt()
1586 		requeue_rt_entity(rt_rq, rt_se, head);  in requeue_task_rt()
1745 	struct sched_rt_entity *rt_se = &p->rt;  in set_next_task_rt()  local
1750 		update_stats_wait_end_rt(rt_rq, rt_se);  in set_next_task_rt()
1787 	struct sched_rt_entity *rt_se;  in _pick_next_task_rt()  local
1791 		rt_se = pick_next_rt_entity(rt_rq);  in _pick_next_task_rt()
1792 		BUG_ON(!rt_se);  in _pick_next_task_rt()
1793 		rt_rq = group_rt_rq(rt_se);  in _pick_next_task_rt()
1796 	return rt_task_of(rt_se);  in _pick_next_task_rt()
1823 	struct sched_rt_entity *rt_se = &p->rt;  in put_prev_task_rt()  local
1827 		update_stats_wait_start_rt(rt_rq, rt_se);  in put_prev_task_rt()
2634 	struct sched_rt_entity *rt_se = &p->rt;  in task_tick_rt()  local
2657 	for_each_sched_rt_entity(rt_se) {  in task_tick_rt()
2658 		if (rt_se->run_list.prev != rt_se->run_list.next) {  in task_tick_rt()