113 #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)  argument
115 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)  in rt_task_of()  argument
118 	WARN_ON_ONCE(!rt_entity_is_task(rt_se));  in rt_task_of()
120 	return container_of(rt_se, struct task_struct, rt);  in rt_task_of()
128 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()  argument
130 	return rt_se->rt_rq;  in rt_rq_of_se()
133 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()  argument
135 	struct rt_rq *rt_rq = rt_se->rt_rq;  in rq_of_rt_se()
144 	if (tg->rt_se)  in free_rt_sched_group()
150 		if (tg->rt_se)  in free_rt_sched_group()
151 			kfree(tg->rt_se[i]);  in free_rt_sched_group()
155 	kfree(tg->rt_se);  in free_rt_sched_group()
159 		struct sched_rt_entity *rt_se, int cpu,  in init_tg_rt_entry()  argument
170 	tg->rt_se[cpu] = rt_se;  in init_tg_rt_entry()
172 	if (!rt_se)  in init_tg_rt_entry()
176 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
178 		rt_se->rt_rq = parent->my_q;  in init_tg_rt_entry()
180 	rt_se->my_q = rt_rq;  in init_tg_rt_entry()
181 	rt_se->parent = parent;  in init_tg_rt_entry()
182 	INIT_LIST_HEAD(&rt_se->run_list);  in init_tg_rt_entry()
188 	struct sched_rt_entity *rt_se;  in alloc_rt_sched_group()  local
194 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);  in alloc_rt_sched_group()
195 	if (!tg->rt_se)  in alloc_rt_sched_group()
207 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),  in alloc_rt_sched_group()
209 		if (!rt_se)  in alloc_rt_sched_group()
214 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
227 #define rt_entity_is_task(rt_se) (1)  argument
229 static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)  in rt_task_of()  argument
231 	return container_of(rt_se, struct task_struct, rt);  in rt_task_of()
239 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()  argument
241 	struct task_struct *p = rt_task_of(rt_se);  in rq_of_rt_se()
246 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()  argument
248 	struct rq *rq = rq_of_rt_se(rt_se);  in rt_rq_of_se()
318 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
322 	if (!rt_entity_is_task(rt_se))  in inc_rt_migration()
325 	p = rt_task_of(rt_se);  in inc_rt_migration()
335 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
339 	if (!rt_entity_is_task(rt_se))  in dec_rt_migration()
342 	p = rt_task_of(rt_se);  in dec_rt_migration()
412 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
417 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
438 static inline int on_rt_rq(struct sched_rt_entity *rt_se)  in on_rt_rq()  argument
440 	return rt_se->on_rq;  in on_rt_rq()
517 #define for_each_sched_rt_entity(rt_se) \  argument
518 	for (; rt_se; rt_se = rt_se->parent)
520 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()  argument
522 	return rt_se->my_q;  in group_rt_rq()
525 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
526 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
532 	struct sched_rt_entity *rt_se;  in sched_rt_rq_enqueue()  local
536 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
539 		if (!rt_se)  in sched_rt_rq_enqueue()
541 		else if (!on_rt_rq(rt_se))  in sched_rt_rq_enqueue()
542 			enqueue_rt_entity(rt_se, 0);  in sched_rt_rq_enqueue()
551 	struct sched_rt_entity *rt_se;  in sched_rt_rq_dequeue()  local
554 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
556 	if (!rt_se) {  in sched_rt_rq_dequeue()
561 	else if (on_rt_rq(rt_se))  in sched_rt_rq_dequeue()
562 		dequeue_rt_entity(rt_se, 0);  in sched_rt_rq_dequeue()
570 static int rt_se_boosted(struct sched_rt_entity *rt_se)  in rt_se_boosted()  argument
572 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_boosted()
578 	p = rt_task_of(rt_se);  in rt_se_boosted()
622 #define for_each_sched_rt_entity(rt_se) \  argument
623 	for (; rt_se; rt_se = NULL)
625 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()  argument
938 static inline int rt_se_prio(struct sched_rt_entity *rt_se)  in rt_se_prio()  argument
941 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_prio()
947 	return rt_task_of(rt_se)->prio;  in rt_se_prio()
1000 	struct sched_rt_entity *rt_se = &curr->rt;  in update_curr_rt()  local
1024 	for_each_sched_rt_entity(rt_se) {  in update_curr_rt()
1025 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in update_curr_rt()
1168 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1170 	if (rt_se_boosted(rt_se))  in inc_rt_group()
1178 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_group()  argument
1180 	if (rt_se_boosted(rt_se))  in dec_rt_group()
1189 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1195 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}  in dec_rt_group()  argument
1200 unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)  in rt_se_nr_running()  argument
1202 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_nr_running()
1211 unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)  in rt_se_rr_nr_running()  argument
1213 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_rr_nr_running()
1219 	tsk = rt_task_of(rt_se);  in rt_se_rr_nr_running()
1225 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_tasks()  argument
1227 	int prio = rt_se_prio(rt_se);  in inc_rt_tasks()
1230 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);  in inc_rt_tasks()
1231 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);  in inc_rt_tasks()
1234 	inc_rt_migration(rt_se, rt_rq);  in inc_rt_tasks()
1235 	inc_rt_group(rt_se, rt_rq);  in inc_rt_tasks()
1239 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_tasks()  argument
1241 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));  in dec_rt_tasks()
1243 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);  in dec_rt_tasks()
1244 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);  in dec_rt_tasks()
1246 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));  in dec_rt_tasks()
1247 	dec_rt_migration(rt_se, rt_rq);  in dec_rt_tasks()
1248 	dec_rt_group(rt_se, rt_rq);  in dec_rt_tasks()
1264 static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)  in __delist_rt_entity()  argument
1266 	list_del_init(&rt_se->run_list);  in __delist_rt_entity()
1268 	if (list_empty(array->queue + rt_se_prio(rt_se)))  in __delist_rt_entity()
1269 		__clear_bit(rt_se_prio(rt_se), array->bitmap);  in __delist_rt_entity()
1271 	rt_se->on_list = 0;  in __delist_rt_entity()
1274 static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in __enqueue_rt_entity()  argument
1276 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __enqueue_rt_entity()
1278 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in __enqueue_rt_entity()
1279 	struct list_head *queue = array->queue + rt_se_prio(rt_se);  in __enqueue_rt_entity()
1288 		if (rt_se->on_list)  in __enqueue_rt_entity()
1289 			__delist_rt_entity(rt_se, array);  in __enqueue_rt_entity()
1294 		WARN_ON_ONCE(rt_se->on_list);  in __enqueue_rt_entity()
1296 			list_add(&rt_se->run_list, queue);  in __enqueue_rt_entity()
1298 			list_add_tail(&rt_se->run_list, queue);  in __enqueue_rt_entity()
1300 		__set_bit(rt_se_prio(rt_se), array->bitmap);  in __enqueue_rt_entity()
1301 		rt_se->on_list = 1;  in __enqueue_rt_entity()
1303 	rt_se->on_rq = 1;  in __enqueue_rt_entity()
1305 	inc_rt_tasks(rt_se, rt_rq);  in __enqueue_rt_entity()
1308 static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in __dequeue_rt_entity()  argument
1310 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __dequeue_rt_entity()
1314 		WARN_ON_ONCE(!rt_se->on_list);  in __dequeue_rt_entity()
1315 		__delist_rt_entity(rt_se, array);  in __dequeue_rt_entity()
1317 	rt_se->on_rq = 0;  in __dequeue_rt_entity()
1319 	dec_rt_tasks(rt_se, rt_rq);  in __dequeue_rt_entity()
1326 static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)  in dequeue_rt_stack()  argument
1330 	for_each_sched_rt_entity(rt_se) {  in dequeue_rt_stack()
1331 		rt_se->back = back;  in dequeue_rt_stack()
1332 		back = rt_se;  in dequeue_rt_stack()
1337 	for (rt_se = back; rt_se; rt_se = rt_se->back) {  in dequeue_rt_stack()
1338 		if (on_rt_rq(rt_se))  in dequeue_rt_stack()
1339 			__dequeue_rt_entity(rt_se, flags);  in dequeue_rt_stack()
1343 static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in enqueue_rt_entity()  argument
1345 	struct rq *rq = rq_of_rt_se(rt_se);  in enqueue_rt_entity()
1347 	dequeue_rt_stack(rt_se, flags);  in enqueue_rt_entity()
1348 	for_each_sched_rt_entity(rt_se)  in enqueue_rt_entity()
1349 		__enqueue_rt_entity(rt_se, flags);  in enqueue_rt_entity()
1353 static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)  in dequeue_rt_entity()  argument
1355 	struct rq *rq = rq_of_rt_se(rt_se);  in dequeue_rt_entity()
1357 	dequeue_rt_stack(rt_se, flags);  in dequeue_rt_entity()
1359 	for_each_sched_rt_entity(rt_se) {  in dequeue_rt_entity()
1360 		struct rt_rq *rt_rq = group_rt_rq(rt_se);  in dequeue_rt_entity()
1363 			__enqueue_rt_entity(rt_se, flags);  in dequeue_rt_entity()
1374 	struct sched_rt_entity *rt_se = &p->rt;  in enqueue_task_rt()  local
1377 		rt_se->timeout = 0;  in enqueue_task_rt()
1379 	enqueue_rt_entity(rt_se, flags);  in enqueue_task_rt()
1387 	struct sched_rt_entity *rt_se = &p->rt;  in dequeue_task_rt()  local
1390 	dequeue_rt_entity(rt_se, flags);  in dequeue_task_rt()
1400 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)  in requeue_rt_entity()  argument
1402 	if (on_rt_rq(rt_se)) {  in requeue_rt_entity()
1404 		struct list_head *queue = array->queue + rt_se_prio(rt_se);  in requeue_rt_entity()
1407 			list_move(&rt_se->run_list, queue);  in requeue_rt_entity()
1409 			list_move_tail(&rt_se->run_list, queue);  in requeue_rt_entity()
1415 	struct sched_rt_entity *rt_se = &p->rt;  in requeue_task_rt()  local
1418 	for_each_sched_rt_entity(rt_se) {  in requeue_task_rt()
1419 		rt_rq = rt_rq_of_se(rt_se);  in requeue_task_rt()
1420 		requeue_rt_entity(rt_rq, rt_se, head);  in requeue_task_rt()
1617 	struct sched_rt_entity *rt_se;  in _pick_next_task_rt()  local
1621 		rt_se = pick_next_rt_entity(rq, rt_rq);  in _pick_next_task_rt()
1622 		BUG_ON(!rt_se);  in _pick_next_task_rt()
1623 		rt_rq = group_rt_rq(rt_se);  in _pick_next_task_rt()
1626 	return rt_task_of(rt_se);  in _pick_next_task_rt()
2446 	struct sched_rt_entity *rt_se = &p->rt;  in task_tick_rt()  local
2469 	for_each_sched_rt_entity(rt_se) {  in task_tick_rt()
2470 		if (rt_se->run_list.prev != rt_se->run_list.next) {  in task_tick_rt()