78 void init_rt_rq(struct rt_rq *rt_rq)  in init_rt_rq()  argument
83 	array = &rt_rq->active;  in init_rt_rq()
92 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in init_rt_rq()
93 	rt_rq->highest_prio.next = MAX_RT_PRIO-1;  in init_rt_rq()
94 	rt_rq->rt_nr_migratory = 0;  in init_rt_rq()
95 	rt_rq->overloaded = 0;  in init_rt_rq()
96 	plist_head_init(&rt_rq->pushable_tasks);  in init_rt_rq()
99 	rt_rq->rt_queued = 0;  in init_rt_rq()
101 	rt_rq->rt_time = 0;  in init_rt_rq()
102 	rt_rq->rt_throttled = 0;  in init_rt_rq()
103 	rt_rq->rt_runtime = 0;  in init_rt_rq()
104 	raw_spin_lock_init(&rt_rq->rt_runtime_lock);  in init_rt_rq()
123 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
125 	return rt_rq->rq;  in rq_of_rt_rq()
128 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
130 	return rt_se->rt_rq;  in rt_rq_of_se()
135 	struct rt_rq *rt_rq = rt_se->rt_rq;  in rq_of_rt_se()  local
137 	return rt_rq->rq;  in rq_of_rt_se()
148 		if (tg->rt_rq)  in free_rt_sched_group()
149 			kfree(tg->rt_rq[i]);  in free_rt_sched_group()
154 	kfree(tg->rt_rq);  in free_rt_sched_group()
158 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,  in init_tg_rt_entry()  argument
164 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in init_tg_rt_entry()
165 	rt_rq->rt_nr_boosted = 0;  in init_tg_rt_entry()
166 	rt_rq->rq = rq;  in init_tg_rt_entry()
167 	rt_rq->tg = tg;  in init_tg_rt_entry()
169 	tg->rt_rq[cpu] = rt_rq;  in init_tg_rt_entry()
176 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
178 		rt_se->rt_rq = parent->my_q;  in init_tg_rt_entry()
180 	rt_se->my_q = rt_rq;  in init_tg_rt_entry()
187 	struct rt_rq *rt_rq;  in alloc_rt_sched_group()  local
191 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);  in alloc_rt_sched_group()
192 	if (!tg->rt_rq)  in alloc_rt_sched_group()
202 		rt_rq = kzalloc_node(sizeof(struct rt_rq),  in alloc_rt_sched_group()
204 		if (!rt_rq)  in alloc_rt_sched_group()
212 		init_rt_rq(rt_rq);  in alloc_rt_sched_group()
213 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;  in alloc_rt_sched_group()
214 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
220 	kfree(rt_rq);  in alloc_rt_sched_group()
234 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
236 	return container_of(rt_rq, struct rq, rt);  in rq_of_rt_rq()
246 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
305 static void update_rt_migration(struct rt_rq *rt_rq)  in update_rt_migration()  argument
307 	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {  in update_rt_migration()
308 		if (!rt_rq->overloaded) {  in update_rt_migration()
309 			rt_set_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
310 			rt_rq->overloaded = 1;  in update_rt_migration()
312 	} else if (rt_rq->overloaded) {  in update_rt_migration()
313 		rt_clear_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
314 		rt_rq->overloaded = 0;  in update_rt_migration()
318 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
326 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in inc_rt_migration()
328 	rt_rq->rt_nr_total++;  in inc_rt_migration()
330 		rt_rq->rt_nr_migratory++;  in inc_rt_migration()
332 	update_rt_migration(rt_rq);  in inc_rt_migration()
335 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
343 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in dec_rt_migration()
345 	rt_rq->rt_nr_total--;  in dec_rt_migration()
347 		rt_rq->rt_nr_migratory--;  in dec_rt_migration()
349 	update_rt_migration(rt_rq);  in dec_rt_migration()
412 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
417 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
435 static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
436 static void dequeue_top_rt_rq(struct rt_rq *rt_rq);
484 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
486 	if (!rt_rq->tg)  in sched_rt_runtime()
489 	return rt_rq->rt_runtime;  in sched_rt_runtime()
492 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
494 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);  in sched_rt_period()
512 #define for_each_rt_rq(rt_rq, iter, rq)					\  argument
515 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
520 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
528 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
530 	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;  in sched_rt_rq_enqueue()
531 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
536 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
538 	if (rt_rq->rt_nr_running) {  in sched_rt_rq_enqueue()
540 			enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
544 		if (rt_rq->highest_prio.curr < curr->prio)  in sched_rt_rq_enqueue()
549 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
552 	int cpu = cpu_of(rq_of_rt_rq(rt_rq));  in sched_rt_rq_dequeue()
554 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
557 		dequeue_top_rt_rq(rt_rq);  in sched_rt_rq_dequeue()
559 		cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);  in sched_rt_rq_dequeue()
565 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
567 	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;  in rt_rq_throttled()
572 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_boosted()  local
575 	if (rt_rq)  in rt_se_boosted()
576 		return !!rt_rq->rt_nr_boosted;  in rt_se_boosted()
595 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
597 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];  in sched_rt_period_rt_rq()
600 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
602 	return &rt_rq->tg->rt_bandwidth;  in sched_rt_bandwidth()
607 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
609 	return rt_rq->rt_runtime;  in sched_rt_runtime()
612 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
617 typedef struct rt_rq *rt_rq_iter_t;
619 #define for_each_rt_rq(rt_rq, iter, rq) \  argument
620 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
625 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
630 static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
632 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
634 	if (!rt_rq->rt_nr_running)  in sched_rt_rq_enqueue()
637 	enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
641 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
643 	dequeue_top_rt_rq(rt_rq);  in sched_rt_rq_dequeue()
646 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
648 	return rt_rq->rt_throttled;  in rt_rq_throttled()
657 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
662 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
669 bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)  in sched_rt_bandwidth_account()  argument
671 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_bandwidth_account()
674 		rt_rq->rt_time < rt_b->rt_runtime);  in sched_rt_bandwidth_account()
681 static void do_balance_runtime(struct rt_rq *rt_rq)  in do_balance_runtime()  argument
683 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in do_balance_runtime()
684 	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;  in do_balance_runtime()
693 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in do_balance_runtime()
696 		if (iter == rt_rq)  in do_balance_runtime()
715 			if (rt_rq->rt_runtime + diff > rt_period)  in do_balance_runtime()
716 				diff = rt_period - rt_rq->rt_runtime;  in do_balance_runtime()
718 			rt_rq->rt_runtime += diff;  in do_balance_runtime()
719 			if (rt_rq->rt_runtime == rt_period) {  in do_balance_runtime()
737 	struct rt_rq *rt_rq;  in __disable_runtime()  local
742 	for_each_rt_rq(rt_rq, iter, rq) {  in __disable_runtime()
743 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __disable_runtime()
748 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
754 		if (rt_rq->rt_runtime == RUNTIME_INF ||  in __disable_runtime()
755 				rt_rq->rt_runtime == rt_b->rt_runtime)  in __disable_runtime()
757 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
764 		want = rt_b->rt_runtime - rt_rq->rt_runtime;  in __disable_runtime()
770 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in __disable_runtime()
776 			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)  in __disable_runtime()
794 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
805 		rt_rq->rt_runtime = RUNTIME_INF;  in __disable_runtime()
806 		rt_rq->rt_throttled = 0;  in __disable_runtime()
807 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
811 		sched_rt_rq_enqueue(rt_rq);  in __disable_runtime()
818 	struct rt_rq *rt_rq;  in __enable_runtime()  local
826 	for_each_rt_rq(rt_rq, iter, rq) {  in __enable_runtime()
827 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __enable_runtime()
830 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
831 		rt_rq->rt_runtime = rt_b->rt_runtime;  in __enable_runtime()
832 		rt_rq->rt_time = 0;  in __enable_runtime()
833 		rt_rq->rt_throttled = 0;  in __enable_runtime()
834 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
839 static void balance_runtime(struct rt_rq *rt_rq)  in balance_runtime()  argument
844 	if (rt_rq->rt_time > rt_rq->rt_runtime) {  in balance_runtime()
845 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in balance_runtime()
846 		do_balance_runtime(rt_rq);  in balance_runtime()
847 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in balance_runtime()
851 static inline void balance_runtime(struct rt_rq *rt_rq) {}  in balance_runtime()  argument
875 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);  in do_sched_rt_period_timer()  local
876 		struct rq *rq = rq_of_rt_rq(rt_rq);  in do_sched_rt_period_timer()
883 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
884 		if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)  in do_sched_rt_period_timer()
885 			rt_rq->rt_runtime = rt_b->rt_runtime;  in do_sched_rt_period_timer()
886 		skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;  in do_sched_rt_period_timer()
887 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
894 		if (rt_rq->rt_time) {  in do_sched_rt_period_timer()
897 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
898 			if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
899 				balance_runtime(rt_rq);  in do_sched_rt_period_timer()
900 			runtime = rt_rq->rt_runtime;  in do_sched_rt_period_timer()
901 			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);  in do_sched_rt_period_timer()
902 			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {  in do_sched_rt_period_timer()
903 				rt_rq->rt_throttled = 0;  in do_sched_rt_period_timer()
913 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)  in do_sched_rt_period_timer()
916 			if (rt_rq->rt_time || rt_rq->rt_nr_running)  in do_sched_rt_period_timer()
918 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
919 		} else if (rt_rq->rt_nr_running) {  in do_sched_rt_period_timer()
921 			if (!rt_rq_throttled(rt_rq))  in do_sched_rt_period_timer()
924 		if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
928 			sched_rt_rq_enqueue(rt_rq);  in do_sched_rt_period_timer()
941 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_prio()  local
943 	if (rt_rq)  in rt_se_prio()
944 		return rt_rq->highest_prio.curr;  in rt_se_prio()
950 static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)  in sched_rt_runtime_exceeded()  argument
952 	u64 runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
954 	if (rt_rq->rt_throttled)  in sched_rt_runtime_exceeded()
955 		return rt_rq_throttled(rt_rq);  in sched_rt_runtime_exceeded()
957 	if (runtime >= sched_rt_period(rt_rq))  in sched_rt_runtime_exceeded()
960 	balance_runtime(rt_rq);  in sched_rt_runtime_exceeded()
961 	runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
965 	if (rt_rq->rt_time > runtime) {  in sched_rt_runtime_exceeded()
966 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_runtime_exceeded()
973 			rt_rq->rt_throttled = 1;  in sched_rt_runtime_exceeded()
981 			rt_rq->rt_time = 0;  in sched_rt_runtime_exceeded()
984 		if (rt_rq_throttled(rt_rq)) {  in sched_rt_runtime_exceeded()
985 			sched_rt_rq_dequeue(rt_rq);  in sched_rt_runtime_exceeded()
1025 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in update_curr_rt()  local
1027 		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {  in update_curr_rt()
1028 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
1029 			rt_rq->rt_time += delta_exec;  in update_curr_rt()
1030 			if (sched_rt_runtime_exceeded(rt_rq))  in update_curr_rt()
1032 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
1038 dequeue_top_rt_rq(struct rt_rq *rt_rq)  in dequeue_top_rt_rq()  argument
1040 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dequeue_top_rt_rq()
1042 	BUG_ON(&rq->rt != rt_rq);  in dequeue_top_rt_rq()
1044 	if (!rt_rq->rt_queued)  in dequeue_top_rt_rq()
1049 	sub_nr_running(rq, rt_rq->rt_nr_running);  in dequeue_top_rt_rq()
1050 	rt_rq->rt_queued = 0;  in dequeue_top_rt_rq()
1055 enqueue_top_rt_rq(struct rt_rq *rt_rq)  in enqueue_top_rt_rq()  argument
1057 	struct rq *rq = rq_of_rt_rq(rt_rq);  in enqueue_top_rt_rq()
1059 	BUG_ON(&rq->rt != rt_rq);  in enqueue_top_rt_rq()
1061 	if (rt_rq->rt_queued)  in enqueue_top_rt_rq()
1064 	if (rt_rq_throttled(rt_rq))  in enqueue_top_rt_rq()
1067 	if (rt_rq->rt_nr_running) {  in enqueue_top_rt_rq()
1068 		add_nr_running(rq, rt_rq->rt_nr_running);  in enqueue_top_rt_rq()
1069 		rt_rq->rt_queued = 1;  in enqueue_top_rt_rq()
1079 inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in inc_rt_prio_smp()  argument
1081 	struct rq *rq = rq_of_rt_rq(rt_rq);  in inc_rt_prio_smp()
1087 	if (&rq->rt != rt_rq)  in inc_rt_prio_smp()
1095 dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in dec_rt_prio_smp()  argument
1097 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dec_rt_prio_smp()
1103 	if (&rq->rt != rt_rq)  in dec_rt_prio_smp()
1106 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)  in dec_rt_prio_smp()
1107 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);  in dec_rt_prio_smp()
1113 void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in inc_rt_prio_smp()  argument
1115 void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in dec_rt_prio_smp()  argument
1121 inc_rt_prio(struct rt_rq *rt_rq, int prio)  in inc_rt_prio()  argument
1123 	int prev_prio = rt_rq->highest_prio.curr;  in inc_rt_prio()
1126 		rt_rq->highest_prio.curr = prio;  in inc_rt_prio()
1128 	inc_rt_prio_smp(rt_rq, prio, prev_prio);  in inc_rt_prio()
1132 dec_rt_prio(struct rt_rq *rt_rq, int prio)  in dec_rt_prio()  argument
1134 	int prev_prio = rt_rq->highest_prio.curr;  in dec_rt_prio()
1136 	if (rt_rq->rt_nr_running) {  in dec_rt_prio()
1145 			struct rt_prio_array *array = &rt_rq->active;  in dec_rt_prio()
1147 			rt_rq->highest_prio.curr =  in dec_rt_prio()
1152 		rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in dec_rt_prio()
1155 	dec_rt_prio_smp(rt_rq, prio, prev_prio);  in dec_rt_prio()
1160 static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}  in inc_rt_prio()  argument
1161 static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}  in dec_rt_prio()  argument
1168 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1171 		rt_rq->rt_nr_boosted++;  in inc_rt_group()
1173 	if (rt_rq->tg)  in inc_rt_group()
1174 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);  in inc_rt_group()
1178 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_group()  argument
1181 		rt_rq->rt_nr_boosted--;  in dec_rt_group()
1183 	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);  in dec_rt_group()
1189 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1195 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}  in dec_rt_group()  argument
1202 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_nr_running()
1213 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_rr_nr_running()
1225 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_tasks()  argument
1230 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);  in inc_rt_tasks()
1231 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);  in inc_rt_tasks()
1233 	inc_rt_prio(rt_rq, prio);  in inc_rt_tasks()
1234 	inc_rt_migration(rt_se, rt_rq);  in inc_rt_tasks()
1235 	inc_rt_group(rt_se, rt_rq);  in inc_rt_tasks()
1239 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_tasks()  argument
1242 	WARN_ON(!rt_rq->rt_nr_running);  in dec_rt_tasks()
1243 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);  in dec_rt_tasks()
1244 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);  in dec_rt_tasks()
1246 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));  in dec_rt_tasks()
1247 	dec_rt_migration(rt_se, rt_rq);  in dec_rt_tasks()
1248 	dec_rt_group(rt_se, rt_rq);  in dec_rt_tasks()
1276 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __enqueue_rt_entity()  local
1277 	struct rt_prio_array *array = &rt_rq->active;  in __enqueue_rt_entity()
1278 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in __enqueue_rt_entity()
1305 	inc_rt_tasks(rt_se, rt_rq);  in __enqueue_rt_entity()
1310 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __dequeue_rt_entity()  local
1311 	struct rt_prio_array *array = &rt_rq->active;  in __dequeue_rt_entity()
1319 	dec_rt_tasks(rt_se, rt_rq);  in __dequeue_rt_entity()
1360 		struct rt_rq *rt_rq = group_rt_rq(rt_se);  in dequeue_rt_entity()  local
1362 		if (rt_rq && rt_rq->rt_nr_running)  in dequeue_rt_entity()
1400 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)  in requeue_rt_entity()  argument
1403 		struct rt_prio_array *array = &rt_rq->active;  in requeue_rt_entity()
1416 	struct rt_rq *rt_rq;  in requeue_task_rt()  local
1419 		rt_rq = rt_rq_of_se(rt_se);  in requeue_task_rt()
1420 		requeue_rt_entity(rt_rq, rt_se, head);  in requeue_task_rt()
1599 						   struct rt_rq *rt_rq)  in pick_next_rt_entity()  argument
1601 	struct rt_prio_array *array = &rt_rq->active;  in pick_next_rt_entity()
1618 	struct rt_rq *rt_rq  = &rq->rt;  in _pick_next_task_rt()  local
1621 		rt_se = pick_next_rt_entity(rq, rt_rq);  in _pick_next_task_rt()
1623 		rt_rq = group_rt_rq(rt_se);  in _pick_next_task_rt()
1624 	} while (rt_rq);  in _pick_next_task_rt()
2665 		struct rt_rq *rt_rq = tg->rt_rq[i];  in tg_set_rt_bandwidth()  local
2667 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in tg_set_rt_bandwidth()
2668 		rt_rq->rt_runtime = rt_runtime;  in tg_set_rt_bandwidth()
2669 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in tg_set_rt_bandwidth()
2754 		struct rt_rq *rt_rq = &cpu_rq(i)->rt;  in sched_rt_global_constraints()  local
2756 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in sched_rt_global_constraints()
2757 		rt_rq->rt_runtime = global_rt_runtime();  in sched_rt_global_constraints()
2758 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in sched_rt_global_constraints()
2851 	struct rt_rq *rt_rq;  in print_rt_stats()  local
2854 	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))  in print_rt_stats()
2855 		print_rt_rq(m, cpu, rt_rq);  in print_rt_stats()