283 	if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))  in cfs_rq_tg_path()
284 		autogroup_path(cfs_rq->tg, path, len);  in cfs_rq_tg_path()
285 	else if (cfs_rq && cfs_rq->tg->css.cgroup)  in cfs_rq_tg_path()
286 		cgroup_path(cfs_rq->tg->css.cgroup, path, len);  in cfs_rq_tg_path()
310 	if (cfs_rq->tg->parent &&  in list_add_leaf_cfs_rq()
311 	    cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {  in list_add_leaf_cfs_rq()
319 			&(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));  in list_add_leaf_cfs_rq()
329 	if (!cfs_rq->tg->parent) {  in list_add_leaf_cfs_rq()
2997 	struct task_group *tg = cfs_rq->tg;  in calc_group_shares()  local
2999 	tg_shares = READ_ONCE(tg->shares);  in calc_group_shares()
3003 	tg_weight = atomic_long_read(&tg->load_avg);  in calc_group_shares()
3091 	runnable = shares = READ_ONCE(gcfs_rq->tg->shares);  in update_cfs_group()
3156 	if (cfs_rq->tg == &root_task_group)  in update_tg_load_avg()
3160 		atomic_long_add(delta, &cfs_rq->tg->load_avg);  in update_tg_load_avg()
4368 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)  in tg_cfs_bandwidth()  argument
4370 	return &tg->cfs_bandwidth;  in tg_cfs_bandwidth()
4376 	struct task_group *tg = cfs_rq->tg;  in assign_cfs_rq_runtime()  local
4377 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);  in assign_cfs_rq_runtime()
4445 static inline int throttled_lb_pair(struct task_group *tg,  in throttled_lb_pair()  argument
4450 	src_cfs_rq = tg->cfs_rq[src_cpu];  in throttled_lb_pair()
4451 	dest_cfs_rq = tg->cfs_rq[dest_cpu];  in throttled_lb_pair()
4457 static int tg_unthrottle_up(struct task_group *tg, void *data)  in tg_unthrottle_up()  argument
4460 	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];  in tg_unthrottle_up()
4475 static int tg_throttle_down(struct task_group *tg, void *data)  in tg_throttle_down()  argument
4478 	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];  in tg_throttle_down()
4493 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()
4498 	se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];  in throttle_cfs_rq()
4502 	walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);  in throttle_cfs_rq()
4553 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()
4558 	se = cfs_rq->tg->se[cpu_of(rq)];  in unthrottle_cfs_rq()
4570 	walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);  in unthrottle_cfs_rq()
4765 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()
4862 static void sync_throttle(struct task_group *tg, int cpu)  in sync_throttle()  argument
4869 	if (!tg->parent)  in sync_throttle()
4872 	cfs_rq = tg->cfs_rq[cpu];  in sync_throttle()
4873 	pcfs_rq = tg->parent->cfs_rq[cpu];  in sync_throttle()
5019 	struct task_group *tg;  in update_runtime_enabled()  local
5024 	list_for_each_entry_rcu(tg, &task_groups, list) {  in update_runtime_enabled()
5025 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()
5026 		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];  in update_runtime_enabled()
5038 	struct task_group *tg;  in unthrottle_offline_cfs_rqs()  local
5043 	list_for_each_entry_rcu(tg, &task_groups, list) {  in unthrottle_offline_cfs_rqs()
5044 		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];  in unthrottle_offline_cfs_rqs()
5076 static inline void sync_throttle(struct task_group *tg, int cpu) {}  in sync_throttle()  argument
5089 static inline int throttled_lb_pair(struct task_group *tg,  in throttled_lb_pair()  argument
5101 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)  in tg_cfs_bandwidth()  argument
7574 		se = cfs_rq->tg->se[cpu];  in update_blocked_averages()
7602 	struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];  in update_cfs_rq_h_load()
10223 void free_fair_sched_group(struct task_group *tg)  in free_fair_sched_group()  argument
10227 	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));  in free_fair_sched_group()
10230 		if (tg->cfs_rq)  in free_fair_sched_group()
10231 			kfree(tg->cfs_rq[i]);  in free_fair_sched_group()
10232 		if (tg->se)  in free_fair_sched_group()
10233 			kfree(tg->se[i]);  in free_fair_sched_group()
10236 	kfree(tg->cfs_rq);  in free_fair_sched_group()
10237 	kfree(tg->se);  in free_fair_sched_group()
10240 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)  in alloc_fair_sched_group()  argument
10246 	tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);  in alloc_fair_sched_group()
10247 	if (!tg->cfs_rq)  in alloc_fair_sched_group()
10249 	tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);  in alloc_fair_sched_group()
10250 	if (!tg->se)  in alloc_fair_sched_group()
10253 	tg->shares = NICE_0_LOAD;  in alloc_fair_sched_group()
10255 	init_cfs_bandwidth(tg_cfs_bandwidth(tg));  in alloc_fair_sched_group()
10269 		init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);  in alloc_fair_sched_group()
10281 void online_fair_sched_group(struct task_group *tg)  in online_fair_sched_group()  argument
10290 		se = tg->se[i];  in online_fair_sched_group()
10294 		sync_throttle(tg, i);  in online_fair_sched_group()
10299 void unregister_fair_sched_group(struct task_group *tg)  in unregister_fair_sched_group()  argument
10306 		if (tg->se[cpu])  in unregister_fair_sched_group()
10307 			remove_entity_load_avg(tg->se[cpu]);  in unregister_fair_sched_group()
10313 		if (!tg->cfs_rq[cpu]->on_list)  in unregister_fair_sched_group()
10319 		list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);  in unregister_fair_sched_group()
10324 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,  in init_tg_cfs_entry()  argument
10330 	cfs_rq->tg = tg;  in init_tg_cfs_entry()
10334 	tg->cfs_rq[cpu] = cfs_rq;  in init_tg_cfs_entry()
10335 	tg->se[cpu] = se;  in init_tg_cfs_entry()
10357 int sched_group_set_shares(struct task_group *tg, unsigned long shares)  in sched_group_set_shares()  argument
10364 	if (!tg->se[0])  in sched_group_set_shares()
10370 	if (tg->shares == shares)  in sched_group_set_shares()
10373 	tg->shares = shares;  in sched_group_set_shares()
10376 		struct sched_entity *se = tg->se[i];  in sched_group_set_shares()
10395 void free_fair_sched_group(struct task_group *tg) { }  in free_fair_sched_group()  argument
10397 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)  in alloc_fair_sched_group()  argument
10402 void online_fair_sched_group(struct task_group *tg) { }  in online_fair_sched_group()  argument
10404 void unregister_fair_sched_group(struct task_group *tg) { }  in unregister_fair_sched_group()  argument