8810 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)  in group_has_capacity()  argument
8812 	if (sgs->sum_nr_running < sgs->group_weight)  in group_has_capacity()
8815 	if ((sgs->group_capacity * imbalance_pct) <  in group_has_capacity()
8816 			(sgs->group_runnable * 100))  in group_has_capacity()
8819 	if ((sgs->group_capacity * 100) >  in group_has_capacity()
8820 			(sgs->group_util * imbalance_pct))  in group_has_capacity()
8835 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)  in group_is_overloaded()  argument
8837 	if (sgs->sum_nr_running <= sgs->group_weight)  in group_is_overloaded()
8840 	if ((sgs->group_capacity * 100) <  in group_is_overloaded()
8841 			(sgs->group_util * imbalance_pct))  in group_is_overloaded()
8844 	if ((sgs->group_capacity * imbalance_pct) <  in group_is_overloaded()
8845 			(sgs->group_runnable * 100))  in group_is_overloaded()
8854 			  struct sg_lb_stats *sgs)  in group_classify()  argument
8856 	if (group_is_overloaded(imbalance_pct, sgs))  in group_classify()
8862 	if (sgs->group_asym_packing)  in group_classify()
8865 	if (sgs->group_misfit_task_load)  in group_classify()
8868 	if (!group_has_capacity(imbalance_pct, sgs))  in group_classify()
8899 				    struct sg_lb_stats *sgs,  in asym_smt_can_pull_tasks()  argument
8909 	sg_busy_cpus = sgs->group_weight - sgs->idle_cpus;  in asym_smt_can_pull_tasks()
8958 sched_asym(struct lb_env *env, struct sd_lb_stats *sds,  struct sg_lb_stats *sgs,  in sched_asym()  argument
8964 		return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group);  in sched_asym()
8993 				      struct sg_lb_stats *sgs,  in update_sg_lb_stats()  argument
8998 	memset(sgs, 0, sizeof(*sgs));  in update_sg_lb_stats()
9006 		sgs->group_load += load;  in update_sg_lb_stats()
9007 		sgs->group_util += cpu_util_cfs(i);  in update_sg_lb_stats()
9008 		sgs->group_runnable += cpu_runnable(rq);  in update_sg_lb_stats()
9009 		sgs->sum_h_nr_running += rq->cfs.h_nr_running;  in update_sg_lb_stats()
9012 		sgs->sum_nr_running += nr_running;  in update_sg_lb_stats()
9021 		sgs->nr_numa_running += rq->nr_numa_running;  in update_sg_lb_stats()
9022 		sgs->nr_preferred_running += rq->nr_preferred_running;  in update_sg_lb_stats()
9028 			sgs->idle_cpus++;  in update_sg_lb_stats()
9038 			if (sgs->group_misfit_task_load < rq->misfit_task_load) {  in update_sg_lb_stats()
9039 				sgs->group_misfit_task_load = rq->misfit_task_load;  in update_sg_lb_stats()
9045 			if (sgs->group_misfit_task_load < load)  in update_sg_lb_stats()
9046 				sgs->group_misfit_task_load = load;  in update_sg_lb_stats()
9050 	sgs->group_capacity = group->sgc->capacity;  in update_sg_lb_stats()
9052 	sgs->group_weight = group->group_weight;  in update_sg_lb_stats()
9056 	    env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running &&  in update_sg_lb_stats()
9057 	    sched_asym(env, sds, sgs, group)) {  in update_sg_lb_stats()
9058 		sgs->group_asym_packing = 1;  in update_sg_lb_stats()
9061 	sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);  in update_sg_lb_stats()
9064 	if (sgs->group_type == group_overloaded)  in update_sg_lb_stats()
9065 		sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /  in update_sg_lb_stats()
9066 				sgs->group_capacity;  in update_sg_lb_stats()
9085 				   struct sg_lb_stats *sgs)  in update_sd_pick_busiest()  argument
9090 	if (!sgs->sum_h_nr_running)  in update_sd_pick_busiest()
9100 	    (sgs->group_type == group_misfit_task) &&  in update_sd_pick_busiest()
9105 	if (sgs->group_type > busiest->group_type)  in update_sd_pick_busiest()
9108 	if (sgs->group_type < busiest->group_type)  in update_sd_pick_busiest()
9116 	switch (sgs->group_type) {  in update_sd_pick_busiest()
9119 		if (sgs->avg_load <= busiest->avg_load)  in update_sd_pick_busiest()
9141 		if (sgs->group_misfit_task_load < busiest->group_misfit_task_load)  in update_sd_pick_busiest()
9156 		if (sgs->avg_load <= busiest->avg_load)  in update_sd_pick_busiest()
9168 		if (sgs->idle_cpus > busiest->idle_cpus)  in update_sd_pick_busiest()
9170 		else if ((sgs->idle_cpus == busiest->idle_cpus) &&  in update_sd_pick_busiest()
9171 			 (sgs->sum_nr_running <= busiest->sum_nr_running))  in update_sd_pick_busiest()
9184 	    (sgs->group_type <= group_fully_busy) &&  in update_sd_pick_busiest()
9192 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)  in fbq_classify_group()  argument
9194 	if (sgs->sum_h_nr_running > sgs->nr_numa_running)  in fbq_classify_group()
9196 	if (sgs->sum_h_nr_running > sgs->nr_preferred_running)  in fbq_classify_group()
9210 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)  in fbq_classify_group()  argument
9277 					  struct sg_lb_stats *sgs,  in update_sg_wakeup_stats()  argument
9282 	memset(sgs, 0, sizeof(*sgs));  in update_sg_wakeup_stats()
9288 		sgs->group_load += cpu_load_without(rq, p);  in update_sg_wakeup_stats()
9289 		sgs->group_util += cpu_util_without(i, p);  in update_sg_wakeup_stats()
9290 		sgs->group_runnable += cpu_runnable_without(rq, p);  in update_sg_wakeup_stats()
9292 		sgs->sum_h_nr_running += rq->cfs.h_nr_running - local;  in update_sg_wakeup_stats()
9295 		sgs->sum_nr_running += nr_running;  in update_sg_wakeup_stats()
9301 			sgs->idle_cpus++;  in update_sg_wakeup_stats()
9308 		sgs->group_misfit_task_load = 1;  in update_sg_wakeup_stats()
9311 	sgs->group_capacity = group->sgc->capacity;  in update_sg_wakeup_stats()
9313 	sgs->group_weight = group->group_weight;  in update_sg_wakeup_stats()
9315 	sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);  in update_sg_wakeup_stats()
9321 	if (sgs->group_type == group_fully_busy ||  in update_sg_wakeup_stats()
9322 		sgs->group_type == group_overloaded)  in update_sg_wakeup_stats()
9323 		sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /  in update_sg_wakeup_stats()
9324 				sgs->group_capacity;  in update_sg_wakeup_stats()
9330 			       struct sg_lb_stats *sgs)  in update_pick_idlest()  argument
9332 	if (sgs->group_type < idlest_sgs->group_type)  in update_pick_idlest()
9335 	if (sgs->group_type > idlest_sgs->group_type)  in update_pick_idlest()
9343 	switch (sgs->group_type) {  in update_pick_idlest()
9347 		if (idlest_sgs->avg_load <= sgs->avg_load)  in update_pick_idlest()
9364 		if (idlest_sgs->idle_cpus > sgs->idle_cpus)  in update_pick_idlest()
9368 		if (idlest_sgs->idle_cpus == sgs->idle_cpus &&  in update_pick_idlest()
9369 			idlest_sgs->group_util <= sgs->group_util)  in update_pick_idlest()
9389 	struct sg_lb_stats *sgs;  in find_idlest_group()  local
9412 			sgs = &local_sgs;  in find_idlest_group()
9415 			sgs = &tmp_sgs;  in find_idlest_group()
9418 		update_sg_wakeup_stats(sd, group, sgs, p);  in find_idlest_group()
9420 		if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {  in find_idlest_group()
9422 			idlest_sgs = *sgs;  in find_idlest_group()
9636 		struct sg_lb_stats *sgs = &tmp_sgs;  in update_sd_lb_stats()  local
9642 			sgs = local;  in update_sd_lb_stats()
9649 		update_sg_lb_stats(env, sds, sg, sgs, &sg_status);  in update_sd_lb_stats()
9655 		if (update_sd_pick_busiest(env, sds, sg, sgs)) {  in update_sd_lb_stats()
9657 			sds->busiest_stat = *sgs;  in update_sd_lb_stats()
9662 		sds->total_load += sgs->group_load;  in update_sd_lb_stats()
9663 		sds->total_capacity += sgs->group_capacity;  in update_sd_lb_stats()
9665 		sum_util += sgs->group_util;  in update_sd_lb_stats()