335 static void free_sched_groups(struct sched_group *sg, int free_sgc)  in free_sched_groups()  argument
339 	if (!sg)  in free_sched_groups()
342 	first = sg;  in free_sched_groups()
344 		tmp = sg->next;  in free_sched_groups()
346 		if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))  in free_sched_groups()
347 			kfree(sg->sgc);  in free_sched_groups()
349 		if (atomic_dec_and_test(&sg->ref))  in free_sched_groups()
350 			kfree(sg);  in free_sched_groups()
351 		sg = tmp;  in free_sched_groups()
352 	} while (sg != first);  in free_sched_groups()
500 int group_balance_cpu(struct sched_group *sg)  in group_balance_cpu()  argument
502 	return cpumask_first(group_balance_mask(sg));  in group_balance_cpu()
612 build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask)  in build_balance_mask()  argument
614 	const struct cpumask *sg_span = sched_group_span(sg);  in build_balance_mask()
651 	struct sched_group *sg;  in build_group_from_child_sched_domain()  local
654 	sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),  in build_group_from_child_sched_domain()
657 	if (!sg)  in build_group_from_child_sched_domain()
660 	sg_span = sched_group_span(sg);  in build_group_from_child_sched_domain()
666 	atomic_inc(&sg->ref);  in build_group_from_child_sched_domain()
667 	return sg;  in build_group_from_child_sched_domain()
671 				     struct sched_group *sg)  in init_overlap_sched_group()  argument
678 	build_balance_mask(sd, sg, mask);  in init_overlap_sched_group()
679 	cpu = cpumask_first_and(sched_group_span(sg), mask);  in init_overlap_sched_group()
681 	sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);  in init_overlap_sched_group()
682 	if (atomic_inc_return(&sg->sgc->ref) == 1)  in init_overlap_sched_group()
683 		cpumask_copy(group_balance_mask(sg), mask);  in init_overlap_sched_group()
685 		WARN_ON_ONCE(!cpumask_equal(group_balance_mask(sg), mask));  in init_overlap_sched_group()
692 	sg_span = sched_group_span(sg);  in init_overlap_sched_group()
693 	sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);  in init_overlap_sched_group()
694 	sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;  in init_overlap_sched_group()
700 	struct sched_group *first = NULL, *last = NULL, *sg;  in build_overlap_sched_groups()  local
730 		sg = build_group_from_child_sched_domain(sibling, cpu);  in build_overlap_sched_groups()
731 		if (!sg)  in build_overlap_sched_groups()
734 		sg_span = sched_group_span(sg);  in build_overlap_sched_groups()
737 		init_overlap_sched_group(sd, sg);  in build_overlap_sched_groups()
740 			first = sg;  in build_overlap_sched_groups()
742 			last->next = sg;  in build_overlap_sched_groups()
743 		last = sg;  in build_overlap_sched_groups()
832 	struct sched_group *sg;  in get_group()  local
837 	sg = *per_cpu_ptr(sdd->sg, cpu);  in get_group()
838 	sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);  in get_group()
841 	atomic_inc(&sg->ref);  in get_group()
842 	atomic_inc(&sg->sgc->ref);  in get_group()
845 		cpumask_copy(sched_group_span(sg), sched_domain_span(child));  in get_group()
846 		cpumask_copy(group_balance_mask(sg), sched_group_span(sg));  in get_group()
848 		cpumask_set_cpu(cpu, sched_group_span(sg));  in get_group()
849 		cpumask_set_cpu(cpu, group_balance_mask(sg));  in get_group()
852 	sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg));  in get_group()
853 	sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;  in get_group()
855 	return sg;  in get_group()
880 		struct sched_group *sg;  in build_sched_groups()  local
885 		sg = get_group(i, sdd);  in build_sched_groups()
887 		cpumask_or(covered, covered, sched_group_span(sg));  in build_sched_groups()
890 			first = sg;  in build_sched_groups()
892 			last->next = sg;  in build_sched_groups()
893 		last = sg;  in build_sched_groups()
913 	struct sched_group *sg = sd->groups;  in init_sched_groups_capacity()  local
915 	WARN_ON(!sg);  in init_sched_groups_capacity()
920 		sg->group_weight = cpumask_weight(sched_group_span(sg));  in init_sched_groups_capacity()
925 		for_each_cpu(cpu, sched_group_span(sg)) {  in init_sched_groups_capacity()
931 		sg->asym_prefer_cpu = max_cpu;  in init_sched_groups_capacity()
934 		sg = sg->next;  in init_sched_groups_capacity()
935 	} while (sg != sd->groups);  in init_sched_groups_capacity()
937 	if (cpu != group_balance_cpu(sg))  in init_sched_groups_capacity()
1035 	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))  in claim_allocations()
1036 		*per_cpu_ptr(sdd->sg, cpu) = NULL;  in claim_allocations()
1516 		sdd->sg = alloc_percpu(struct sched_group *);  in __sdt_alloc()
1517 		if (!sdd->sg)  in __sdt_alloc()
1527 			struct sched_group *sg;  in __sdt_alloc()  local
1544 			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),  in __sdt_alloc()
1546 			if (!sg)  in __sdt_alloc()
1549 			sg->next = sg;  in __sdt_alloc()
1551 			*per_cpu_ptr(sdd->sg, j) = sg;  in __sdt_alloc()
1589 			if (sdd->sg)  in __sdt_free()
1590 				kfree(*per_cpu_ptr(sdd->sg, j));  in __sdt_free()
1598 		free_percpu(sdd->sg);  in __sdt_free()
1599 		sdd->sg = NULL;  in __sdt_free()