1532 static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
2123 	struct pcpu_alloc_info *ai;  in pcpu_alloc_alloc_info()  local
2128 	base_size = ALIGN(struct_size(ai, groups, nr_groups),  in pcpu_alloc_alloc_info()
2129 			  __alignof__(ai->groups[0].cpu_map[0]));  in pcpu_alloc_alloc_info()
2130 	ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]);  in pcpu_alloc_alloc_info()
2135 	ai = ptr;  in pcpu_alloc_alloc_info()
2138 	ai->groups[0].cpu_map = ptr;  in pcpu_alloc_alloc_info()
2141 		ai->groups[0].cpu_map[unit] = NR_CPUS;  in pcpu_alloc_alloc_info()
2143 	ai->nr_groups = nr_groups;  in pcpu_alloc_alloc_info()
2144 	ai->__ai_size = PFN_ALIGN(ai_size);  in pcpu_alloc_alloc_info()
2146 	return ai;  in pcpu_alloc_alloc_info()
2155 void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai)  in pcpu_free_alloc_info()  argument
2157 	memblock_free_early(__pa(ai), ai->__ai_size);  in pcpu_free_alloc_info()
2168 				 const struct pcpu_alloc_info *ai)  in pcpu_dump_alloc_info()  argument
2176 	v = ai->nr_groups;  in pcpu_dump_alloc_info()
2185 	upa = ai->alloc_size / ai->unit_size;  in pcpu_dump_alloc_info()
2190 	       lvl, ai->static_size, ai->reserved_size, ai->dyn_size,  in pcpu_dump_alloc_info()
2191 	       ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size);  in pcpu_dump_alloc_info()
2193 	for (group = 0; group < ai->nr_groups; group++) {  in pcpu_dump_alloc_info()
2194 		const struct pcpu_group_info *gi = &ai->groups[group];  in pcpu_dump_alloc_info()
2271 void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,  in pcpu_setup_first_chunk()  argument
2274 	size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;  in pcpu_setup_first_chunk()
2292 		pcpu_dump_alloc_info(KERN_EMERG, ai);			\  in pcpu_setup_first_chunk()
2298 	PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);  in pcpu_setup_first_chunk()
2300 	PCPU_SETUP_BUG_ON(!ai->static_size);  in pcpu_setup_first_chunk()
2305 	PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);  in pcpu_setup_first_chunk()
2306 	PCPU_SETUP_BUG_ON(offset_in_page(ai->unit_size));  in pcpu_setup_first_chunk()
2307 	PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);  in pcpu_setup_first_chunk()
2308 	PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->unit_size, PCPU_BITMAP_BLOCK_SIZE));  in pcpu_setup_first_chunk()
2309 	PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE);  in pcpu_setup_first_chunk()
2310 	PCPU_SETUP_BUG_ON(!ai->dyn_size);  in pcpu_setup_first_chunk()
2311 	PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->reserved_size, PCPU_MIN_ALLOC_SIZE));  in pcpu_setup_first_chunk()
2314 	PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0);  in pcpu_setup_first_chunk()
2317 	alloc_size = ai->nr_groups * sizeof(group_offsets[0]);  in pcpu_setup_first_chunk()
2323 	alloc_size = ai->nr_groups * sizeof(group_sizes[0]);  in pcpu_setup_first_chunk()
2347 	for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {  in pcpu_setup_first_chunk()
2348 		const struct pcpu_group_info *gi = &ai->groups[group];  in pcpu_setup_first_chunk()
2351 		group_sizes[group] = gi->nr_units * ai->unit_size;  in pcpu_setup_first_chunk()
2363 			unit_off[cpu] = gi->base_offset + i * ai->unit_size;  in pcpu_setup_first_chunk()
2381 	pcpu_dump_alloc_info(KERN_DEBUG, ai);  in pcpu_setup_first_chunk()
2383 	pcpu_nr_groups = ai->nr_groups;  in pcpu_setup_first_chunk()
2390 	pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT;  in pcpu_setup_first_chunk()
2392 	pcpu_atom_size = ai->atom_size;  in pcpu_setup_first_chunk()
2396 	pcpu_stats_save_ai(ai);  in pcpu_setup_first_chunk()
2419 	static_size = ALIGN(ai->static_size, PCPU_MIN_ALLOC_SIZE);  in pcpu_setup_first_chunk()
2420 	dyn_size = ai->dyn_size - (static_size - ai->static_size);  in pcpu_setup_first_chunk()
2431 	map_size = ai->reserved_size ?: dyn_size;  in pcpu_setup_first_chunk()
2435 	if (ai->reserved_size) {  in pcpu_setup_first_chunk()
2439 			   ai->reserved_size;  in pcpu_setup_first_chunk()
2542 	struct pcpu_alloc_info *ai;  in pcpu_build_alloc_info()  local
2626 	ai = pcpu_alloc_alloc_info(nr_groups, nr_units);  in pcpu_build_alloc_info()
2627 	if (!ai)  in pcpu_build_alloc_info()
2629 	cpu_map = ai->groups[0].cpu_map;  in pcpu_build_alloc_info()
2632 		ai->groups[group].cpu_map = cpu_map;  in pcpu_build_alloc_info()
2636 	ai->static_size = static_size;  in pcpu_build_alloc_info()
2637 	ai->reserved_size = reserved_size;  in pcpu_build_alloc_info()
2638 	ai->dyn_size = dyn_size;  in pcpu_build_alloc_info()
2639 	ai->unit_size = alloc_size / upa;  in pcpu_build_alloc_info()
2640 	ai->atom_size = atom_size;  in pcpu_build_alloc_info()
2641 	ai->alloc_size = alloc_size;  in pcpu_build_alloc_info()
2644 		struct pcpu_group_info *gi = &ai->groups[group];  in pcpu_build_alloc_info()
2651 		gi->base_offset = unit * ai->unit_size;  in pcpu_build_alloc_info()
2661 	return ai;  in pcpu_build_alloc_info()
2706 	struct pcpu_alloc_info *ai;  in pcpu_embed_first_chunk()  local
2711 	ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size,  in pcpu_embed_first_chunk()
2713 	if (IS_ERR(ai))  in pcpu_embed_first_chunk()
2714 		return PTR_ERR(ai);  in pcpu_embed_first_chunk()
2716 	size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;  in pcpu_embed_first_chunk()
2717 	areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));  in pcpu_embed_first_chunk()
2727 	for (group = 0; group < ai->nr_groups; group++) {  in pcpu_embed_first_chunk()
2728 		struct pcpu_group_info *gi = &ai->groups[group];  in pcpu_embed_first_chunk()
2737 		ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size);  in pcpu_embed_first_chunk()
2751 	max_distance += ai->unit_size * ai->groups[highest_group].nr_units;  in pcpu_embed_first_chunk()
2769 	for (group = 0; group < ai->nr_groups; group++) {  in pcpu_embed_first_chunk()
2770 		struct pcpu_group_info *gi = &ai->groups[group];  in pcpu_embed_first_chunk()
2773 		for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {  in pcpu_embed_first_chunk()
2776 				free_fn(ptr, ai->unit_size);  in pcpu_embed_first_chunk()
2780 			memcpy(ptr, __per_cpu_load, ai->static_size);  in pcpu_embed_first_chunk()
2781 			free_fn(ptr + size_sum, ai->unit_size - size_sum);  in pcpu_embed_first_chunk()
2786 	for (group = 0; group < ai->nr_groups; group++) {  in pcpu_embed_first_chunk()
2787 		ai->groups[group].base_offset = areas[group] - base;  in pcpu_embed_first_chunk()
2791 		PFN_DOWN(size_sum), ai->static_size, ai->reserved_size,  in pcpu_embed_first_chunk()
2792 		ai->dyn_size, ai->unit_size);  in pcpu_embed_first_chunk()
2794 	pcpu_setup_first_chunk(ai, base);  in pcpu_embed_first_chunk()
2798 	for (group = 0; group < ai->nr_groups; group++)  in pcpu_embed_first_chunk()
2801 				ai->groups[group].nr_units * ai->unit_size);  in pcpu_embed_first_chunk()
2803 	pcpu_free_alloc_info(ai);  in pcpu_embed_first_chunk()
2833 	struct pcpu_alloc_info *ai;  in pcpu_page_first_chunk()  local
2844 	ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL);  in pcpu_page_first_chunk()
2845 	if (IS_ERR(ai))  in pcpu_page_first_chunk()
2846 		return PTR_ERR(ai);  in pcpu_page_first_chunk()
2847 	BUG_ON(ai->nr_groups != 1);  in pcpu_page_first_chunk()
2848 	upa = ai->alloc_size/ai->unit_size;  in pcpu_page_first_chunk()
2850 	if (WARN_ON(ai->groups[0].nr_units != nr_g0_units)) {  in pcpu_page_first_chunk()
2851 		pcpu_free_alloc_info(ai);  in pcpu_page_first_chunk()
2855 	unit_pages = ai->unit_size >> PAGE_SHIFT;  in pcpu_page_first_chunk()
2868 		unsigned int cpu = ai->groups[0].cpu_map[unit];  in pcpu_page_first_chunk()
2886 	vm.size = num_possible_cpus() * ai->unit_size;  in pcpu_page_first_chunk()
2891 			(unsigned long)vm.addr + unit * ai->unit_size;  in pcpu_page_first_chunk()
2911 		memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);  in pcpu_page_first_chunk()
2916 		unit_pages, psize_str, ai->static_size,  in pcpu_page_first_chunk()
2917 		ai->reserved_size, ai->dyn_size);  in pcpu_page_first_chunk()
2919 	pcpu_setup_first_chunk(ai, vm.addr);  in pcpu_page_first_chunk()
2928 	pcpu_free_alloc_info(ai);  in pcpu_page_first_chunk()
2996 	struct pcpu_alloc_info *ai;  in setup_per_cpu_areas()  local
2999 	ai = pcpu_alloc_alloc_info(1, 1);  in setup_per_cpu_areas()
3001 	if (!ai || !fc)  in setup_per_cpu_areas()
3006 	ai->dyn_size = unit_size;  in setup_per_cpu_areas()
3007 	ai->unit_size = unit_size;  in setup_per_cpu_areas()
3008 	ai->atom_size = unit_size;  in setup_per_cpu_areas()
3009 	ai->alloc_size = unit_size;  in setup_per_cpu_areas()
3010 	ai->groups[0].nr_units = 1;  in setup_per_cpu_areas()
3011 	ai->groups[0].cpu_map[0] = 0;  in setup_per_cpu_areas()
3013 	pcpu_setup_first_chunk(ai, fc);  in setup_per_cpu_areas()
3014 	pcpu_free_alloc_info(ai);  in setup_per_cpu_areas()