213 static bool pcpu_addr_in_chunk(struct pcpu_chunk *chunk, void *addr)  in pcpu_addr_in_chunk()  argument
217 	if (!chunk)  in pcpu_addr_in_chunk()
220 	start_addr = chunk->base_addr + chunk->start_offset;  in pcpu_addr_in_chunk()
221 	end_addr = chunk->base_addr + chunk->nr_pages * PAGE_SIZE -  in pcpu_addr_in_chunk()
222 		   chunk->end_offset;  in pcpu_addr_in_chunk()
240 static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)  in pcpu_chunk_slot()  argument
242 	const struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_chunk_slot()
244 	if (chunk->free_bytes < PCPU_MIN_ALLOC_SIZE ||  in pcpu_chunk_slot()
273 static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,  in pcpu_chunk_addr()  argument
276 	return (unsigned long)chunk->base_addr +  in pcpu_chunk_addr()
284 static unsigned long *pcpu_index_alloc_map(struct pcpu_chunk *chunk, int index)  in pcpu_index_alloc_map()  argument
286 	return chunk->alloc_map +  in pcpu_index_alloc_map()
344 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
352 	for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk);  in pcpu_next_md_free_region()
399 static void pcpu_next_fit_region(struct pcpu_chunk *chunk, int alloc_bits,  in pcpu_next_fit_region()  argument
407 	for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk);  in pcpu_next_fit_region()
447 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
456 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
457 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
458 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
460 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
462 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
463 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
465 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
467 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
505 static void __pcpu_chunk_move(struct pcpu_chunk *chunk, int slot,  in __pcpu_chunk_move()  argument
508 	if (chunk != pcpu_reserved_chunk) {  in __pcpu_chunk_move()
511 		pcpu_slot = pcpu_chunk_list(pcpu_chunk_type(chunk));  in __pcpu_chunk_move()
513 			list_move(&chunk->list, &pcpu_slot[slot]);  in __pcpu_chunk_move()
515 			list_move_tail(&chunk->list, &pcpu_slot[slot]);  in __pcpu_chunk_move()
519 static void pcpu_chunk_move(struct pcpu_chunk *chunk, int slot)  in pcpu_chunk_move()  argument
521 	__pcpu_chunk_move(chunk, slot, true);  in pcpu_chunk_move()
537 static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)  in pcpu_chunk_relocate()  argument
539 	int nslot = pcpu_chunk_slot(chunk);  in pcpu_chunk_relocate()
542 		__pcpu_chunk_move(chunk, nslot, oslot < nslot);  in pcpu_chunk_relocate()
554 static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)  in pcpu_update_empty_pages()  argument
556 	chunk->nr_empty_pop_pages += nr;  in pcpu_update_empty_pages()
557 	if (chunk != pcpu_reserved_chunk)  in pcpu_update_empty_pages()
668 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_scan()  argument
680 	block = chunk->md_blocks + s_index;  in pcpu_block_update_scan()
683 	l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index), s_off);  in pcpu_block_update_scan()
701 static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan)  in pcpu_chunk_refresh_hint()  argument
703 	struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_chunk_refresh_hint()
718 	pcpu_for_each_md_free_region(chunk, bit_off, bits)  in pcpu_chunk_refresh_hint()
730 static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index)  in pcpu_block_refresh_hint()  argument
732 	struct pcpu_block_md *block = chunk->md_blocks + index;  in pcpu_block_refresh_hint()
733 	unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index);  in pcpu_block_refresh_hint()
765 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
768 	struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_block_update_hint_alloc()
785 	s_block = chunk->md_blocks + s_index;  in pcpu_block_update_hint_alloc()
786 	e_block = chunk->md_blocks + e_index;  in pcpu_block_update_hint_alloc()
799 					pcpu_index_alloc_map(chunk, s_index),  in pcpu_block_update_hint_alloc()
817 		pcpu_block_refresh_hint(chunk, s_index);  in pcpu_block_update_hint_alloc()
840 				pcpu_index_alloc_map(chunk, e_index),  in pcpu_block_update_hint_alloc()
853 				pcpu_block_refresh_hint(chunk, e_index);  in pcpu_block_update_hint_alloc()
872 		pcpu_update_empty_pages(chunk, -nr_empty_pages);  in pcpu_block_update_hint_alloc()
891 		pcpu_chunk_refresh_hint(chunk, false);  in pcpu_block_update_hint_alloc()
912 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
932 	s_block = chunk->md_blocks + s_index;  in pcpu_block_update_hint_free()
933 	e_block = chunk->md_blocks + e_index;  in pcpu_block_update_hint_free()
955 		int l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index),  in pcpu_block_update_hint_free()
964 		end = find_next_bit(pcpu_index_alloc_map(chunk, e_index),  in pcpu_block_update_hint_free()
993 		pcpu_update_empty_pages(chunk, nr_empty_pages);  in pcpu_block_update_hint_free()
1002 		pcpu_chunk_refresh_hint(chunk, true);  in pcpu_block_update_hint_free()
1004 		pcpu_block_update(&chunk->chunk_md,  in pcpu_block_update_hint_free()
1022 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
1031 	bitmap_next_clear_region(chunk->populated, &rs, &re, page_end);  in pcpu_is_populated()
1058 static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits,  in pcpu_find_block_fit()  argument
1061 	struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_find_block_fit()
1077 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
1078 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
1086 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
1167 static int pcpu_alloc_area(struct pcpu_chunk *chunk, int alloc_bits,  in pcpu_alloc_area()  argument
1170 	struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_alloc_area()
1177 	oslot = pcpu_chunk_slot(chunk);  in pcpu_alloc_area()
1183 		    pcpu_chunk_map_bits(chunk));  in pcpu_alloc_area()
1184 	bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,  in pcpu_alloc_area()
1190 		pcpu_block_update_scan(chunk, area_off, area_bits);  in pcpu_alloc_area()
1193 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1196 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1197 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1198 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1200 	chunk->free_bytes -= alloc_bits * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1205 					chunk->alloc_map,  in pcpu_alloc_area()
1206 					pcpu_chunk_map_bits(chunk),  in pcpu_alloc_area()
1209 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1211 	pcpu_chunk_relocate(chunk, oslot);  in pcpu_alloc_area()
1227 static int pcpu_free_area(struct pcpu_chunk *chunk, int off)  in pcpu_free_area()  argument
1229 	struct pcpu_block_md *chunk_md = &chunk->chunk_md;  in pcpu_free_area()
1233 	pcpu_stats_area_dealloc(chunk);  in pcpu_free_area()
1235 	oslot = pcpu_chunk_slot(chunk);  in pcpu_free_area()
1240 	end = find_next_bit(chunk->bound_map, pcpu_chunk_map_bits(chunk),  in pcpu_free_area()
1243 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1248 	chunk->free_bytes += freed;  in pcpu_free_area()
1253 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()
1255 	pcpu_chunk_relocate(chunk, oslot);  in pcpu_free_area()
1270 static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)  in pcpu_init_md_blocks()  argument
1275 	pcpu_init_md_block(&chunk->chunk_md, pcpu_chunk_map_bits(chunk));  in pcpu_init_md_blocks()
1277 	for (md_block = chunk->md_blocks;  in pcpu_init_md_blocks()
1278 	     md_block != chunk->md_blocks + pcpu_chunk_nr_blocks(chunk);  in pcpu_init_md_blocks()
1299 	struct pcpu_chunk *chunk;  in pcpu_alloc_first_chunk()  local
1318 	alloc_size = struct_size(chunk, populated,  in pcpu_alloc_first_chunk()
1320 	chunk = memblock_alloc(alloc_size, SMP_CACHE_BYTES);  in pcpu_alloc_first_chunk()
1321 	if (!chunk)  in pcpu_alloc_first_chunk()
1325 	INIT_LIST_HEAD(&chunk->list);  in pcpu_alloc_first_chunk()
1327 	chunk->base_addr = (void *)aligned_addr;  in pcpu_alloc_first_chunk()
1328 	chunk->start_offset = start_offset;  in pcpu_alloc_first_chunk()
1329 	chunk->end_offset = region_size - chunk->start_offset - map_size;  in pcpu_alloc_first_chunk()
1331 	chunk->nr_pages = region_size >> PAGE_SHIFT;  in pcpu_alloc_first_chunk()
1332 	region_bits = pcpu_chunk_map_bits(chunk);  in pcpu_alloc_first_chunk()
1334 	alloc_size = BITS_TO_LONGS(region_bits) * sizeof(chunk->alloc_map[0]);  in pcpu_alloc_first_chunk()
1335 	chunk->alloc_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES);  in pcpu_alloc_first_chunk()
1336 	if (!chunk->alloc_map)  in pcpu_alloc_first_chunk()
1341 		BITS_TO_LONGS(region_bits + 1) * sizeof(chunk->bound_map[0]);  in pcpu_alloc_first_chunk()
1342 	chunk->bound_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES);  in pcpu_alloc_first_chunk()
1343 	if (!chunk->bound_map)  in pcpu_alloc_first_chunk()
1347 	alloc_size = pcpu_chunk_nr_blocks(chunk) * sizeof(chunk->md_blocks[0]);  in pcpu_alloc_first_chunk()
1348 	chunk->md_blocks = memblock_alloc(alloc_size, SMP_CACHE_BYTES);  in pcpu_alloc_first_chunk()
1349 	if (!chunk->md_blocks)  in pcpu_alloc_first_chunk()
1355 	chunk->obj_cgroups = NULL;  in pcpu_alloc_first_chunk()
1357 	pcpu_init_md_blocks(chunk);  in pcpu_alloc_first_chunk()
1360 	chunk->immutable = true;  in pcpu_alloc_first_chunk()
1361 	bitmap_fill(chunk->populated, chunk->nr_pages);  in pcpu_alloc_first_chunk()
1362 	chunk->nr_populated = chunk->nr_pages;  in pcpu_alloc_first_chunk()
1363 	chunk->nr_empty_pop_pages = chunk->nr_pages;  in pcpu_alloc_first_chunk()
1365 	chunk->free_bytes = map_size;  in pcpu_alloc_first_chunk()
1367 	if (chunk->start_offset) {  in pcpu_alloc_first_chunk()
1369 		offset_bits = chunk->start_offset / PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_first_chunk()
1370 		bitmap_set(chunk->alloc_map, 0, offset_bits);  in pcpu_alloc_first_chunk()
1371 		set_bit(0, chunk->bound_map);  in pcpu_alloc_first_chunk()
1372 		set_bit(offset_bits, chunk->bound_map);  in pcpu_alloc_first_chunk()
1374 		chunk->chunk_md.first_free = offset_bits;  in pcpu_alloc_first_chunk()
1376 		pcpu_block_update_hint_alloc(chunk, 0, offset_bits);  in pcpu_alloc_first_chunk()
1379 	if (chunk->end_offset) {  in pcpu_alloc_first_chunk()
1381 		offset_bits = chunk->end_offset / PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_first_chunk()
1382 		bitmap_set(chunk->alloc_map,  in pcpu_alloc_first_chunk()
1383 			   pcpu_chunk_map_bits(chunk) - offset_bits,  in pcpu_alloc_first_chunk()
1386 			chunk->bound_map);  in pcpu_alloc_first_chunk()
1387 		set_bit(region_bits, chunk->bound_map);  in pcpu_alloc_first_chunk()
1389 		pcpu_block_update_hint_alloc(chunk, pcpu_chunk_map_bits(chunk)  in pcpu_alloc_first_chunk()
1393 	return chunk;  in pcpu_alloc_first_chunk()
1398 	struct pcpu_chunk *chunk;  in pcpu_alloc_chunk()  local
1401 	chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size, gfp);  in pcpu_alloc_chunk()
1402 	if (!chunk)  in pcpu_alloc_chunk()
1405 	INIT_LIST_HEAD(&chunk->list);  in pcpu_alloc_chunk()
1406 	chunk->nr_pages = pcpu_unit_pages;  in pcpu_alloc_chunk()
1407 	region_bits = pcpu_chunk_map_bits(chunk);  in pcpu_alloc_chunk()
1409 	chunk->alloc_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits) *  in pcpu_alloc_chunk()
1410 					   sizeof(chunk->alloc_map[0]), gfp);  in pcpu_alloc_chunk()
1411 	if (!chunk->alloc_map)  in pcpu_alloc_chunk()
1414 	chunk->bound_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits + 1) *  in pcpu_alloc_chunk()
1415 					   sizeof(chunk->bound_map[0]), gfp);  in pcpu_alloc_chunk()
1416 	if (!chunk->bound_map)  in pcpu_alloc_chunk()
1419 	chunk->md_blocks = pcpu_mem_zalloc(pcpu_chunk_nr_blocks(chunk) *  in pcpu_alloc_chunk()
1420 					   sizeof(chunk->md_blocks[0]), gfp);  in pcpu_alloc_chunk()
1421 	if (!chunk->md_blocks)  in pcpu_alloc_chunk()
1426 		chunk->obj_cgroups =  in pcpu_alloc_chunk()
1427 			pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) *  in pcpu_alloc_chunk()
1429 		if (!chunk->obj_cgroups)  in pcpu_alloc_chunk()
1434 	pcpu_init_md_blocks(chunk);  in pcpu_alloc_chunk()
1437 	chunk->free_bytes = chunk->nr_pages * PAGE_SIZE;  in pcpu_alloc_chunk()
1439 	return chunk;  in pcpu_alloc_chunk()
1443 	pcpu_mem_free(chunk->md_blocks);  in pcpu_alloc_chunk()
1446 	pcpu_mem_free(chunk->bound_map);  in pcpu_alloc_chunk()
1448 	pcpu_mem_free(chunk->alloc_map);  in pcpu_alloc_chunk()
1450 	pcpu_mem_free(chunk);  in pcpu_alloc_chunk()
1455 static void pcpu_free_chunk(struct pcpu_chunk *chunk)  in pcpu_free_chunk()  argument
1457 	if (!chunk)  in pcpu_free_chunk()
1460 	pcpu_mem_free(chunk->obj_cgroups);  in pcpu_free_chunk()
1462 	pcpu_mem_free(chunk->md_blocks);  in pcpu_free_chunk()
1463 	pcpu_mem_free(chunk->bound_map);  in pcpu_free_chunk()
1464 	pcpu_mem_free(chunk->alloc_map);  in pcpu_free_chunk()
1465 	pcpu_mem_free(chunk);  in pcpu_free_chunk()
1481 static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,  in pcpu_chunk_populated()  argument
1488 	bitmap_set(chunk->populated, page_start, nr);  in pcpu_chunk_populated()
1489 	chunk->nr_populated += nr;  in pcpu_chunk_populated()
1492 	pcpu_update_empty_pages(chunk, nr);  in pcpu_chunk_populated()
1505 static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,  in pcpu_chunk_depopulated()  argument
1512 	bitmap_clear(chunk->populated, page_start, nr);  in pcpu_chunk_depopulated()
1513 	chunk->nr_populated -= nr;  in pcpu_chunk_depopulated()
1516 	pcpu_update_empty_pages(chunk, -nr);  in pcpu_chunk_depopulated()
1534 static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
1536 static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
1540 static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
1604 				       struct pcpu_chunk *chunk, int off,  in pcpu_memcg_post_alloc_hook()  argument
1610 	if (chunk) {  in pcpu_memcg_post_alloc_hook()
1611 		chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = objcg;  in pcpu_memcg_post_alloc_hook()
1623 static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)  in pcpu_memcg_free_hook()  argument
1627 	if (!pcpu_is_memcg_chunk(pcpu_chunk_type(chunk)))  in pcpu_memcg_free_hook()
1630 	objcg = chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT];  in pcpu_memcg_free_hook()
1631 	chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = NULL;  in pcpu_memcg_free_hook()
1651 				       struct pcpu_chunk *chunk, int off,  in pcpu_memcg_post_alloc_hook()  argument
1656 static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)  in pcpu_memcg_free_hook()  argument
1686 	struct pcpu_chunk *chunk, *next;  in pcpu_alloc()  local
1742 		chunk = pcpu_reserved_chunk;  in pcpu_alloc()
1744 		off = pcpu_find_block_fit(chunk, bits, bit_align, is_atomic);  in pcpu_alloc()
1750 		off = pcpu_alloc_area(chunk, bits, bit_align, off);  in pcpu_alloc()
1761 		list_for_each_entry_safe(chunk, next, &pcpu_slot[slot], list) {  in pcpu_alloc()
1762 			off = pcpu_find_block_fit(chunk, bits, bit_align,  in pcpu_alloc()
1766 					pcpu_chunk_move(chunk, 0);  in pcpu_alloc()
1770 			off = pcpu_alloc_area(chunk, bits, bit_align, off);  in pcpu_alloc()
1790 		chunk = pcpu_create_chunk(type, pcpu_gfp);  in pcpu_alloc()
1791 		if (!chunk) {  in pcpu_alloc()
1797 		pcpu_chunk_relocate(chunk, -1);  in pcpu_alloc()
1805 	pcpu_stats_area_alloc(chunk, size);  in pcpu_alloc()
1815 		bitmap_for_each_clear_region(chunk->populated, rs, re,  in pcpu_alloc()
1817 			WARN_ON(chunk->immutable);  in pcpu_alloc()
1819 			ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp);  in pcpu_alloc()
1823 				pcpu_free_area(chunk, off);  in pcpu_alloc()
1827 			pcpu_chunk_populated(chunk, rs, re);  in pcpu_alloc()
1839 		memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);  in pcpu_alloc()
1841 	ptr = __addr_to_pcpu_ptr(chunk->base_addr + off);  in pcpu_alloc()
1845 			chunk->base_addr, off, ptr);  in pcpu_alloc()
1847 	pcpu_memcg_post_alloc_hook(objcg, chunk, off, size);  in pcpu_alloc()
1949 	struct pcpu_chunk *chunk, *next;  in __pcpu_balance_workfn()  local
1959 	list_for_each_entry_safe(chunk, next, free_head, list) {  in __pcpu_balance_workfn()
1960 		WARN_ON(chunk->immutable);  in __pcpu_balance_workfn()
1963 		if (chunk == list_first_entry(free_head, struct pcpu_chunk, list))  in __pcpu_balance_workfn()
1966 		list_move(&chunk->list, &to_free);  in __pcpu_balance_workfn()
1971 	list_for_each_entry_safe(chunk, next, &to_free, list) {  in __pcpu_balance_workfn()
1974 		bitmap_for_each_set_region(chunk->populated, rs, re, 0,  in __pcpu_balance_workfn()
1975 					   chunk->nr_pages) {  in __pcpu_balance_workfn()
1976 			pcpu_depopulate_chunk(chunk, rs, re);  in __pcpu_balance_workfn()
1978 			pcpu_chunk_depopulated(chunk, rs, re);  in __pcpu_balance_workfn()
1981 		pcpu_destroy_chunk(chunk);  in __pcpu_balance_workfn()
2013 		list_for_each_entry(chunk, &pcpu_slot[slot], list) {  in __pcpu_balance_workfn()
2014 			nr_unpop = chunk->nr_pages - chunk->nr_populated;  in __pcpu_balance_workfn()
2024 		bitmap_for_each_clear_region(chunk->populated, rs, re, 0,  in __pcpu_balance_workfn()
2025 					     chunk->nr_pages) {  in __pcpu_balance_workfn()
2028 			ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp);  in __pcpu_balance_workfn()
2032 				pcpu_chunk_populated(chunk, rs, rs + nr);  in __pcpu_balance_workfn()
2045 		chunk = pcpu_create_chunk(type, gfp);  in __pcpu_balance_workfn()
2046 		if (chunk) {  in __pcpu_balance_workfn()
2048 			pcpu_chunk_relocate(chunk, -1);  in __pcpu_balance_workfn()
2083 	struct pcpu_chunk *chunk;  in free_percpu()  local
2098 	chunk = pcpu_chunk_addr_search(addr);  in free_percpu()
2099 	off = addr - chunk->base_addr;  in free_percpu()
2101 	size = pcpu_free_area(chunk, off);  in free_percpu()
2103 	pcpu_slot = pcpu_chunk_list(pcpu_chunk_type(chunk));  in free_percpu()
2105 	pcpu_memcg_free_hook(chunk, off, size);  in free_percpu()
2108 	if (chunk->free_bytes == pcpu_unit_size) {  in free_percpu()
2112 			if (pos != chunk) {  in free_percpu()
2118 	trace_percpu_free_percpu(chunk->base_addr, off, ptr);  in free_percpu()
2406 	struct pcpu_chunk *chunk;  in pcpu_setup_first_chunk()  local
2524 	pcpu_chunk_struct_size = struct_size(chunk, populated,  in pcpu_setup_first_chunk()
2568 	chunk = pcpu_alloc_first_chunk(tmp_addr, map_size);  in pcpu_setup_first_chunk()
2572 		pcpu_reserved_chunk = chunk;  in pcpu_setup_first_chunk()
2577 		chunk = pcpu_alloc_first_chunk(tmp_addr, map_size);  in pcpu_setup_first_chunk()
2581 	pcpu_first_chunk = chunk;  in pcpu_setup_first_chunk()