332 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
335 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_md_free_region()
336 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_md_free_region()
361 			*bit_off = pcpu_block_off_to_off(i,  in pcpu_next_md_free_region()
369 		*bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;  in pcpu_next_md_free_region()
388 				 int align, int *bit_off, int *bits)  in pcpu_next_fit_region()  argument
390 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_fit_region()
391 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_fit_region()
418 			*bit_off = pcpu_block_off_to_off(i, block->first_free);  in pcpu_next_fit_region()
424 		*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,  in pcpu_next_fit_region()
426 		*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;  in pcpu_next_fit_region()
427 		*bit_off = pcpu_block_off_to_off(i, *bit_off);  in pcpu_next_fit_region()
433 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
442 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
443 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
444 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
445 	     (bit_off) += (bits) + 1,					\
446 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
448 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
449 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
451 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
452 	     (bit_off) += (bits),					      \
453 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
529 static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off,  in pcpu_cnt_pop_pages()  argument
532 	int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_cnt_pop_pages()
533 	int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_cnt_pop_pages()
558 static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits)  in pcpu_chunk_update()  argument
561 		chunk->contig_bits_start = bit_off;  in pcpu_chunk_update()
564 		   (!bit_off ||  in pcpu_chunk_update()
565 		    __ffs(bit_off) > __ffs(chunk->contig_bits_start))) {  in pcpu_chunk_update()
567 		chunk->contig_bits_start = bit_off;  in pcpu_chunk_update()
586 	int bit_off, bits, nr_empty_pop_pages;  in pcpu_chunk_refresh_hint()  local
591 	bit_off = chunk->first_bit;  in pcpu_chunk_refresh_hint()
593 	pcpu_for_each_md_free_region(chunk, bit_off, bits) {  in pcpu_chunk_refresh_hint()
594 		pcpu_chunk_update(chunk, bit_off, bits);  in pcpu_chunk_refresh_hint()
596 		nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits);  in pcpu_chunk_refresh_hint()
680 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
693 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_alloc()
694 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_alloc()
695 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_alloc()
696 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_alloc()
767 	if (bit_off >= chunk->contig_bits_start  &&  in pcpu_block_update_hint_alloc()
768 	    bit_off < chunk->contig_bits_start + chunk->contig_bits)  in pcpu_block_update_hint_alloc()
789 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
803 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_free()
804 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_free()
805 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_free()
806 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_free()
890 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
895 	page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
896 	page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
929 	int bit_off, bits, next_off;  in pcpu_find_block_fit()  local
937 	bit_off = ALIGN(chunk->contig_bits_start, align) -  in pcpu_find_block_fit()
939 	if (bit_off + alloc_bits > chunk->contig_bits)  in pcpu_find_block_fit()
942 	bit_off = chunk->first_bit;  in pcpu_find_block_fit()
944 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
945 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
949 		bit_off = next_off;  in pcpu_find_block_fit()
953 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
956 	return bit_off;  in pcpu_find_block_fit()
982 	int bit_off, end, oslot;  in pcpu_alloc_area()  local
992 	bit_off = bitmap_find_next_zero_area(chunk->alloc_map, end, start,  in pcpu_alloc_area()
994 	if (bit_off >= end)  in pcpu_alloc_area()
998 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1001 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1002 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1003 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1008 	if (bit_off == chunk->first_bit)  in pcpu_alloc_area()
1012 					bit_off + alloc_bits);  in pcpu_alloc_area()
1014 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1018 	return bit_off * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1031 	int bit_off, bits, end, oslot;  in pcpu_free_area()  local
1038 	bit_off = off / PCPU_MIN_ALLOC_SIZE;  in pcpu_free_area()
1042 			    bit_off + 1);  in pcpu_free_area()
1043 	bits = end - bit_off;  in pcpu_free_area()
1044 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1050 	chunk->first_bit = min(chunk->first_bit, bit_off);  in pcpu_free_area()
1052 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()