585 static void memory_bm_position_reset(struct memory_bitmap *bm)  in memory_bm_position_reset()  argument
587 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
589 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
591 	bm->cur.node_pfn = 0;  in memory_bm_position_reset()
592 	bm->cur.node_bit = 0;  in memory_bm_position_reset()
595 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
680 static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask,  in memory_bm_create()  argument
689 	INIT_LIST_HEAD(&bm->zones);  in memory_bm_create()
704 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
707 	bm->p_list = ca.chain;  in memory_bm_create()
708 	memory_bm_position_reset(bm);  in memory_bm_create()
714 	bm->p_list = ca.chain;  in memory_bm_create()
715 	memory_bm_free(bm, PG_UNSAFE_CLEAR);  in memory_bm_create()
721  * @bm: Memory bitmap.
723 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)  in memory_bm_free()  argument
727 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
730 	free_list_of_pages(bm->p_list, clear_nosave_free);  in memory_bm_free()
732 	INIT_LIST_HEAD(&bm->zones);  in memory_bm_free()
738  * Find the bit in memory bitmap @bm that corresponds to the given PFN.
739  * The cur.zone, cur.block and cur.node_pfn members of @bm are updated.
744 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,  in memory_bm_find_bit()  argument
751 	zone = bm->cur.zone;  in memory_bm_find_bit()
759 	list_for_each_entry(curr, &bm->zones, list) {  in memory_bm_find_bit()
780 	node = bm->cur.node;  in memory_bm_find_bit()
781 	if (zone == bm->cur.zone &&  in memory_bm_find_bit()
782 	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
799 	bm->cur.zone = zone;  in memory_bm_find_bit()
800 	bm->cur.node = node;  in memory_bm_find_bit()
801 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
810 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_set_bit()  argument
816 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_set_bit()
821 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)  in mem_bm_set_bit_check()  argument
827 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in mem_bm_set_bit_check()
834 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_clear_bit()  argument
840 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_clear_bit()
845 static void memory_bm_clear_current(struct memory_bitmap *bm)  in memory_bm_clear_current()  argument
849 	bit = max(bm->cur.node_bit - 1, 0);  in memory_bm_clear_current()
850 	clear_bit(bit, bm->cur.node->data);  in memory_bm_clear_current()
853 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_test_bit()  argument
859 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_test_bit()
864 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_pfn_present()  argument
869 	return !memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_pfn_present()
882 static bool rtree_next_node(struct memory_bitmap *bm)  in rtree_next_node()  argument
884 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
885 		bm->cur.node = list_entry(bm->cur.node->list.next,  in rtree_next_node()
887 		bm->cur.node_pfn += BM_BITS_PER_BLOCK;  in rtree_next_node()
888 		bm->cur.node_bit  = 0;  in rtree_next_node()
894 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
895 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
897 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
899 		bm->cur.node_pfn = 0;  in rtree_next_node()
900 		bm->cur.node_bit = 0;  in rtree_next_node()
910  * @bm: Memory bitmap.
913  * set bit in @bm and returns the PFN represented by it.  If no more bits are
919 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)  in memory_bm_next_pfn()  argument
925 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
926 		bits      = min(pages - bm->cur.node_pfn, BM_BITS_PER_BLOCK);  in memory_bm_next_pfn()
927 		bit	  = find_next_bit(bm->cur.node->data, bits,  in memory_bm_next_pfn()
928 					  bm->cur.node_bit);  in memory_bm_next_pfn()
930 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
931 			bm->cur.node_bit = bit + 1;  in memory_bm_next_pfn()
934 	} while (rtree_next_node(bm));  in memory_bm_next_pfn()
962 static void memory_bm_recycle(struct memory_bitmap *bm)  in memory_bm_recycle()  argument
967 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
970 	p_list = bm->p_list;  in memory_bm_recycle()
1068  * @bm: Memory bitmap.
1070  * Set the bits in @bm that correspond to the page frames the contents of which
1073 static void mark_nosave_pages(struct memory_bitmap *bm)  in mark_nosave_pages()  argument
1096 				mem_bm_set_bit_check(bm, pfn);  in mark_nosave_pages()
1188 	struct memory_bitmap *bm = free_pages_map;  in clear_or_poison_free_pages()  local
1195 		memory_bm_position_reset(bm);  in clear_or_poison_free_pages()
1196 		pfn = memory_bm_next_pfn(bm);  in clear_or_poison_free_pages()
1201 			pfn = memory_bm_next_pfn(bm);  in clear_or_poison_free_pages()
1203 		memory_bm_position_reset(bm);  in clear_or_poison_free_pages()
1949 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,  in alloc_highmem_pages()  argument
1962 		memory_bm_set_bit(bm, page_to_pfn(page));  in alloc_highmem_pages()
1969 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,  in alloc_highmem_pages()  argument
2099  * @bm: Memory bitmap.
2102  * PFNs corresponding to set bits in @bm are stored in the area of memory
2105 static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm)  in pack_pfns()  argument
2110 		buf[j] = memory_bm_next_pfn(bm);  in pack_pfns()
2198 static void mark_unsafe_pages(struct memory_bitmap *bm)  in mark_unsafe_pages()  argument
2211 	duplicate_memory_bitmap(free_pages_map, bm);  in mark_unsafe_pages()
2248  * @bm: Memory bitmap.
2252  * corresponding bit in @bm.
2254 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)  in unpack_orig_pfns()  argument
2262 		if (pfn_valid(buf[j]) && memory_bm_pfn_present(bm, buf[j]))  in unpack_orig_pfns()
2263 			memory_bm_set_bit(bm, buf[j]);  in unpack_orig_pfns()
2293  * @bm: Memory bitmap.
2295  * The bits in @bm that correspond to image pages are assumed to be set.
2297 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)  in count_highmem_image_pages()  argument
2302 	memory_bm_position_reset(bm);  in count_highmem_image_pages()
2303 	pfn = memory_bm_next_pfn(bm);  in count_highmem_image_pages()
2308 		pfn = memory_bm_next_pfn(bm);  in count_highmem_image_pages()
2319  * @bm: Pointer to an uninitialized memory bitmap structure.
2326  * @bm (it must be uninitialized).
2330 static int prepare_highmem_image(struct memory_bitmap *bm,  in prepare_highmem_image()  argument
2335 	if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))  in prepare_highmem_image()
2354 			memory_bm_set_bit(bm, page_to_pfn(page));  in prepare_highmem_image()
2361 	memory_bm_position_reset(bm);  in prepare_highmem_image()
2362 	safe_highmem_bm = bm;  in prepare_highmem_image()
2463 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }  in count_highmem_image_pages()  argument
2465 static inline int prepare_highmem_image(struct memory_bitmap *bm,  in prepare_highmem_image()  argument
2484  * @bm: Memory bitmap with unsafe pages marked.
2486  * Use @bm to mark the pages that will be overwritten in the process of
2496 static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)  in prepare_image()  argument
2506 	nr_highmem = count_highmem_image_pages(bm);  in prepare_image()
2507 	mark_unsafe_pages(bm);  in prepare_image()
2513 	duplicate_memory_bitmap(new_bm, bm);  in prepare_image()
2514 	memory_bm_free(bm, PG_UNSAFE_KEEP);  in prepare_image()
2516 		error = prepare_highmem_image(bm, &nr_highmem);  in prepare_image()
2572 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)  in get_buffer()  argument
2576 	unsigned long pfn = memory_bm_next_pfn(bm);  in get_buffer()