586 static void memory_bm_position_reset(struct memory_bitmap *bm)  in memory_bm_position_reset()  argument
588 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
590 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
592 	bm->cur.node_pfn = 0;  in memory_bm_position_reset()
593 	bm->cur.cur_pfn = BM_END_OF_MAP;  in memory_bm_position_reset()
594 	bm->cur.node_bit = 0;  in memory_bm_position_reset()
597 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
682 static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask,  in memory_bm_create()  argument
691 	INIT_LIST_HEAD(&bm->zones);  in memory_bm_create()
706 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
709 	bm->p_list = ca.chain;  in memory_bm_create()
710 	memory_bm_position_reset(bm);  in memory_bm_create()
716 	bm->p_list = ca.chain;  in memory_bm_create()
717 	memory_bm_free(bm, PG_UNSAFE_CLEAR);  in memory_bm_create()
723  * @bm: Memory bitmap.
725 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)  in memory_bm_free()  argument
729 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
732 	free_list_of_pages(bm->p_list, clear_nosave_free);  in memory_bm_free()
734 	INIT_LIST_HEAD(&bm->zones);  in memory_bm_free()
740  * Find the bit in memory bitmap @bm that corresponds to the given PFN.
741  * The cur.zone, cur.block and cur.node_pfn members of @bm are updated.
746 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,  in memory_bm_find_bit()  argument
753 	zone = bm->cur.zone;  in memory_bm_find_bit()
761 	list_for_each_entry(curr, &bm->zones, list) {  in memory_bm_find_bit()
782 	node = bm->cur.node;  in memory_bm_find_bit()
783 	if (zone == bm->cur.zone &&  in memory_bm_find_bit()
784 	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
801 	bm->cur.zone = zone;  in memory_bm_find_bit()
802 	bm->cur.node = node;  in memory_bm_find_bit()
803 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
804 	bm->cur.cur_pfn = pfn;  in memory_bm_find_bit()
813 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_set_bit()  argument
819 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_set_bit()
824 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)  in mem_bm_set_bit_check()  argument
830 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in mem_bm_set_bit_check()
837 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_clear_bit()  argument
843 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_clear_bit()
848 static void memory_bm_clear_current(struct memory_bitmap *bm)  in memory_bm_clear_current()  argument
852 	bit = max(bm->cur.node_bit - 1, 0);  in memory_bm_clear_current()
853 	clear_bit(bit, bm->cur.node->data);  in memory_bm_clear_current()
856 static unsigned long memory_bm_get_current(struct memory_bitmap *bm)  in memory_bm_get_current()  argument
858 	return bm->cur.cur_pfn;  in memory_bm_get_current()
861 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_test_bit()  argument
867 	error = memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_test_bit()
872 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)  in memory_bm_pfn_present()  argument
877 	return !memory_bm_find_bit(bm, pfn, &addr, &bit);  in memory_bm_pfn_present()
890 static bool rtree_next_node(struct memory_bitmap *bm)  in rtree_next_node()  argument
892 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
893 		bm->cur.node = list_entry(bm->cur.node->list.next,  in rtree_next_node()
895 		bm->cur.node_pfn += BM_BITS_PER_BLOCK;  in rtree_next_node()
896 		bm->cur.node_bit  = 0;  in rtree_next_node()
902 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
903 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
905 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
907 		bm->cur.node_pfn = 0;  in rtree_next_node()
908 		bm->cur.node_bit = 0;  in rtree_next_node()
918  * @bm: Memory bitmap.
921  * set bit in @bm and returns the PFN represented by it.  If no more bits are
927 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)  in memory_bm_next_pfn()  argument
933 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
934 		bits      = min(pages - bm->cur.node_pfn, BM_BITS_PER_BLOCK);  in memory_bm_next_pfn()
935 		bit	  = find_next_bit(bm->cur.node->data, bits,  in memory_bm_next_pfn()
936 					  bm->cur.node_bit);  in memory_bm_next_pfn()
938 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
939 			bm->cur.node_bit = bit + 1;  in memory_bm_next_pfn()
940 			bm->cur.cur_pfn = pfn;  in memory_bm_next_pfn()
943 	} while (rtree_next_node(bm));  in memory_bm_next_pfn()
945 	bm->cur.cur_pfn = BM_END_OF_MAP;  in memory_bm_next_pfn()
972 static void memory_bm_recycle(struct memory_bitmap *bm)  in memory_bm_recycle()  argument
977 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
980 	p_list = bm->p_list;  in memory_bm_recycle()
1078  * @bm: Memory bitmap.
1080  * Set the bits in @bm that correspond to the page frames the contents of which
1083 static void mark_nosave_pages(struct memory_bitmap *bm)  in mark_nosave_pages()  argument
1106 				mem_bm_set_bit_check(bm, pfn);  in mark_nosave_pages()
1198 	struct memory_bitmap *bm = free_pages_map;  in clear_or_poison_free_pages()  local
1205 		memory_bm_position_reset(bm);  in clear_or_poison_free_pages()
1206 		pfn = memory_bm_next_pfn(bm);  in clear_or_poison_free_pages()
1211 			pfn = memory_bm_next_pfn(bm);  in clear_or_poison_free_pages()
1213 		memory_bm_position_reset(bm);  in clear_or_poison_free_pages()
2052 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,  in alloc_highmem_pages()  argument
2065 		memory_bm_set_bit(bm, page_to_pfn(page));  in alloc_highmem_pages()
2072 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,  in alloc_highmem_pages()  argument
2205  * @bm: Memory bitmap.
2209  * PFNs corresponding to set bits in @bm are stored in the area of memory
2214 static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm,  in pack_pfns()  argument
2220 		buf[j] = memory_bm_next_pfn(bm);  in pack_pfns()
2310 static void mark_unsafe_pages(struct memory_bitmap *bm)  in mark_unsafe_pages()  argument
2323 	duplicate_memory_bitmap(free_pages_map, bm);  in mark_unsafe_pages()
2360  * @bm: Memory bitmap.
2365  * corresponding bit in @bm. If the page was originally populated with only
2368 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm,  in unpack_orig_pfns()  argument
2381 		if (pfn_valid(decoded_pfn) && memory_bm_pfn_present(bm, decoded_pfn)) {  in unpack_orig_pfns()
2382 			memory_bm_set_bit(bm, decoded_pfn);  in unpack_orig_pfns()
2420  * @bm: Memory bitmap.
2422  * The bits in @bm that correspond to image pages are assumed to be set.
2424 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)  in count_highmem_image_pages()  argument
2429 	memory_bm_position_reset(bm);  in count_highmem_image_pages()
2430 	pfn = memory_bm_next_pfn(bm);  in count_highmem_image_pages()
2435 		pfn = memory_bm_next_pfn(bm);  in count_highmem_image_pages()
2446  * @bm: Pointer to an uninitialized memory bitmap structure.
2453  * @bm (it must be uninitialized).
2457 static int prepare_highmem_image(struct memory_bitmap *bm,  in prepare_highmem_image()  argument
2462 	if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))  in prepare_highmem_image()
2481 			memory_bm_set_bit(bm, page_to_pfn(page));  in prepare_highmem_image()
2488 	memory_bm_position_reset(bm);  in prepare_highmem_image()
2489 	safe_highmem_bm = bm;  in prepare_highmem_image()
2590 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }  in count_highmem_image_pages()  argument
2592 static inline int prepare_highmem_image(struct memory_bitmap *bm,  in prepare_highmem_image()  argument
2611  * @bm: Memory bitmap with unsafe pages marked.
2614  * Use @bm to mark the pages that will be overwritten in the process of
2627 static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm,  in prepare_image()  argument
2639 	nr_highmem = count_highmem_image_pages(bm);  in prepare_image()
2640 	mark_unsafe_pages(bm);  in prepare_image()
2646 	duplicate_memory_bitmap(new_bm, bm);  in prepare_image()
2647 	memory_bm_free(bm, PG_UNSAFE_KEEP);  in prepare_image()
2667 		error = prepare_highmem_image(bm, &nr_highmem);  in prepare_image()
2723 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)  in get_buffer()  argument
2727 	unsigned long pfn = memory_bm_next_pfn(bm);  in get_buffer()