373 	struct mem_zone_bm_rtree *zone;  member
430 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
437 	block_nr = zone->blocks;  in add_rtree_block()
447 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
449 					&zone->nodes);  in add_rtree_block()
453 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
454 		zone->rtree = node;  in add_rtree_block()
455 		zone->levels += 1;  in add_rtree_block()
459 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
464 	node = zone->rtree;  in add_rtree_block()
465 	dst = &zone->rtree;  in add_rtree_block()
466 	block_nr = zone->blocks;  in add_rtree_block()
467 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
472 						&zone->nodes);  in add_rtree_block()
484 	zone->blocks += 1;  in add_rtree_block()
490 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
506 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
511 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
512 	if (!zone)  in create_zone_bm_rtree()
515 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
516 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
517 	zone->start_pfn = start;  in create_zone_bm_rtree()
518 	zone->end_pfn = end;  in create_zone_bm_rtree()
522 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
523 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
528 	return zone;  in create_zone_bm_rtree()
538 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
543 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
546 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
552 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
554 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
591 	struct zone *zone;  in create_mem_extents()  local
595 	for_each_populated_zone(zone) {  in create_mem_extents()
599 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
600 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
661 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
663 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
665 		if (!zone) {  in memory_bm_create()
669 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
690 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
692 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
693 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
712 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
716 	zone = bm->cur.zone;  in memory_bm_find_bit()
718 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
721 	zone = NULL;  in memory_bm_find_bit()
726 			zone = curr;  in memory_bm_find_bit()
731 	if (!zone)  in memory_bm_find_bit()
740 	if (((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
743 	node      = zone->rtree;  in memory_bm_find_bit()
744 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
746 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
757 	bm->cur.zone = zone;  in memory_bm_find_bit()
759 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
763 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
842 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
852 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
853 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
855 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
883 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
888 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
909 static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)  in recycle_zone_bm_rtree()  argument
913 	list_for_each_entry(node, &zone->nodes, list)  in recycle_zone_bm_rtree()
916 	list_for_each_entry(node, &zone->leaves, list)  in recycle_zone_bm_rtree()
922 	struct mem_zone_bm_rtree *zone;  in memory_bm_recycle()  local
925 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
926 		recycle_zone_bm_rtree(zone);  in memory_bm_recycle()
1169 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1173 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1192 	struct zone *zone;  in count_free_highmem_pages()  local
1195 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1196 		if (is_highmem(zone))  in count_free_highmem_pages()
1197 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1210 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1218 	if (page_zone(page) != zone)  in saveable_highmem_page()
1238 	struct zone *zone;  in count_highmem_pages()  local
1241 	for_each_populated_zone(zone) {  in count_highmem_pages()
1244 		if (!is_highmem(zone))  in count_highmem_pages()
1247 		mark_free_pages(zone);  in count_highmem_pages()
1248 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1249 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1250 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1256 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)  in saveable_highmem_page()
1272 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1280 	if (page_zone(page) != zone)  in saveable_page()
1303 	struct zone *zone;  in count_data_pages()  local
1307 	for_each_populated_zone(zone) {  in count_data_pages()
1308 		if (is_highmem(zone))  in count_data_pages()
1311 		mark_free_pages(zone);  in count_data_pages()
1312 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1313 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1314 			if (saveable_page(zone, pfn))  in count_data_pages()
1351 static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1353 	return is_highmem(zone) ?  in page_is_saveable()
1354 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1386 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1398 	struct zone *zone;  in copy_data_pages()  local
1401 	for_each_populated_zone(zone) {  in copy_data_pages()
1404 		mark_free_pages(zone);  in copy_data_pages()
1405 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1406 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1407 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1687 	struct zone *zone;  in hibernate_preallocate_memory()  local
1719 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1720 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1721 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1722 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1724 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1862 	struct zone *zone;  in enough_free_mem()  local
1865 	for_each_populated_zone(zone)  in enough_free_mem()
1866 		if (!is_highmem(zone))  in enough_free_mem()
1867 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()