404 	struct mem_zone_bm_rtree *zone;  member
465 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
472 	block_nr = zone->blocks;  in add_rtree_block()
482 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
484 					&zone->nodes);  in add_rtree_block()
488 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
489 		zone->rtree = node;  in add_rtree_block()
490 		zone->levels += 1;  in add_rtree_block()
494 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
499 	node = zone->rtree;  in add_rtree_block()
500 	dst = &zone->rtree;  in add_rtree_block()
501 	block_nr = zone->blocks;  in add_rtree_block()
502 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
507 						&zone->nodes);  in add_rtree_block()
519 	zone->blocks += 1;  in add_rtree_block()
525 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
541 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
546 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
547 	if (!zone)  in create_zone_bm_rtree()
550 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
551 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
552 	zone->start_pfn = start;  in create_zone_bm_rtree()
553 	zone->end_pfn = end;  in create_zone_bm_rtree()
557 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
558 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
563 	return zone;  in create_zone_bm_rtree()
573 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
578 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
581 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
587 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
589 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
626 	struct zone *zone;  in create_mem_extents()  local
630 	for_each_populated_zone(zone) {  in create_mem_extents()
634 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
635 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
696 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
698 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
700 		if (!zone) {  in memory_bm_create()
704 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
725 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
727 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
728 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
747 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
751 	zone = bm->cur.zone;  in memory_bm_find_bit()
753 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
756 	zone = NULL;  in memory_bm_find_bit()
761 			zone = curr;  in memory_bm_find_bit()
766 	if (!zone)  in memory_bm_find_bit()
781 	if (zone == bm->cur.zone &&  in memory_bm_find_bit()
782 	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
785 	node      = zone->rtree;  in memory_bm_find_bit()
786 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
788 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
799 	bm->cur.zone = zone;  in memory_bm_find_bit()
801 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
805 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
884 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
894 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
895 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
897 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
925 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
930 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
951 static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)  in recycle_zone_bm_rtree()  argument
955 	list_for_each_entry(node, &zone->nodes, list)  in recycle_zone_bm_rtree()
958 	list_for_each_entry(node, &zone->leaves, list)  in recycle_zone_bm_rtree()
964 	struct mem_zone_bm_rtree *zone;  in memory_bm_recycle()  local
967 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
968 		recycle_zone_bm_rtree(zone);  in memory_bm_recycle()
1216 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1220 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1239 	struct zone *zone;  in count_free_highmem_pages()  local
1242 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1243 		if (is_highmem(zone))  in count_free_highmem_pages()
1244 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1257 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1265 	if (!page || page_zone(page) != zone)  in saveable_highmem_page()
1287 	struct zone *zone;  in count_highmem_pages()  local
1290 	for_each_populated_zone(zone) {  in count_highmem_pages()
1293 		if (!is_highmem(zone))  in count_highmem_pages()
1296 		mark_free_pages(zone);  in count_highmem_pages()
1297 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1298 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1299 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1305 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)  in saveable_highmem_page()
1321 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1329 	if (!page || page_zone(page) != zone)  in saveable_page()
1355 	struct zone *zone;  in count_data_pages()  local
1359 	for_each_populated_zone(zone) {  in count_data_pages()
1360 		if (is_highmem(zone))  in count_data_pages()
1363 		mark_free_pages(zone);  in count_data_pages()
1364 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1365 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1366 			if (saveable_page(zone, pfn))  in count_data_pages()
1404 static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1406 	return is_highmem(zone) ?  in page_is_saveable()
1407 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1439 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1451 	struct zone *zone;  in copy_data_pages()  local
1454 	for_each_populated_zone(zone) {  in copy_data_pages()
1457 		mark_free_pages(zone);  in copy_data_pages()
1458 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1459 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1460 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1738 	struct zone *zone;  in hibernate_preallocate_memory()  local
1774 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1775 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1776 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1777 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1779 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1916 	struct zone *zone;  in enough_free_mem()  local
1919 	for_each_populated_zone(zone)  in enough_free_mem()
1920 		if (!is_highmem(zone))  in enough_free_mem()
1921 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()