370 	struct mem_zone_bm_rtree *zone;  member
427 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
434 	block_nr = zone->blocks;  in add_rtree_block()
444 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
446 					&zone->nodes);  in add_rtree_block()
450 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
451 		zone->rtree = node;  in add_rtree_block()
452 		zone->levels += 1;  in add_rtree_block()
456 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
461 	node = zone->rtree;  in add_rtree_block()
462 	dst = &zone->rtree;  in add_rtree_block()
463 	block_nr = zone->blocks;  in add_rtree_block()
464 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
469 						&zone->nodes);  in add_rtree_block()
481 	zone->blocks += 1;  in add_rtree_block()
487 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
503 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
508 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
509 	if (!zone)  in create_zone_bm_rtree()
512 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
513 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
514 	zone->start_pfn = start;  in create_zone_bm_rtree()
515 	zone->end_pfn = end;  in create_zone_bm_rtree()
519 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
520 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
525 	return zone;  in create_zone_bm_rtree()
535 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
540 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
543 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
549 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
551 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
588 	struct zone *zone;  in create_mem_extents()  local
592 	for_each_populated_zone(zone) {  in create_mem_extents()
596 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
597 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
658 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
660 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
662 		if (!zone) {  in memory_bm_create()
666 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
687 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
689 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
690 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
709 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
713 	zone = bm->cur.zone;  in memory_bm_find_bit()
715 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
718 	zone = NULL;  in memory_bm_find_bit()
723 			zone = curr;  in memory_bm_find_bit()
728 	if (!zone)  in memory_bm_find_bit()
743 	if (zone == bm->cur.zone &&  in memory_bm_find_bit()
744 	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
747 	node      = zone->rtree;  in memory_bm_find_bit()
748 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
750 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
761 	bm->cur.zone = zone;  in memory_bm_find_bit()
763 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
767 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
846 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
856 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
857 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
859 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
887 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
892 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
913 static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)  in recycle_zone_bm_rtree()  argument
917 	list_for_each_entry(node, &zone->nodes, list)  in recycle_zone_bm_rtree()
920 	list_for_each_entry(node, &zone->leaves, list)  in recycle_zone_bm_rtree()
926 	struct mem_zone_bm_rtree *zone;  in memory_bm_recycle()  local
929 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
930 		recycle_zone_bm_rtree(zone);  in memory_bm_recycle()
1177 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1181 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1200 	struct zone *zone;  in count_free_highmem_pages()  local
1203 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1204 		if (is_highmem(zone))  in count_free_highmem_pages()
1205 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1218 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1226 	if (!page || page_zone(page) != zone)  in saveable_highmem_page()
1248 	struct zone *zone;  in count_highmem_pages()  local
1251 	for_each_populated_zone(zone) {  in count_highmem_pages()
1254 		if (!is_highmem(zone))  in count_highmem_pages()
1257 		mark_free_pages(zone);  in count_highmem_pages()
1258 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1259 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1260 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1266 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)  in saveable_highmem_page()
1282 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1290 	if (!page || page_zone(page) != zone)  in saveable_page()
1316 	struct zone *zone;  in count_data_pages()  local
1320 	for_each_populated_zone(zone) {  in count_data_pages()
1321 		if (is_highmem(zone))  in count_data_pages()
1324 		mark_free_pages(zone);  in count_data_pages()
1325 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1326 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1327 			if (saveable_page(zone, pfn))  in count_data_pages()
1365 static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1367 	return is_highmem(zone) ?  in page_is_saveable()
1368 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1400 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1412 	struct zone *zone;  in copy_data_pages()  local
1415 	for_each_populated_zone(zone) {  in copy_data_pages()
1418 		mark_free_pages(zone);  in copy_data_pages()
1419 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1420 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1421 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1699 	struct zone *zone;  in hibernate_preallocate_memory()  local
1735 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1736 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1737 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1738 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1740 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1877 	struct zone *zone;  in enough_free_mem()  local
1880 	for_each_populated_zone(zone)  in enough_free_mem()
1881 		if (!is_highmem(zone))  in enough_free_mem()
1882 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()