371 	struct mem_zone_bm_rtree *zone;  member
428 static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,  in add_rtree_block()  argument
435 	block_nr = zone->blocks;  in add_rtree_block()
445 	for (i = zone->levels; i < levels_needed; i++) {  in add_rtree_block()
447 					&zone->nodes);  in add_rtree_block()
451 		node->data[0] = (unsigned long)zone->rtree;  in add_rtree_block()
452 		zone->rtree = node;  in add_rtree_block()
453 		zone->levels += 1;  in add_rtree_block()
457 	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);  in add_rtree_block()
462 	node = zone->rtree;  in add_rtree_block()
463 	dst = &zone->rtree;  in add_rtree_block()
464 	block_nr = zone->blocks;  in add_rtree_block()
465 	for (i = zone->levels; i > 0; i--) {  in add_rtree_block()
470 						&zone->nodes);  in add_rtree_block()
482 	zone->blocks += 1;  in add_rtree_block()
488 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
504 	struct mem_zone_bm_rtree *zone;  in create_zone_bm_rtree()  local
509 	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));  in create_zone_bm_rtree()
510 	if (!zone)  in create_zone_bm_rtree()
513 	INIT_LIST_HEAD(&zone->nodes);  in create_zone_bm_rtree()
514 	INIT_LIST_HEAD(&zone->leaves);  in create_zone_bm_rtree()
515 	zone->start_pfn = start;  in create_zone_bm_rtree()
516 	zone->end_pfn = end;  in create_zone_bm_rtree()
520 		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {  in create_zone_bm_rtree()
521 			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);  in create_zone_bm_rtree()
526 	return zone;  in create_zone_bm_rtree()
536 static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,  in free_zone_bm_rtree()  argument
541 	list_for_each_entry(node, &zone->nodes, list)  in free_zone_bm_rtree()
544 	list_for_each_entry(node, &zone->leaves, list)  in free_zone_bm_rtree()
550 	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,  in memory_bm_position_reset()
552 	bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in memory_bm_position_reset()
589 	struct zone *zone;  in create_mem_extents()  local
593 	for_each_populated_zone(zone) {  in create_mem_extents()
597 		zone_start = zone->zone_start_pfn;  in create_mem_extents()
598 		zone_end = zone_end_pfn(zone);  in create_mem_extents()
659 		struct mem_zone_bm_rtree *zone;  in memory_bm_create()  local
661 		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,  in memory_bm_create()
663 		if (!zone) {  in memory_bm_create()
667 		list_add_tail(&zone->list, &bm->zones);  in memory_bm_create()
688 	struct mem_zone_bm_rtree *zone;  in memory_bm_free()  local
690 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_free()
691 		free_zone_bm_rtree(zone, clear_nosave_free);  in memory_bm_free()
710 	struct mem_zone_bm_rtree *curr, *zone;  in memory_bm_find_bit()  local
714 	zone = bm->cur.zone;  in memory_bm_find_bit()
716 	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)  in memory_bm_find_bit()
719 	zone = NULL;  in memory_bm_find_bit()
724 			zone = curr;  in memory_bm_find_bit()
729 	if (!zone)  in memory_bm_find_bit()
738 	if (((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)  in memory_bm_find_bit()
741 	node      = zone->rtree;  in memory_bm_find_bit()
742 	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;  in memory_bm_find_bit()
744 	for (i = zone->levels; i > 0; i--) {  in memory_bm_find_bit()
755 	bm->cur.zone = zone;  in memory_bm_find_bit()
757 	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;  in memory_bm_find_bit()
761 	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;  in memory_bm_find_bit()
840 	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {  in rtree_next_node()
850 	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {  in rtree_next_node()
851 		bm->cur.zone = list_entry(bm->cur.zone->list.next,  in rtree_next_node()
853 		bm->cur.node = list_entry(bm->cur.zone->leaves.next,  in rtree_next_node()
881 		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;  in memory_bm_next_pfn()
886 			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;  in memory_bm_next_pfn()
907 static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)  in recycle_zone_bm_rtree()  argument
911 	list_for_each_entry(node, &zone->nodes, list)  in recycle_zone_bm_rtree()
914 	list_for_each_entry(node, &zone->leaves, list)  in recycle_zone_bm_rtree()
920 	struct mem_zone_bm_rtree *zone;  in memory_bm_recycle()  local
923 	list_for_each_entry(zone, &bm->zones, list)  in memory_bm_recycle()
924 		recycle_zone_bm_rtree(zone);  in memory_bm_recycle()
1171 unsigned int snapshot_additional_pages(struct zone *zone)  in snapshot_additional_pages()  argument
1175 	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);  in snapshot_additional_pages()
1194 	struct zone *zone;  in count_free_highmem_pages()  local
1197 	for_each_populated_zone(zone)  in count_free_highmem_pages()
1198 		if (is_highmem(zone))  in count_free_highmem_pages()
1199 			cnt += zone_page_state(zone, NR_FREE_PAGES);  in count_free_highmem_pages()
1212 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)  in saveable_highmem_page()  argument
1220 	if (!page || page_zone(page) != zone)  in saveable_highmem_page()
1242 	struct zone *zone;  in count_highmem_pages()  local
1245 	for_each_populated_zone(zone) {  in count_highmem_pages()
1248 		if (!is_highmem(zone))  in count_highmem_pages()
1251 		mark_free_pages(zone);  in count_highmem_pages()
1252 		max_zone_pfn = zone_end_pfn(zone);  in count_highmem_pages()
1253 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_highmem_pages()
1254 			if (saveable_highmem_page(zone, pfn))  in count_highmem_pages()
1260 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)  in saveable_highmem_page()
1276 static struct page *saveable_page(struct zone *zone, unsigned long pfn)  in saveable_page()  argument
1284 	if (!page || page_zone(page) != zone)  in saveable_page()
1310 	struct zone *zone;  in count_data_pages()  local
1314 	for_each_populated_zone(zone) {  in count_data_pages()
1315 		if (is_highmem(zone))  in count_data_pages()
1318 		mark_free_pages(zone);  in count_data_pages()
1319 		max_zone_pfn = zone_end_pfn(zone);  in count_data_pages()
1320 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in count_data_pages()
1321 			if (saveable_page(zone, pfn))  in count_data_pages()
1359 static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)  in page_is_saveable()  argument
1361 	return is_highmem(zone) ?  in page_is_saveable()
1362 		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);  in page_is_saveable()
1394 #define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)  argument
1406 	struct zone *zone;  in copy_data_pages()  local
1409 	for_each_populated_zone(zone) {  in copy_data_pages()
1412 		mark_free_pages(zone);  in copy_data_pages()
1413 		max_zone_pfn = zone_end_pfn(zone);  in copy_data_pages()
1414 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in copy_data_pages()
1415 			if (page_is_saveable(zone, pfn))  in copy_data_pages()
1695 	struct zone *zone;  in hibernate_preallocate_memory()  local
1727 	for_each_populated_zone(zone) {  in hibernate_preallocate_memory()
1728 		size += snapshot_additional_pages(zone);  in hibernate_preallocate_memory()
1729 		if (is_highmem(zone))  in hibernate_preallocate_memory()
1730 			highmem += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1732 			count += zone_page_state(zone, NR_FREE_PAGES);  in hibernate_preallocate_memory()
1870 	struct zone *zone;  in enough_free_mem()  local
1873 	for_each_populated_zone(zone)  in enough_free_mem()
1874 		if (!is_highmem(zone))  in enough_free_mem()
1875 			free += zone_page_state(zone, NR_FREE_PAGES);  in enough_free_mem()