347 static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,  in find_smallest_section_pfn()  argument
358 		if (zone != page_zone(pfn_to_page(start_pfn)))  in find_smallest_section_pfn()
368 static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,  in find_biggest_section_pfn()  argument
383 		if (zone != page_zone(pfn_to_page(pfn)))  in find_biggest_section_pfn()
392 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,  in shrink_zone_span()  argument
396 	int nid = zone_to_nid(zone);  in shrink_zone_span()
398 	if (zone->zone_start_pfn == start_pfn) {  in shrink_zone_span()
405 		pfn = find_smallest_section_pfn(nid, zone, end_pfn,  in shrink_zone_span()
406 						zone_end_pfn(zone));  in shrink_zone_span()
408 			zone->spanned_pages = zone_end_pfn(zone) - pfn;  in shrink_zone_span()
409 			zone->zone_start_pfn = pfn;  in shrink_zone_span()
411 			zone->zone_start_pfn = 0;  in shrink_zone_span()
412 			zone->spanned_pages = 0;  in shrink_zone_span()
414 	} else if (zone_end_pfn(zone) == end_pfn) {  in shrink_zone_span()
421 		pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn,  in shrink_zone_span()
424 			zone->spanned_pages = pfn - zone->zone_start_pfn + 1;  in shrink_zone_span()
426 			zone->zone_start_pfn = 0;  in shrink_zone_span()
427 			zone->spanned_pages = 0;  in shrink_zone_span()
435 	struct zone *zone;  in update_pgdat_span()  local
437 	for (zone = pgdat->node_zones;  in update_pgdat_span()
438 	     zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {  in update_pgdat_span()
439 		unsigned long end_pfn = zone_end_pfn(zone);  in update_pgdat_span()
442 		if (!zone->spanned_pages)  in update_pgdat_span()
445 			node_start_pfn = zone->zone_start_pfn;  in update_pgdat_span()
452 		if (zone->zone_start_pfn < node_start_pfn)  in update_pgdat_span()
453 			node_start_pfn = zone->zone_start_pfn;  in update_pgdat_span()
460 void __ref remove_pfn_range_from_zone(struct zone *zone,  in remove_pfn_range_from_zone()  argument
465 	struct pglist_data *pgdat = zone->zone_pgdat;  in remove_pfn_range_from_zone()
484 	if (zone_is_zone_device(zone))  in remove_pfn_range_from_zone()
487 	clear_zone_contiguous(zone);  in remove_pfn_range_from_zone()
489 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);  in remove_pfn_range_from_zone()
492 	set_zone_contiguous(zone);  in remove_pfn_range_from_zone()
620 	struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_online()  argument
622 	int nid = zone_to_nid(zone);  in node_states_check_changes_online()
629 	if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))  in node_states_check_changes_online()
642 static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,  in resize_zone_range()  argument
645 	unsigned long old_end_pfn = zone_end_pfn(zone);  in resize_zone_range()
647 	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)  in resize_zone_range()
648 		zone->zone_start_pfn = start_pfn;  in resize_zone_range()
650 	zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;  in resize_zone_range()
687 void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,  in move_pfn_range_to_zone()  argument
691 	struct pglist_data *pgdat = zone->zone_pgdat;  in move_pfn_range_to_zone()
694 	clear_zone_contiguous(zone);  in move_pfn_range_to_zone()
696 	if (zone_is_empty(zone))  in move_pfn_range_to_zone()
697 		init_currently_empty_zone(zone, start_pfn, nr_pages);  in move_pfn_range_to_zone()
698 	resize_zone_range(zone, start_pfn, nr_pages);  in move_pfn_range_to_zone()
707 	if (zone_is_zone_device(zone)) {  in move_pfn_range_to_zone()
720 	memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0,  in move_pfn_range_to_zone()
723 	set_zone_contiguous(zone);  in move_pfn_range_to_zone()
732 					    struct zone *zone)  in auto_movable_stats_account_zone()  argument
734 	if (zone_idx(zone) == ZONE_MOVABLE) {  in auto_movable_stats_account_zone()
735 		stats->movable_pages += zone->present_pages;  in auto_movable_stats_account_zone()
737 		stats->kernel_early_pages += zone->present_early_pages;  in auto_movable_stats_account_zone()
743 		stats->movable_pages += zone->cma_pages;  in auto_movable_stats_account_zone()
744 		stats->kernel_early_pages -= zone->cma_pages;  in auto_movable_stats_account_zone()
787 	struct zone *zone;  in auto_movable_can_online_movable()  local
793 		for_each_populated_zone(zone)  in auto_movable_can_online_movable()
794 			auto_movable_stats_account_zone(&stats, zone);  in auto_movable_can_online_movable()
797 			zone = pgdat->node_zones + i;  in auto_movable_can_online_movable()
798 			if (populated_zone(zone))  in auto_movable_can_online_movable()
799 				auto_movable_stats_account_zone(&stats, zone);  in auto_movable_can_online_movable()
834 static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,  in default_kernel_zone_for_pfn()
841 		struct zone *zone = &pgdat->node_zones[zid];  in default_kernel_zone_for_pfn()  local
843 		if (zone_intersects(zone, start_pfn, nr_pages))  in default_kernel_zone_for_pfn()
844 			return zone;  in default_kernel_zone_for_pfn()
900 static struct zone *auto_movable_zone_for_pfn(int nid,  in auto_movable_zone_for_pfn()
961 static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,  in default_zone_for_pfn()
964 	struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,  in default_zone_for_pfn()
966 	struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];  in default_zone_for_pfn()
985 struct zone *zone_for_pfn_range(int online_type, int nid,  in zone_for_pfn_range()
1008 	struct zone *zone = page_zone(page);  in adjust_present_page_count()  local
1009 	const bool movable = zone_idx(zone) == ZONE_MOVABLE;  in adjust_present_page_count()
1016 		zone->present_early_pages += nr_pages;  in adjust_present_page_count()
1017 	zone->present_pages += nr_pages;  in adjust_present_page_count()
1018 	zone->zone_pgdat->node_present_pages += nr_pages;  in adjust_present_page_count()
1027 			      struct zone *zone)  in mhp_init_memmap_on_memory()  argument
1036 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE);  in mhp_init_memmap_on_memory()
1073 		       struct zone *zone, struct memory_group *group)  in online_pages()  argument
1077 	const int nid = zone_to_nid(zone);  in online_pages()
1095 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);  in online_pages()
1099 	node_states_check_changes_online(nr_pages, zone, &arg);  in online_pages()
1110 	spin_lock_irqsave(&zone->lock, flags);  in online_pages()
1111 	zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages;  in online_pages()
1112 	spin_unlock_irqrestore(&zone->lock, flags);  in online_pages()
1119 	if (!populated_zone(zone)) {  in online_pages()
1121 		setup_zone_pageset(zone);  in online_pages()
1140 	shuffle_zone(zone);  in online_pages()
1159 	remove_pfn_range_from_zone(zone, pfn, nr_pages);  in online_pages()
1166 	struct zone *z;  in reset_node_present_pages()
1735 		struct zone *zone, struct memory_notify *arg)  in node_states_check_changes_offline()  argument
1737 	struct pglist_data *pgdat = zone->zone_pgdat;  in node_states_check_changes_offline()
1754 	if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)  in node_states_check_changes_offline()
1755 		arg->status_change_nid_normal = zone_to_nid(zone);  in node_states_check_changes_offline()
1769 		arg->status_change_nid = zone_to_nid(zone);  in node_states_check_changes_offline()
1791 			struct zone *zone, struct memory_group *group)  in offline_pages()  argument
1795 	const int node = zone_to_nid(zone);  in offline_pages()
1835 	if (WARN_ON_ONCE(page_zone(pfn_to_page(start_pfn)) != zone ||  in offline_pages()
1836 			 page_zone(pfn_to_page(end_pfn - 1)) != zone)) {  in offline_pages()
1846 	zone_pcp_disable(zone);  in offline_pages()
1861 	node_states_check_changes_offline(nr_pages, zone, &arg);  in offline_pages()
1920 	spin_lock_irqsave(&zone->lock, flags);  in offline_pages()
1921 	zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages;  in offline_pages()
1922 	spin_unlock_irqrestore(&zone->lock, flags);  in offline_pages()
1925 	zone_pcp_enable(zone);  in offline_pages()
1934 	if (!populated_zone(zone)) {  in offline_pages()
1935 		zone_pcp_reset(zone);  in offline_pages()
1948 	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);  in offline_pages()
1958 	zone_pcp_enable(zone);  in offline_pages()