39 /* zero numa counters within a zone */
40 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
45 		atomic_long_set(&zone->vm_numa_stat[item], 0);  in zero_zone_numa_counters()
47 			per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item]  in zero_zone_numa_counters()
55 	struct zone *zone;  in zero_zones_numa_counters()  local
57 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
58 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
159  * Manage combined zone based / global counters
172 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
185 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
196 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
203 	 * of memory per zone. More memory means that we can defer updates for  in calculate_normal_threshold()
231 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
244  * Refresh the thresholds for each zone.
249 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
260 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
261 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
264 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
269 			per_cpu_ptr(zone->pageset, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
272 			/* Base nodestat threshold on the largest populated zone. */  in refresh_zone_stat_thresholds()
283 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
286 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
292 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
294 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
300 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
301 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
304 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
306 			per_cpu_ptr(zone->pageset, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
316 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
319 	struct per_cpu_pageset __percpu *pcp = zone->pageset;  in __mod_zone_page_state()
329 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
365  * argument instead of struct zone *. This allows the inclusion of the code
378  * However, the code must first determine the differential location in a zone
384 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
386 	struct per_cpu_pageset __percpu *pcp = zone->pageset;  in __inc_zone_state()
395 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
430 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
432 	struct per_cpu_pageset __percpu *pcp = zone->pageset;  in __dec_zone_state()
441 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
481  * mod_state() modifies the zone counter state through atomic per cpu
489 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
492 	struct per_cpu_pageset __percpu *pcp = zone->pageset;  in mod_zone_state()
497 		z = 0;  /* overflow to zone counters */  in mod_zone_state()
507 		 * for all cpus in a zone.  in mod_zone_state()
517 			/* Overflow must be added to zone counters */  in mod_zone_state()
524 		zone_page_state_add(z, zone, item);  in mod_zone_state()
527 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
530 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
616 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
622 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
630 	struct zone *zone;  in inc_zone_page_state()  local
632 	zone = page_zone(page);  in inc_zone_page_state()
634 	__inc_zone_state(zone, item);  in inc_zone_page_state()
744  * Update the zone counters for the current cpu.
753  * statistics in the remote zone struct as well as the global cachelines
762 	struct zone *zone;  in refresh_cpu_vm_stats()  local
771 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
772 		struct per_cpu_pageset __percpu *p = zone->pageset;  in refresh_cpu_vm_stats()
780 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
795 				atomic_long_add(v, &zone->vm_numa_stat[i]);  in refresh_cpu_vm_stats()
817 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
826 				drain_zone_pages(zone, this_cpu_ptr(&p->pcp));  in refresh_cpu_vm_stats()
864 	struct zone *zone;  in cpu_vm_stats_fold()  local
872 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
875 		p = per_cpu_ptr(zone->pageset, cpu);  in cpu_vm_stats_fold()
883 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
894 				atomic_long_add(v, &zone->vm_numa_stat[i]);  in cpu_vm_stats_fold()
924  * this is only called if !populated_zone(zone), which implies no other users of
927 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)  in drain_zonestat()  argument
935 			atomic_long_add(v, &zone->vm_stat[i]);  in drain_zonestat()
945 			atomic_long_add(v, &zone->vm_numa_stat[i]);  in drain_zonestat()
953 void __inc_numa_state(struct zone *zone,  in __inc_numa_state()  argument
956 	struct per_cpu_pageset __percpu *pcp = zone->pageset;  in __inc_numa_state()
963 		zone_numa_state_add(v, zone, item);  in __inc_numa_state()
976 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
993 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_state()
1035  * Calculate the number of free pages in a zone, how many contiguous
1042 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1056 		blocks = zone->free_area[order].nr_free;  in fill_contig_page_info()
1100  * Calculates external fragmentation within a zone wrt the given order.
1104 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1108 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1118 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1122 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1390 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1392 	struct zone *zone;  in walk_zones_in_node()  local
1393 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1396 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1397 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1401 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1402 		print(m, pgdat, zone);  in walk_zones_in_node()
1404 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1411 						struct zone *zone)  in frag_show_print()  argument
1415 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1417 		seq_printf(m, "%6lu ", zone->free_area[order].nr_free);  in frag_show_print()
1422  * This walks the free areas for each zone.
1432 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1437 		seq_printf(m, "Node %4d, zone %8s, type %12s ",  in pagetypeinfo_showfree_print()
1439 					zone->name,  in pagetypeinfo_showfree_print()
1447 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1465 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1467 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1491 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1495 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1496 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1507 		if (!memmap_valid_within(pfn, page, zone))  in pagetypeinfo_showblockcount_print()
1510 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1520 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1604 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1609 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1612 			return zone == compare;  in is_zone_first_populated()
1619 							struct zone *zone)  in zoneinfo_show_print()  argument
1622 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1623 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1638 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1639 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1640 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1641 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1642 		   zone->spanned_pages,  in zoneinfo_show_print()
1643 		   zone->present_pages,  in zoneinfo_show_print()
1644 		   zone_managed_pages(zone));  in zoneinfo_show_print()
1648 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1649 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1650 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1654 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1661 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1666 			   zone_numa_state_snapshot(zone, i));  in zoneinfo_show_print()
1673 		pageset = per_cpu_ptr(zone->pageset, i);  in zoneinfo_show_print()
1692 		   zone->zone_start_pfn);  in zoneinfo_show_print()
1889 	struct zone *zone;  in need_update()  local
1891 	for_each_populated_zone(zone) {  in need_update()
1892 		struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu);  in need_update()
2079 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2085 	seq_printf(m, "Node %d, zone %8s ",  in unusable_show_print()
2087 				zone->name);  in unusable_show_print()
2089 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2129 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2134 	/* Alloc on stack as interrupts are disabled for zone walk */  in extfrag_show_print()
2137 	seq_printf(m, "Node %d, zone %8s ",  in extfrag_show_print()
2139 				zone->name);  in extfrag_show_print()
2141 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()