154 static void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
156 	zone->compact_considered = 0;  in defer_compaction()
157 	zone->compact_defer_shift++;  in defer_compaction()
159 	if (order < zone->compact_order_failed)  in defer_compaction()
160 		zone->compact_order_failed = order;  in defer_compaction()
162 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
163 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
165 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
169 static bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
171 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
173 	if (order < zone->compact_order_failed)  in compaction_deferred()
177 	if (++zone->compact_considered >= defer_limit) {  in compaction_deferred()
178 		zone->compact_considered = defer_limit;  in compaction_deferred()
182 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
192 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
196 		zone->compact_considered = 0;  in compaction_defer_reset()
197 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
199 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
200 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
202 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
206 static bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
208 	if (order < zone->compact_order_failed)  in compaction_restarting()
211 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
212 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
225 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
227 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
228 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
229 	zone->compact_cached_free_pfn =  in reset_cached_positions()
230 				pageblock_start_pfn(zone_end_pfn(zone) - 1);  in reset_cached_positions()
252 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,  in __reset_isolation_pfn()  argument
262 	if (zone != page_zone(page))  in __reset_isolation_pfn()
284 	block_pfn = max(block_pfn, zone->zone_start_pfn);  in __reset_isolation_pfn()
293 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);  in __reset_isolation_pfn()
325 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
327 	unsigned long migrate_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
328 	unsigned long free_pfn = zone_end_pfn(zone) - 1;  in __reset_isolation_suitable()
334 	if (!zone->compact_blockskip_flush)  in __reset_isolation_suitable()
337 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
350 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&  in __reset_isolation_suitable()
354 			zone->compact_init_migrate_pfn = reset_migrate;  in __reset_isolation_suitable()
355 			zone->compact_cached_migrate_pfn[0] = reset_migrate;  in __reset_isolation_suitable()
356 			zone->compact_cached_migrate_pfn[1] = reset_migrate;  in __reset_isolation_suitable()
360 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&  in __reset_isolation_suitable()
364 			zone->compact_init_free_pfn = reset_free;  in __reset_isolation_suitable()
365 			zone->compact_cached_free_pfn = reset_free;  in __reset_isolation_suitable()
371 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;  in __reset_isolation_suitable()
372 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;  in __reset_isolation_suitable()
373 		zone->compact_cached_free_pfn = free_pfn;  in __reset_isolation_suitable()
382 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
383 		if (!populated_zone(zone))  in reset_isolation_suitable()
387 		if (zone->compact_blockskip_flush)  in reset_isolation_suitable()
388 			__reset_isolation_suitable(zone);  in reset_isolation_suitable()
417 	struct zone *zone = cc->zone;  in update_cached_migrate()  local
425 	if (pfn > zone->compact_cached_migrate_pfn[0])  in update_cached_migrate()
426 		zone->compact_cached_migrate_pfn[0] = pfn;  in update_cached_migrate()
428 	    pfn > zone->compact_cached_migrate_pfn[1])  in update_cached_migrate()
429 		zone->compact_cached_migrate_pfn[1] = pfn;  in update_cached_migrate()
439 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
450 	if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
451 		zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
572 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
599 			locked = compact_lock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
637 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
689 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
690 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
713 					block_end_pfn, cc->zone))  in isolate_freepages_range()
789 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()
1220 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
1221 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
1231 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
1361 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);  in fast_isolate_around()
1362 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));  in fast_isolate_around()
1364 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);  in fast_isolate_around()
1423 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {  in fast_isolate_freepages()
1448 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1458 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_isolate_freepages()
1469 					      cc->zone->zone_start_pfn);  in fast_isolate_freepages()
1516 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_isolate_freepages()
1541 						    zone_end_pfn(cc->zone)),  in fast_isolate_freepages()
1542 						cc->zone);  in fast_isolate_freepages()
1549 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {  in fast_isolate_freepages()
1551 		cc->zone->compact_cached_free_pfn = highest;  in fast_isolate_freepages()
1569 	struct zone *zone = cc->zone;  in isolate_freepages()  local
1597 						zone_end_pfn(zone));  in isolate_freepages()
1620 									zone);  in isolate_freepages()
1778 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))  in fast_find_migrateblock()
1805 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1812 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
1820 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_find_migrateblock()
1846 				if (pfn < cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1847 					pfn = cc->zone->zone_start_pfn;  in fast_find_migrateblock()
1853 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_find_migrateblock()
1892 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages()
1893 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages()
1924 						block_end_pfn, cc->zone);  in isolate_migratepages()
1998 static unsigned int fragmentation_score_zone(struct zone *zone)  in fragmentation_score_zone()  argument
2000 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);  in fragmentation_score_zone()
2013 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)  in fragmentation_score_zone_weighted()  argument
2017 	score = zone->present_pages * fragmentation_score_zone(zone);  in fragmentation_score_zone_weighted()
2018 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);  in fragmentation_score_zone_weighted()
2034 		struct zone *zone;  in fragmentation_score_node()  local
2036 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
2037 		score += fragmentation_score_zone_weighted(zone);  in fragmentation_score_node()
2076 		reset_cached_positions(cc->zone);  in __compact_finished()
2085 			cc->zone->compact_blockskip_flush = true;  in __compact_finished()
2097 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
2101 		score = fragmentation_score_zone(cc->zone);  in __compact_finished()
2127 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2169 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2176 static enum compact_result __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2186 	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);  in __compaction_suitable()
2191 	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,  in __compaction_suitable()
2210 				low_wmark_pages(zone) : min_wmark_pages(zone);  in __compaction_suitable()
2212 	if (!__zone_watermark_ok(zone, 0, watermark, highest_zoneidx,  in __compaction_suitable()
2226 enum compact_result compaction_suitable(struct zone *zone, int order,  in compaction_suitable()  argument
2233 	ret = __compaction_suitable(zone, order, alloc_flags, highest_zoneidx,  in compaction_suitable()
2234 				    zone_page_state(zone, NR_FREE_PAGES));  in compaction_suitable()
2252 		fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2257 	trace_mm_compaction_suitable(zone, order, ret);  in compaction_suitable()
2267 	struct zone *zone;  in compaction_zonelist_suitable()  local
2274 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in compaction_zonelist_suitable()
2285 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2286 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in compaction_zonelist_suitable()
2287 		compact_result = __compaction_suitable(zone, order, alloc_flags,  in compaction_zonelist_suitable()
2300 	unsigned long start_pfn = cc->zone->zone_start_pfn;  in compact_zone()
2301 	unsigned long end_pfn = zone_end_pfn(cc->zone);  in compact_zone()
2319 	ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,  in compact_zone()
2332 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2333 		__reset_isolation_suitable(cc->zone);  in compact_zone()
2346 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];  in compact_zone()
2347 		cc->free_pfn = cc->zone->compact_cached_free_pfn;  in compact_zone()
2350 			cc->zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
2354 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
2355 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
2358 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)  in compact_zone()
2373 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];  in compact_zone()
2406 				cc->zone->compact_cached_migrate_pfn[1] =  in compact_zone()
2407 					cc->zone->compact_cached_migrate_pfn[0];  in compact_zone()
2465 				lru_add_drain_cpu_zone(cc->zone);  in compact_zone()
2494 		if (free_pfn > cc->zone->compact_cached_free_pfn)  in compact_zone()
2495 			cc->zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
2506 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2516 		.zone = zone,  in compact_zone_order()
2582 	struct zone *zone;  in try_to_compact_pages()  local
2595 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in try_to_compact_pages()
2600 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2605 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2617 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2629 			defer_compaction(zone, order);  in try_to_compact_pages()
2656 	struct zone *zone;  in proactive_compact_node()  local
2667 		zone = &pgdat->node_zones[zoneid];  in proactive_compact_node()
2668 		if (!populated_zone(zone))  in proactive_compact_node()
2671 		cc.zone = zone;  in proactive_compact_node()
2685 	struct zone *zone;  in compact_node()  local
2697 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2698 		if (!populated_zone(zone))  in compact_node()
2701 		cc.zone = zone;  in compact_node()
2804 	struct zone *zone;  in kcompactd_node_suitable()  local
2808 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
2810 		if (!populated_zone(zone))  in kcompactd_node_suitable()
2813 		if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,  in kcompactd_node_suitable()
2828 	struct zone *zone;  in kcompactd_do_work()  local
2844 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
2845 		if (!populated_zone(zone))  in kcompactd_do_work()
2848 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
2851 		if (compaction_suitable(zone, cc.order, 0, zoneid) !=  in kcompactd_do_work()
2858 		cc.zone = zone;  in kcompactd_do_work()
2862 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
2870 			drain_all_pages(zone);  in kcompactd_do_work()
2876 			defer_compaction(zone, cc.order);  in kcompactd_do_work()