204 static inline bool isolation_suitable(struct compact_control *cc,  in isolation_suitable()  argument
207 	if (cc->ignore_skip_hint)  in isolation_suitable()
291 static void update_pageblock_skip(struct compact_control *cc,  in update_pageblock_skip()  argument
295 	struct zone *zone = cc->zone;  in update_pageblock_skip()
298 	if (cc->no_set_skip_hint)  in update_pageblock_skip()
315 		if (cc->mode != MIGRATE_ASYNC &&  in update_pageblock_skip()
324 static inline bool isolation_suitable(struct compact_control *cc,  in isolation_suitable()  argument
335 static inline void update_pageblock_skip(struct compact_control *cc,  in update_pageblock_skip()  argument
351 						struct compact_control *cc)  in compact_trylock_irqsave()  argument
353 	if (cc->mode == MIGRATE_ASYNC) {  in compact_trylock_irqsave()
355 			cc->contended = true;  in compact_trylock_irqsave()
381 		unsigned long flags, bool *locked, struct compact_control *cc)  in compact_unlock_should_abort()  argument
389 		cc->contended = true;  in compact_unlock_should_abort()
394 		if (cc->mode == MIGRATE_ASYNC) {  in compact_unlock_should_abort()
395 			cc->contended = true;  in compact_unlock_should_abort()
413 static inline bool compact_should_abort(struct compact_control *cc)  in compact_should_abort()  argument
417 		if (cc->mode == MIGRATE_ASYNC) {  in compact_should_abort()
418 			cc->contended = true;  in compact_should_abort()
433 static unsigned long isolate_freepages_block(struct compact_control *cc,  in isolate_freepages_block()  argument
459 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
460 								&locked, cc))  in isolate_freepages_block()
505 			locked = compact_trylock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
506 								&flags, cc);  in isolate_freepages_block()
523 		cc->nr_freepages += isolated;  in isolate_freepages_block()
526 		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {  in isolate_freepages_block()
544 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
569 		update_pageblock_skip(cc, valid_page, total_isolated, false);  in isolate_freepages_block()
571 	cc->total_free_scanned += nr_scanned;  in isolate_freepages_block()
592 isolate_freepages_range(struct compact_control *cc,  in isolate_freepages_range()  argument
600 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
601 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
624 					block_end_pfn, cc->zone))  in isolate_freepages_range()
627 		isolated = isolate_freepages_block(cc, &isolate_start_pfn,  in isolate_freepages_range()
692 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,  in isolate_migratepages_block()  argument
695 	struct zone *zone = cc->zone;  in isolate_migratepages_block()
712 		if (cc->mode == MIGRATE_ASYNC)  in isolate_migratepages_block()
721 	if (compact_should_abort(cc))  in isolate_migratepages_block()
724 	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {  in isolate_migratepages_block()
726 		next_skip_pfn = block_end_pfn(low_pfn, cc->order);  in isolate_migratepages_block()
751 			next_skip_pfn = block_end_pfn(low_pfn, cc->order);  in isolate_migratepages_block()
761 								&locked, cc))  in isolate_migratepages_block()
845 		if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))  in isolate_migratepages_block()
851 								&flags, cc);  in isolate_migratepages_block()
884 		list_add(&page->lru, &cc->migratepages);  in isolate_migratepages_block()
885 		cc->nr_migratepages++;  in isolate_migratepages_block()
894 		if (!cc->last_migrated_pfn)  in isolate_migratepages_block()
895 			cc->last_migrated_pfn = low_pfn;  in isolate_migratepages_block()
898 		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) {  in isolate_migratepages_block()
918 			putback_movable_pages(&cc->migratepages);  in isolate_migratepages_block()
919 			cc->nr_migratepages = 0;  in isolate_migratepages_block()
920 			cc->last_migrated_pfn = 0;  in isolate_migratepages_block()
930 			next_skip_pfn += 1UL << cc->order;  in isolate_migratepages_block()
949 		update_pageblock_skip(cc, valid_page, nr_isolated, true);  in isolate_migratepages_block()
954 	cc->total_migrate_scanned += nr_scanned;  in isolate_migratepages_block()
972 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,  in isolate_migratepages_range()  argument
980 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
981 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
991 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
994 		pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,  in isolate_migratepages_range()
1000 		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)  in isolate_migratepages_range()
1010 static bool suitable_migration_source(struct compact_control *cc,  in suitable_migration_source()  argument
1015 	if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)  in suitable_migration_source()
1020 	if (cc->migratetype == MIGRATE_MOVABLE)  in suitable_migration_source()
1023 		return block_mt == cc->migratetype;  in suitable_migration_source()
1027 static bool suitable_migration_target(struct compact_control *cc,  in suitable_migration_target()  argument
1041 	if (cc->ignore_block_suitable)  in suitable_migration_target()
1056 static inline bool compact_scanners_met(struct compact_control *cc)  in compact_scanners_met()  argument
1058 	return (cc->free_pfn >> pageblock_order)  in compact_scanners_met()
1059 		<= (cc->migrate_pfn >> pageblock_order);  in compact_scanners_met()
1066 static void isolate_freepages(struct compact_control *cc)  in isolate_freepages()  argument
1068 	struct zone *zone = cc->zone;  in isolate_freepages()
1074 	struct list_head *freelist = &cc->freepages;  in isolate_freepages()
1087 	isolate_start_pfn = cc->free_pfn;  in isolate_freepages()
1088 	block_start_pfn = pageblock_start_pfn(cc->free_pfn);  in isolate_freepages()
1091 	low_pfn = pageblock_end_pfn(cc->migrate_pfn);  in isolate_freepages()
1108 						&& compact_should_abort(cc))  in isolate_freepages()
1117 		if (!suitable_migration_target(cc, page))  in isolate_freepages()
1121 		if (!isolation_suitable(cc, page))  in isolate_freepages()
1125 		isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn,  in isolate_freepages()
1132 		if ((cc->nr_freepages >= cc->nr_migratepages)  in isolate_freepages()
1133 							|| cc->contended) {  in isolate_freepages()
1161 	cc->free_pfn = isolate_start_pfn;  in isolate_freepages()
1171 	struct compact_control *cc = (struct compact_control *)data;  in compaction_alloc()  local
1178 	if (list_empty(&cc->freepages)) {  in compaction_alloc()
1179 		if (!cc->contended)  in compaction_alloc()
1180 			isolate_freepages(cc);  in compaction_alloc()
1182 		if (list_empty(&cc->freepages))  in compaction_alloc()
1186 	freepage = list_entry(cc->freepages.next, struct page, lru);  in compaction_alloc()
1188 	cc->nr_freepages--;  in compaction_alloc()
1200 	struct compact_control *cc = (struct compact_control *)data;  in compaction_free()  local
1202 	list_add(&page->lru, &cc->freepages);  in compaction_free()
1203 	cc->nr_freepages++;  in compaction_free()
1225 					struct compact_control *cc)  in isolate_migratepages()  argument
1233 		(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);  in isolate_migratepages()
1239 	low_pfn = cc->migrate_pfn;  in isolate_migratepages()
1251 	for (; block_end_pfn <= cc->free_pfn;  in isolate_migratepages()
1262 						&& compact_should_abort(cc))  in isolate_migratepages()
1271 		if (!isolation_suitable(cc, page))  in isolate_migratepages()
1279 		if (!suitable_migration_source(cc, page))  in isolate_migratepages()
1283 		low_pfn = isolate_migratepages_block(cc, low_pfn,  in isolate_migratepages()
1286 		if (!low_pfn || cc->contended)  in isolate_migratepages()
1298 	cc->migrate_pfn = low_pfn;  in isolate_migratepages()
1300 	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;  in isolate_migratepages()
1313 						struct compact_control *cc)  in __compact_finished()  argument
1316 	const int migratetype = cc->migratetype;  in __compact_finished()
1318 	if (cc->contended || fatal_signal_pending(current))  in __compact_finished()
1322 	if (compact_scanners_met(cc)) {  in __compact_finished()
1332 		if (cc->direct_compaction)  in __compact_finished()
1335 		if (cc->whole_zone)  in __compact_finished()
1341 	if (is_via_compact_memory(cc->order))  in __compact_finished()
1344 	if (cc->finishing_block) {  in __compact_finished()
1349 		if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))  in __compact_finished()
1350 			cc->finishing_block = false;  in __compact_finished()
1356 	for (order = cc->order; order < MAX_ORDER; order++) {  in __compact_finished()
1389 			if (cc->mode == MIGRATE_ASYNC ||  in __compact_finished()
1390 					IS_ALIGNED(cc->migrate_pfn,  in __compact_finished()
1395 			cc->finishing_block = true;  in __compact_finished()
1404 			struct compact_control *cc)  in compact_finished()  argument
1408 	ret = __compact_finished(zone, cc);  in compact_finished()
1409 	trace_mm_compaction_finished(zone, cc->order, ret);  in compact_finished()
1536 static enum compact_result compact_zone(struct zone *zone, struct compact_control *cc)  in compact_zone()  argument
1541 	const bool sync = cc->mode != MIGRATE_ASYNC;  in compact_zone()
1543 	cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);  in compact_zone()
1544 	ret = compaction_suitable(zone, cc->order, cc->alloc_flags,  in compact_zone()
1545 							cc->classzone_idx);  in compact_zone()
1557 	if (compaction_restarting(zone, cc->order))  in compact_zone()
1566 	if (cc->whole_zone) {  in compact_zone()
1567 		cc->migrate_pfn = start_pfn;  in compact_zone()
1568 		cc->free_pfn = pageblock_start_pfn(end_pfn - 1);  in compact_zone()
1570 		cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];  in compact_zone()
1571 		cc->free_pfn = zone->compact_cached_free_pfn;  in compact_zone()
1572 		if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {  in compact_zone()
1573 			cc->free_pfn = pageblock_start_pfn(end_pfn - 1);  in compact_zone()
1574 			zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
1576 		if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {  in compact_zone()
1577 			cc->migrate_pfn = start_pfn;  in compact_zone()
1578 			zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
1579 			zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
1582 		if (cc->migrate_pfn == start_pfn)  in compact_zone()
1583 			cc->whole_zone = true;  in compact_zone()
1586 	cc->last_migrated_pfn = 0;  in compact_zone()
1588 	trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,  in compact_zone()
1589 				cc->free_pfn, end_pfn, sync);  in compact_zone()
1593 	while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) {  in compact_zone()
1596 		switch (isolate_migratepages(zone, cc)) {  in compact_zone()
1599 			putback_movable_pages(&cc->migratepages);  in compact_zone()
1600 			cc->nr_migratepages = 0;  in compact_zone()
1613 		err = migrate_pages(&cc->migratepages, compaction_alloc,  in compact_zone()
1614 				compaction_free, (unsigned long)cc, cc->mode,  in compact_zone()
1617 		trace_mm_compaction_migratepages(cc->nr_migratepages, err,  in compact_zone()
1618 							&cc->migratepages);  in compact_zone()
1621 		cc->nr_migratepages = 0;  in compact_zone()
1623 			putback_movable_pages(&cc->migratepages);  in compact_zone()
1628 			if (err == -ENOMEM && !compact_scanners_met(cc)) {  in compact_zone()
1636 			if (cc->direct_compaction &&  in compact_zone()
1637 						(cc->mode == MIGRATE_ASYNC)) {  in compact_zone()
1638 				cc->migrate_pfn = block_end_pfn(  in compact_zone()
1639 						cc->migrate_pfn - 1, cc->order);  in compact_zone()
1641 				cc->last_migrated_pfn = 0;  in compact_zone()
1654 		if (cc->order > 0 && cc->last_migrated_pfn) {  in compact_zone()
1657 				block_start_pfn(cc->migrate_pfn, cc->order);  in compact_zone()
1659 			if (cc->last_migrated_pfn < current_block_start) {  in compact_zone()
1665 				cc->last_migrated_pfn = 0;  in compact_zone()
1676 	if (cc->nr_freepages > 0) {  in compact_zone()
1677 		unsigned long free_pfn = release_freepages(&cc->freepages);  in compact_zone()
1679 		cc->nr_freepages = 0;  in compact_zone()
1691 	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);  in compact_zone()
1692 	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);  in compact_zone()
1694 	trace_mm_compaction_end(start_pfn, cc->migrate_pfn,  in compact_zone()
1695 				cc->free_pfn, end_pfn, sync, ret);  in compact_zone()
1705 	struct compact_control cc = {  in compact_zone_order()  local
1722 	INIT_LIST_HEAD(&cc.freepages);  in compact_zone_order()
1723 	INIT_LIST_HEAD(&cc.migratepages);  in compact_zone_order()
1725 	ret = compact_zone(zone, &cc);  in compact_zone_order()
1727 	VM_BUG_ON(!list_empty(&cc.freepages));  in compact_zone_order()
1728 	VM_BUG_ON(!list_empty(&cc.migratepages));  in compact_zone_order()
1820 	struct compact_control cc = {  in compact_node()  local
1837 		cc.nr_freepages = 0;  in compact_node()
1838 		cc.nr_migratepages = 0;  in compact_node()
1839 		cc.zone = zone;  in compact_node()
1840 		INIT_LIST_HEAD(&cc.freepages);  in compact_node()
1841 		INIT_LIST_HEAD(&cc.migratepages);  in compact_node()
1843 		compact_zone(zone, &cc);  in compact_node()
1845 		VM_BUG_ON(!list_empty(&cc.freepages));  in compact_node()
1846 		VM_BUG_ON(!list_empty(&cc.migratepages));  in compact_node()
1948 	struct compact_control cc = {  in kcompactd_do_work()  local
1957 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,  in kcompactd_do_work()
1958 							cc.classzone_idx);  in kcompactd_do_work()
1961 	for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {  in kcompactd_do_work()
1968 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
1971 		if (compaction_suitable(zone, cc.order, 0, zoneid) !=  in kcompactd_do_work()
1975 		cc.nr_freepages = 0;  in kcompactd_do_work()
1976 		cc.nr_migratepages = 0;  in kcompactd_do_work()
1977 		cc.total_migrate_scanned = 0;  in kcompactd_do_work()
1978 		cc.total_free_scanned = 0;  in kcompactd_do_work()
1979 		cc.zone = zone;  in kcompactd_do_work()
1980 		INIT_LIST_HEAD(&cc.freepages);  in kcompactd_do_work()
1981 		INIT_LIST_HEAD(&cc.migratepages);  in kcompactd_do_work()
1985 		status = compact_zone(zone, &cc);  in kcompactd_do_work()
1988 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
2002 			defer_compaction(zone, cc.order);  in kcompactd_do_work()
2006 				     cc.total_migrate_scanned);  in kcompactd_do_work()
2008 				     cc.total_free_scanned);  in kcompactd_do_work()
2010 		VM_BUG_ON(!list_empty(&cc.freepages));  in kcompactd_do_work()
2011 		VM_BUG_ON(!list_empty(&cc.migratepages));  in kcompactd_do_work()
2019 	if (pgdat->kcompactd_max_order <= cc.order)  in kcompactd_do_work()
2021 	if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)  in kcompactd_do_work()