386 void reset_isolation_suitable(pg_data_t *pgdat)  in reset_isolation_suitable()  argument
391 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()
767 static bool too_many_isolated(pg_data_t *pgdat)  in too_many_isolated()  argument
771 	inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +  in too_many_isolated()
772 			node_page_state(pgdat, NR_INACTIVE_ANON);  in too_many_isolated()
773 	active = node_page_state(pgdat, NR_ACTIVE_FILE) +  in too_many_isolated()
774 			node_page_state(pgdat, NR_ACTIVE_ANON);  in too_many_isolated()
775 	isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +  in too_many_isolated()
776 			node_page_state(pgdat, NR_ISOLATED_ANON);  in too_many_isolated()
803 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()  local
819 	while (unlikely(too_many_isolated(pgdat))) {  in isolate_migratepages_block()
872 		    && compact_unlock_should_abort(&pgdat->lru_lock,  in isolate_migratepages_block()
946 					spin_unlock_irqrestore(&pgdat->lru_lock,  in isolate_migratepages_block()
976 			locked = compact_lock_irqsave(&pgdat->lru_lock,  in isolate_migratepages_block()
1001 		lruvec = mem_cgroup_page_lruvec(page, pgdat);  in isolate_migratepages_block()
1046 				spin_unlock_irqrestore(&pgdat->lru_lock, flags);  in isolate_migratepages_block()
1073 		spin_unlock_irqrestore(&pgdat->lru_lock, flags);  in isolate_migratepages_block()
1881 static bool kswapd_is_running(pg_data_t *pgdat)  in kswapd_is_running()  argument
1883 	return pgdat->kswapd && (pgdat->kswapd->state == TASK_RUNNING);  in kswapd_is_running()
1912 static unsigned int fragmentation_score_node(pg_data_t *pgdat)  in fragmentation_score_node()  argument
1920 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
1927 static unsigned int fragmentation_score_wmark(pg_data_t *pgdat, bool low)  in fragmentation_score_wmark()  argument
1940 static bool should_proactive_compact_node(pg_data_t *pgdat)  in should_proactive_compact_node()  argument
1944 	if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))  in should_proactive_compact_node()
1947 	wmark_high = fragmentation_score_wmark(pgdat, false);  in should_proactive_compact_node()
1948 	return fragmentation_score_node(pgdat) > wmark_high;  in should_proactive_compact_node()
1979 		pg_data_t *pgdat;  in __compact_finished()  local
1981 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
1982 		if (kswapd_is_running(pgdat))  in __compact_finished()
1986 		wmark_low = fragmentation_score_wmark(pgdat, true);  in __compact_finished()
2544 static void proactive_compact_node(pg_data_t *pgdat)  in proactive_compact_node()  argument
2558 		zone = &pgdat->node_zones[zoneid];  in proactive_compact_node()
2574 	pg_data_t *pgdat = NODE_DATA(nid);  in compact_node()  local
2588 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2665 static inline bool kcompactd_work_requested(pg_data_t *pgdat)  in kcompactd_work_requested()  argument
2667 	return pgdat->kcompactd_max_order > 0 || kthread_should_stop();  in kcompactd_work_requested()
2670 static bool kcompactd_node_suitable(pg_data_t *pgdat)  in kcompactd_node_suitable()  argument
2674 	enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;  in kcompactd_node_suitable()
2677 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
2682 		if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,  in kcompactd_node_suitable()
2690 static void kcompactd_do_work(pg_data_t *pgdat)  in kcompactd_do_work()  argument
2699 		.order = pgdat->kcompactd_max_order,  in kcompactd_do_work()
2700 		.search_order = pgdat->kcompactd_max_order,  in kcompactd_do_work()
2701 		.highest_zoneidx = pgdat->kcompactd_highest_zoneidx,  in kcompactd_do_work()
2706 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,  in kcompactd_do_work()
2713 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
2762 	if (pgdat->kcompactd_max_order <= cc.order)  in kcompactd_do_work()
2763 		pgdat->kcompactd_max_order = 0;  in kcompactd_do_work()
2764 	if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)  in kcompactd_do_work()
2765 		pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;  in kcompactd_do_work()
2768 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)  in wakeup_kcompactd()  argument
2773 	if (pgdat->kcompactd_max_order < order)  in wakeup_kcompactd()
2774 		pgdat->kcompactd_max_order = order;  in wakeup_kcompactd()
2776 	if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)  in wakeup_kcompactd()
2777 		pgdat->kcompactd_highest_zoneidx = highest_zoneidx;  in wakeup_kcompactd()
2783 	if (!wq_has_sleeper(&pgdat->kcompactd_wait))  in wakeup_kcompactd()
2786 	if (!kcompactd_node_suitable(pgdat))  in wakeup_kcompactd()
2789 	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,  in wakeup_kcompactd()
2791 	wake_up_interruptible(&pgdat->kcompactd_wait);  in wakeup_kcompactd()
2800 	pg_data_t *pgdat = (pg_data_t*)p;  in kcompactd()  local
2804 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);  in kcompactd()
2811 	pgdat->kcompactd_max_order = 0;  in kcompactd()
2812 	pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;  in kcompactd()
2817 		trace_mm_compaction_kcompactd_sleep(pgdat->node_id);  in kcompactd()
2818 		if (wait_event_freezable_timeout(pgdat->kcompactd_wait,  in kcompactd()
2819 			kcompactd_work_requested(pgdat),  in kcompactd()
2823 			kcompactd_do_work(pgdat);  in kcompactd()
2829 		if (should_proactive_compact_node(pgdat)) {  in kcompactd()
2836 			prev_score = fragmentation_score_node(pgdat);  in kcompactd()
2837 			proactive_compact_node(pgdat);  in kcompactd()
2838 			score = fragmentation_score_node(pgdat);  in kcompactd()
2857 	pg_data_t *pgdat = NODE_DATA(nid);  in kcompactd_run()  local
2860 	if (pgdat->kcompactd)  in kcompactd_run()
2863 	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);  in kcompactd_run()
2864 	if (IS_ERR(pgdat->kcompactd)) {  in kcompactd_run()
2866 		ret = PTR_ERR(pgdat->kcompactd);  in kcompactd_run()
2867 		pgdat->kcompactd = NULL;  in kcompactd_run()
2897 		pg_data_t *pgdat = NODE_DATA(nid);  in kcompactd_cpu_online()  local
2900 		mask = cpumask_of_node(pgdat->node_id);  in kcompactd_cpu_online()
2904 			set_cpus_allowed_ptr(pgdat->kcompactd, mask);  in kcompactd_cpu_online()