1 // SPDX-License-Identifier: GPL-2.0-only
50 #include <linux/backing-dev.h>
51 #include <linux/fault-inject.h>
52 #include <linux/page-isolation.h>
81 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
90  * reporting it and marking it "reported" -  it only skips notifying
99  * page shuffling (relevant code - e.g., memory onlining - is expected to
102  * Note: No code should rely on this flag for correctness - it's purely
109 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
131 /* work_structs for global per-cpu drains */
191 		pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n");  in early_init_on_alloc()
209 		pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n");  in early_init_on_free()
224  * other index - this ensures that it will be put on the correct CMA freelist.
228 	return page->index;  in get_pcppage_migratetype()
233 	page->index = migratetype;  in set_pcppage_migratetype()
283  *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
284  *	1G machine -> (16M dma, 784M normal, 224M high)
289  * TBD: should special case ZONE_DMA32 machines here - in those we normally
348 int user_min_free_kbytes = -1;
355  * many cases very high-order allocations like THP are likely to be
356  * unsupported and the premature reclaim offsets the advantage of long-term
393  * During boot we initialize deferred pages on-demand, as needed, but once
409  * on-demand allocation and then freed again before the deferred pages
423 	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)  in early_page_uninitialised()
447 	/* Always populate low zones for address-constrained allocations */  in defer_init()
457 	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {  in defer_init()
458 		NODE_DATA(nid)->first_deferred_pfn = pfn;  in defer_init()
484 	return page_zone(page)->pageblock_flags;  in get_pageblock_bitmap()
491 	pfn &= (PAGES_PER_SECTION-1);  in pfn_to_bitidx()
493 	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);  in pfn_to_bitidx()
499 …* get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block o…
518 	bitidx &= (BITS_PER_LONG-1);  in __get_pfnblock_flags_mask()
536 …* set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pag…
556 	bitidx &= (BITS_PER_LONG-1);  in set_pfnblock_flags_mask()
592 		start_pfn = zone->zone_start_pfn;  in page_outside_zone_boundaries()
593 		sp = zone->spanned_pages;  in page_outside_zone_boundaries()
599 		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",  in page_outside_zone_boundaries()
600 			pfn, zone_to_nid(zone), zone->name,  in page_outside_zone_boundaries()
661 		current->comm, page_to_pfn(page));  in bad_page()
674  * Higher-order pages are called "compound pages".  They are structured thusly:
679  * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
681  * The first tail page's ->compound_dtor holds the offset in array of compound
684  * The first tail page's ->compound_order holds the order of allocation.
685  * This usage means that zero-order pages may not be compound.
703 		p->mapping = TAIL_MAPPING;  in prep_compound_page()
709 	atomic_set(compound_mapcount_ptr(page), -1);  in prep_compound_page()
768 	INIT_LIST_HEAD(&page->lru);  in set_page_guard()
771 	__mod_zone_freepage_state(zone, -(1 << order), migratetype);  in set_page_guard()
810  * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
838 	struct capture_control *capc = current->capture_control;  in task_capc()
841 		!(current->flags & PF_KTHREAD) &&  in task_capc()
842 		!capc->page &&  in task_capc()
843 		capc->cc->zone == zone ? capc : NULL;  in task_capc()
850 	if (!capc || order != capc->cc->order)  in compaction_capture()
861 	 * and vice-versa but no more than normal fallback logic which can  in compaction_capture()
862 	 * have trouble finding a high-order free page.  in compaction_capture()
867 	capc->page = page;  in compaction_capture()
889 	struct free_area *area = &zone->free_area[order];  in add_to_free_list()
891 	list_add(&page->lru, &area->free_list[migratetype]);  in add_to_free_list()
892 	area->nr_free++;  in add_to_free_list()
899 	struct free_area *area = &zone->free_area[order];  in add_to_free_list_tail()
901 	list_add_tail(&page->lru, &area->free_list[migratetype]);  in add_to_free_list_tail()
902 	area->nr_free++;  in add_to_free_list_tail()
907  * of the list - so the moved pages won't immediately be considered for
913 	struct free_area *area = &zone->free_area[order];  in move_to_free_list()
915 	list_move_tail(&page->lru, &area->free_list[migratetype]);  in move_to_free_list()
925 	list_del(&page->lru);  in del_page_from_free_list()
928 	zone->free_area[order].nr_free--;  in del_page_from_free_list()
933  * of the next-highest order is free. If it is, it's possible
946 	if (order >= MAX_ORDER - 2)  in buddy_merge_likely()
953 	higher_page = page + (combined_pfn - pfn);  in buddy_merge_likely()
955 	higher_buddy = higher_page + (buddy_pfn - combined_pfn);  in buddy_merge_likely()
964  * The concept of a buddy system is to maintain direct-mapped table
982  * -- nyc
1000 	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);  in __free_one_page()
1002 	VM_BUG_ON(migratetype == -1);  in __free_one_page()
1006 	VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);  in __free_one_page()
1010 	while (order < max_order - 1) {  in __free_one_page()
1012 			__mod_zone_freepage_state(zone, -(1 << order),  in __free_one_page()
1017 		buddy = page + (buddy_pfn - pfn);  in __free_one_page()
1032 		page = page + (combined_pfn - pfn);  in __free_one_page()
1043 		 * low-order merging.  in __free_one_page()
1049 			buddy = page + (buddy_pfn - pfn);  in __free_one_page()
1089 	if (unlikely(atomic_read(&page->_mapcount) != -1))  in page_expected_state()
1092 	if (unlikely((unsigned long)page->mapping |  in page_expected_state()
1095 			(unsigned long)page->mem_cgroup |  in page_expected_state()
1097 			(page->flags & check_flags)))  in page_expected_state()
1107 	if (unlikely(atomic_read(&page->_mapcount) != -1))  in page_bad_reason()
1109 	if (unlikely(page->mapping != NULL))  in page_bad_reason()
1110 		bad_reason = "non-NULL mapping";  in page_bad_reason()
1113 	if (unlikely(page->flags & flags)) {  in page_bad_reason()
1120 	if (unlikely(page->mem_cgroup))  in page_bad_reason()
1147 	 * We rely page->lru.next never has bit 0 set, unless the page  in free_tail_pages_check()
1148 	 * is PageTail(). Let's make sure that's true even for poisoned ->lru.  in free_tail_pages_check()
1156 	switch (page - head_page) {  in free_tail_pages_check()
1158 		/* the first tail page: ->mapping may be compound_mapcount() */  in free_tail_pages_check()
1166 		 * the second tail page: ->mapping is  in free_tail_pages_check()
1167 		 * deferred_list.next -- ignore value.  in free_tail_pages_check()
1171 		if (page->mapping != TAIL_MAPPING) {  in free_tail_pages_check()
1172 			bad_page(page, "corrupted mapping in tail page");  in free_tail_pages_check()
1187 	page->mapping = NULL;  in free_tail_pages_check()
1225 	 * avoid checking PageCompound for order-0 pages.  in free_pages_prepare()
1242 			(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;  in free_pages_prepare()
1246 		page->mapping = NULL;  in free_pages_prepare()
1255 	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;  in free_pages_prepare()
1285  * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1303  * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1326 	struct page *buddy = page + (buddy_pfn - pfn);  in prefetch_buddy()
1356 	count = min(pcp->count, count);  in free_pcppages_bulk()
1361 		 * Remove pages from lists in a round-robin fashion. A  in free_pcppages_bulk()
1371 			list = &pcp->lists[migratetype];  in free_pcppages_bulk()
1374 		/* This is the only non-empty list. Free them all. */  in free_pcppages_bulk()
1381 			list_del(&page->lru);  in free_pcppages_bulk()
1382 			pcp->count--;  in free_pcppages_bulk()
1387 			list_add_tail(&page->lru, &head);  in free_pcppages_bulk()
1392 			 * under zone->lock. It is believed the overhead of  in free_pcppages_bulk()
1396 			 * prefetch buddy for the first pcp->batch nr of pages.  in free_pcppages_bulk()
1398 			if (prefetch_nr++ < pcp->batch)  in free_pcppages_bulk()
1400 		} while (--count && --batch_free && !list_empty(list));  in free_pcppages_bulk()
1403 	spin_lock(&zone->lock);  in free_pcppages_bulk()
1408 	 * page->lru.next will not point to original list.  in free_pcppages_bulk()
1421 	spin_unlock(&zone->lock);  in free_pcppages_bulk()
1429 	spin_lock(&zone->lock);  in free_one_page()
1435 	spin_unlock(&zone->lock);  in free_one_page()
1448 	INIT_LIST_HEAD(&page->lru);  in __init_single_page()
1469 		struct zone *zone = &pgdat->node_zones[zid];  in init_reserved_page()
1471 		if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))  in init_reserved_page()
1499 			/* Avoid false-positive PageTail() */  in reserve_bootmem_region()
1500 			INIT_LIST_HEAD(&page->lru);  in reserve_bootmem_region()
1542 	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {  in __free_pages_core()
1550 	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);  in __free_pages_core()
1574 	if (state->last_start <= pfn && pfn < state->last_end)  in __early_pfn_to_nid()
1575 		return state->last_nid;  in __early_pfn_to_nid()
1579 		state->last_start = start_pfn;  in __early_pfn_to_nid()
1580 		state->last_end = end_pfn;  in __early_pfn_to_nid()
1581 		state->last_nid = nid;  in __early_pfn_to_nid()
1635 	end_pfn--;  in __pageblock_pfn_to_page()
1658 	unsigned long block_start_pfn = zone->zone_start_pfn;  in set_zone_contiguous()
1675 	zone->contiguous = true;  in set_zone_contiguous()
1680 	zone->contiguous = false;  in clear_zone_contiguous()
1695 	/* Free a large naturally-aligned chunk if possible */  in deferred_free_range()
1697 	    (pfn & (pageblock_nr_pages - 1)) == 0) {  in deferred_free_range()
1704 		if ((pfn & (pageblock_nr_pages - 1)) == 0)  in deferred_free_range()
1727  * Then, we check if a current large page is valid by only checking the validity
1734 	if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))  in deferred_pfn_valid()
1746 	unsigned long nr_pgmask = pageblock_nr_pages - 1;  in deferred_free_pages()
1751 			deferred_free_range(pfn - nr_free, nr_free);  in deferred_free_pages()
1754 			deferred_free_range(pfn - nr_free, nr_free);  in deferred_free_pages()
1761 	deferred_free_range(pfn - nr_free, nr_free);  in deferred_free_pages()
1773 	unsigned long nr_pgmask = pageblock_nr_pages - 1;  in deferred_init_pages()
1795  * This function is meant to pre-load the iterator for the zone init.
1909 	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);  in deferred_init_memmap()
1919 		set_cpus_allowed_ptr(current, cpumask);  in deferred_init_memmap()
1922 	first_init_pfn = pgdat->first_deferred_pfn;  in deferred_init_memmap()
1930 	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);  in deferred_init_memmap()
1931 	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));  in deferred_init_memmap()
1932 	pgdat->first_deferred_pfn = ULONG_MAX;  in deferred_init_memmap()
1937 	 * pre-grown prior to start of deferred page initialization.  in deferred_init_memmap()
1943 		zone = pgdat->node_zones + zid;  in deferred_init_memmap()
1961 			.size        = epfn_align - spfn,  in deferred_init_memmap()
1976 		pgdat->node_id, jiffies_to_msecs(jiffies - start));  in deferred_init_memmap()
2001 	pg_data_t *pgdat = zone->zone_pgdat;  in deferred_grow_zone()
2002 	unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;  in deferred_grow_zone()
2017 	if (first_deferred_pfn != pgdat->first_deferred_pfn) {  in deferred_grow_zone()
2025 		pgdat->first_deferred_pfn = ULONG_MAX;  in deferred_grow_zone()
2052 	pgdat->first_deferred_pfn = spfn;  in deferred_grow_zone()
2098 	 * on-demand struct page initialization.  in page_alloc_init_late()
2126 	} while (++p, --i);  in init_cma_reserved_pageblock()
2135 			__free_pages(p, MAX_ORDER - 1);  in init_cma_reserved_pageblock()
2137 		} while (i -= MAX_ORDER_NR_PAGES);  in init_cma_reserved_pageblock()
2159  * -- nyc
2167 		high--;  in expand()
2187 	if (unlikely(page->flags & __PG_HWPOISON)) {  in check_new_page_bad()
2218  * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2236  * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2317 		area = &(zone->free_area[current_order]);  in __rmqueue_smallest()
2414 	start_pfn = start_pfn & ~(pageblock_nr_pages-1);  in move_freepages_block()
2416 	end_page = start_page + pageblock_nr_pages - 1;  in move_freepages_block()
2417 	end_pfn = start_pfn + pageblock_nr_pages - 1;  in move_freepages_block()
2432 	int nr_pageblocks = 1 << (start_order - pageblock_order);  in change_pageblock_range()
2434 	while (nr_pageblocks--) {  in change_pageblock_range()
2488 	max_boost = mult_frac(zone->_watermark[WMARK_HIGH],  in boost_watermark()
2504 	zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,  in boost_watermark()
2511  * pageblock to our migratetype and determine how many already-allocated pages
2545 		set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);  in steal_suitable_fallback()
2563 		 * to MOVABLE pageblock, consider all non-movable pages as  in steal_suitable_fallback()
2566 		 * exact migratetype of non-movable pages.  in steal_suitable_fallback()
2570 						- (free_pages + movable_pages);  in steal_suitable_fallback()
2583 	if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||  in steal_suitable_fallback()
2605 	if (area->nr_free == 0)  in find_suitable_fallback()
2606 		return -1;  in find_suitable_fallback()
2627 	return -1;  in find_suitable_fallback()
2631  * Reserve a pageblock for exclusive use of high-order atomic allocations if
2641 	 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.  in reserve_highatomic_pageblock()
2642 	 * Check is race-prone but harmless.  in reserve_highatomic_pageblock()
2645 	if (zone->nr_reserved_highatomic >= max_managed)  in reserve_highatomic_pageblock()
2648 	spin_lock_irqsave(&zone->lock, flags);  in reserve_highatomic_pageblock()
2650 	/* Recheck the nr_reserved_highatomic limit under the lock */  in reserve_highatomic_pageblock()
2651 	if (zone->nr_reserved_highatomic >= max_managed)  in reserve_highatomic_pageblock()
2658 		zone->nr_reserved_highatomic += pageblock_nr_pages;  in reserve_highatomic_pageblock()
2664 	spin_unlock_irqrestore(&zone->lock, flags);  in reserve_highatomic_pageblock()
2669  * potentially hurts the reliability of high-order allocations when under
2679 	struct zonelist *zonelist = ac->zonelist;  in unreserve_highatomic_pageblock()
2687 	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,  in unreserve_highatomic_pageblock()
2688 								ac->nodemask) {  in unreserve_highatomic_pageblock()
2693 		if (!force && zone->nr_reserved_highatomic <=  in unreserve_highatomic_pageblock()
2697 		spin_lock_irqsave(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2699 			struct free_area *area = &(zone->free_area[order]);  in unreserve_highatomic_pageblock()
2709 			 * from highatomic to ac->migratetype. So we should  in unreserve_highatomic_pageblock()
2715 				 * locking could inadvertently allow a per-cpu  in unreserve_highatomic_pageblock()
2720 				zone->nr_reserved_highatomic -= min(  in unreserve_highatomic_pageblock()
2722 						zone->nr_reserved_highatomic);  in unreserve_highatomic_pageblock()
2726 			 * Convert to ac->migratetype and avoid the normal  in unreserve_highatomic_pageblock()
2734 			set_pageblock_migratetype(page, ac->migratetype);  in unreserve_highatomic_pageblock()
2735 			ret = move_freepages_block(zone, page, ac->migratetype,  in unreserve_highatomic_pageblock()
2738 				spin_unlock_irqrestore(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2742 		spin_unlock_irqrestore(&zone->lock, flags);  in unreserve_highatomic_pageblock()
2782 	for (current_order = MAX_ORDER - 1; current_order >= min_order;  in __rmqueue_fallback()
2783 				--current_order) {  in __rmqueue_fallback()
2784 		area = &(zone->free_area[current_order]);  in __rmqueue_fallback()
2787 		if (fallback_mt == -1)  in __rmqueue_fallback()
2810 		area = &(zone->free_area[current_order]);  in __rmqueue_fallback()
2813 		if (fallback_mt != -1)  in __rmqueue_fallback()
2818 	 * This should not happen - we already found a suitable fallback  in __rmqueue_fallback()
2838  * Call me with the zone->lock already held.
2886 	spin_lock(&zone->lock);  in rmqueue_bulk()
2906 		list_add_tail(&page->lru, list);  in rmqueue_bulk()
2910 					      -(1 << order));  in rmqueue_bulk()
2919 	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));  in rmqueue_bulk()
2920 	spin_unlock(&zone->lock);  in rmqueue_bulk()
2939 	batch = READ_ONCE(pcp->batch);  in drain_zone_pages()
2940 	to_drain = min(pcp->count, batch);  in drain_zone_pages()
2950  * The processor must either be the current processor and the
2951  * thread pinned to the current processor or a processor that
2961 	pset = per_cpu_ptr(zone->pageset, cpu);  in drain_pages_zone()
2963 	pcp = &pset->pcp;  in drain_pages_zone()
2964 	if (pcp->count)  in drain_pages_zone()
2965 		free_pcppages_bulk(zone, pcp->count, pcp);  in drain_pages_zone()
2972  * The processor must either be the current processor and the
2973  * thread pinned to the current processor or a processor that
2986  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2988  * The CPU has to be pinned. When zone parameter is non-NULL, spill just
3015 	drain_local_pages(drain->zone);  in drain_local_pages_wq()
3020  * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3022  * When zone parameter is non-NULL, spill just the single zone's pages.
3066 			pcp = per_cpu_ptr(zone->pageset, cpu);  in drain_all_pages()
3067 			if (pcp->pcp.count)  in drain_all_pages()
3071 				pcp = per_cpu_ptr(z->pageset, cpu);  in drain_all_pages()
3072 				if (pcp->pcp.count) {  in drain_all_pages()
3088 		drain->zone = zone;  in drain_all_pages()
3089 		INIT_WORK(&drain->work, drain_local_pages_wq);  in drain_all_pages()
3090 		queue_work_on(cpu, mm_percpu_wq, &drain->work);  in drain_all_pages()
3093 		flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);  in drain_all_pages()
3115 	spin_lock_irqsave(&zone->lock, flags);  in mark_free_pages()
3118 	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)  in mark_free_pages()
3122 			if (!--page_count) {  in mark_free_pages()
3136 				&zone->free_area[order].free_list[t], lru) {  in mark_free_pages()
3141 				if (!--page_count) {  in mark_free_pages()
3149 	spin_unlock_irqrestore(&zone->lock, flags);  in mark_free_pages()
3190 	pcp = &this_cpu_ptr(zone->pageset)->pcp;  in free_unref_page_commit()
3191 	list_add(&page->lru, &pcp->lists[migratetype]);  in free_unref_page_commit()
3192 	pcp->count++;  in free_unref_page_commit()
3193 	if (pcp->count >= pcp->high) {  in free_unref_page_commit()
3194 		unsigned long batch = READ_ONCE(pcp->batch);  in free_unref_page_commit()
3200  * Free a 0-order page
3216  * Free a list of 0-order pages
3228 			list_del(&page->lru);  in free_unref_page_list()
3254  * split_page takes a non-compound higher-order page, and splits it into
3255  * n (1<<order) sub-pages: page[0..n]
3256  * Each sub-page must be freed individually.
3288 		 * emulate a high-order watermark check with a raised order-0  in __isolate_free_page()
3289 		 * watermark, because we already know our high-order page  in __isolate_free_page()
3292 		watermark = zone->_watermark[WMARK_MIN] + (1UL << order);  in __isolate_free_page()
3296 		__mod_zone_freepage_state(zone, -(1UL << order), mt);  in __isolate_free_page()
3307 	if (order >= pageblock_order - 1) {  in __isolate_free_page()
3308 		struct page *endpage = page + (1 << order) - 1;  in __isolate_free_page()
3323  * __putback_isolated_page - Return a now-isolated page back where we got it
3336 	lockdep_assert_held(&zone->lock);  in __putback_isolated_page()
3370 /* Remove page from the per-cpu list, caller must protect the list */
3380 			pcp->count += rmqueue_bulk(zone, 0,  in __rmqueue_pcplist()
3381 					pcp->batch, list,  in __rmqueue_pcplist()
3388 		list_del(&page->lru);  in __rmqueue_pcplist()
3389 		pcp->count--;  in __rmqueue_pcplist()
3395 /* Lock and remove page from the per-cpu list */
3406 	pcp = &this_cpu_ptr(zone->pageset)->pcp;  in rmqueue_pcplist()
3407 	list = &pcp->lists[migratetype];  in rmqueue_pcplist()
3418  * Allocate a page from the given zone. Use pcplists for order-0 allocations.
3444 	 * allocate greater than order-1 page units with __GFP_NOFAIL.  in rmqueue()
3447 	spin_lock_irqsave(&zone->lock, flags);  in rmqueue()
3452 		 * order-0 request can reach here when the pcplist is skipped  in rmqueue()
3453 		 * due to non-CMA allocation context. HIGHATOMIC area is  in rmqueue()
3454 		 * reserved for high-order atomic allocation, so order-0  in rmqueue()
3465 	spin_unlock(&zone->lock);  in rmqueue()
3468 	__mod_zone_freepage_state(zone, -(1 << order),  in rmqueue()
3477 	if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {  in rmqueue()
3478 		clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);  in rmqueue()
3536 	debugfs_create_bool("ignore-gfp-wait", mode, dir,  in fail_page_alloc_debugfs()
3538 	debugfs_create_bool("ignore-gfp-highmem", mode, dir,  in fail_page_alloc_debugfs()
3540 	debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);  in fail_page_alloc_debugfs()
3568 	long unusable_free = (1 << order) - 1;  in __zone_watermark_unusable_free()
3572 	 * the high-atomic reserves. This will over-estimate the size of the  in __zone_watermark_unusable_free()
3576 		unusable_free += z->nr_reserved_highatomic;  in __zone_watermark_unusable_free()
3588  * Return true if free base pages are above 'mark'. For high-order checks it
3589  * will return true of the order-0 watermark is reached and there is at least
3601 	/* free_pages may go negative - that's OK */  in __zone_watermark_ok()
3602 	free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);  in __zone_watermark_ok()
3605 		min -= min / 2;  in __zone_watermark_ok()
3612 		 * makes during the free path will be small and short-lived.  in __zone_watermark_ok()
3615 			min -= min / 2;  in __zone_watermark_ok()
3617 			min -= min / 4;  in __zone_watermark_ok()
3621 	 * Check watermarks for an order-0 allocation request. If these  in __zone_watermark_ok()
3622 	 * are not met, then a high-order request also cannot go ahead  in __zone_watermark_ok()
3625 	if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])  in __zone_watermark_ok()
3628 	/* If this is an order-0 request then the watermark is fine */  in __zone_watermark_ok()
3632 	/* For a high-order request, check at least one suitable page is free */  in __zone_watermark_ok()
3634 		struct free_area *area = &z->free_area[o];  in __zone_watermark_ok()
3637 		if (!area->nr_free)  in __zone_watermark_ok()
3673 	 * Fast check for order-0 only. If this fails then the reserves  in zone_watermark_fast()
3680 		fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);  in zone_watermark_fast()
3681 		if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])  in zone_watermark_fast()
3689 	 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations  in zone_watermark_fast()
3694 	if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost  in zone_watermark_fast()
3696 		mark = z->_watermark[WMARK_MIN];  in zone_watermark_fast()
3709 	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)  in zone_watermark_ok_safe()
3757 	 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume  in alloc_flags_nofragment()
3760 	BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);  in alloc_flags_nofragment()
3761 	if (nr_online_nodes > 1 && !populated_zone(--zone))  in alloc_flags_nofragment()
3773 	unsigned int pflags = current->flags;  in current_alloc_flags()
3802 	z = ac->preferred_zoneref;  in get_page_from_freelist()
3803 	for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,  in get_page_from_freelist()
3804 					ac->nodemask) {  in get_page_from_freelist()
3815 		 * limit, such that no single node holds more than its  in get_page_from_freelist()
3823 		 * exceed the per-node dirty limit in the slowpath  in get_page_from_freelist()
3827 		 * global limit.  The proper fix for these situations  in get_page_from_freelist()
3829 		 * dirty-throttling and the flusher threads.  in get_page_from_freelist()
3831 		if (ac->spread_dirty_pages) {  in get_page_from_freelist()
3832 			if (last_pgdat_dirty_limit == zone->zone_pgdat)  in get_page_from_freelist()
3835 			if (!node_dirty_ok(zone->zone_pgdat)) {  in get_page_from_freelist()
3836 				last_pgdat_dirty_limit = zone->zone_pgdat;  in get_page_from_freelist()
3842 		    zone != ac->preferred_zoneref->zone) {  in get_page_from_freelist()
3850 			local_nid = zone_to_nid(ac->preferred_zoneref->zone);  in get_page_from_freelist()
3859 				       ac->highest_zoneidx, alloc_flags,  in get_page_from_freelist()
3879 			    !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))  in get_page_from_freelist()
3882 			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);  in get_page_from_freelist()
3893 					ac->highest_zoneidx, alloc_flags))  in get_page_from_freelist()
3901 		page = rmqueue(ac->preferred_zoneref->zone, zone, order,  in get_page_from_freelist()
3902 				gfp_mask, alloc_flags, ac->migratetype);  in get_page_from_freelist()
3907 			 * If this is a high-order atomic allocation then check  in get_page_from_freelist()
3943 	 * contexts that are allowed to allocate outside current's set  in warn_alloc_show_mem()
3947 		if (tsk_is_oom_victim(current) ||  in warn_alloc_show_mem()
3948 		    (current->flags & (PF_MEMALLOC | PF_EXITING)))  in warn_alloc_show_mem()
3969 			current->comm, &vaf, gfp_mask, &gfp_mask,  in warn_alloc()
4004 		.zonelist = ac->zonelist,  in __alloc_pages_may_oom()
4005 		.nodemask = ac->nodemask,  in __alloc_pages_may_oom()
4038 	if (current->flags & PF_DUMPCORE)  in __alloc_pages_may_oom()
4054 	if (ac->highest_zoneidx < ZONE_NORMAL)  in __alloc_pages_may_oom()
4063 	 * access to memory reserved if the current task is killed (see  in __alloc_pages_may_oom()
4073 		 * Help non-failing allocations by giving them access to memory  in __alloc_pages_may_oom()
4092 /* Try memory compaction for high-order allocations before reclaim */
4131 		zone->compact_blockskip_flush = false;  in __alloc_pages_direct_compact()
4175 	 * compaction was skipped because there are not enough order-0 pages  in should_compact_retry()
4217 		(*compact_priority)--;  in should_compact_retry()
4250 	 * Let's give them a good hope and keep retrying while the order-0  in should_compact_retry()
4253 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in should_compact_retry()
4254 				ac->highest_zoneidx, ac->nodemask) {  in should_compact_retry()
4256 					ac->highest_zoneidx, alloc_flags))  in should_compact_retry()
4276 	if (current->flags & PF_MEMALLOC)  in __need_fs_reclaim()
4330 	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,  in __perform_reclaim()
4331 								ac->nodemask);  in __perform_reclaim()
4360 	 * pages are pinned on the per-cpu lists or in high alloc reserves.  in __alloc_pages_direct_reclaim()
4379 	enum zone_type highest_zoneidx = ac->highest_zoneidx;  in wake_all_kswapds()
4381 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,  in wake_all_kswapds()
4382 					ac->nodemask) {  in wake_all_kswapds()
4383 		if (last_pgdat != zone->zone_pgdat)  in wake_all_kswapds()
4385 		last_pgdat = zone->zone_pgdat;  in wake_all_kswapds()
4423 	} else if (unlikely(rt_task(current)) && !in_interrupt())  in gfp_to_alloc_flags()
4456 	if (in_serving_softirq() && (current->flags & PF_MEMALLOC))  in __gfp_pfmemalloc_flags()
4459 		if (current->flags & PF_MEMALLOC)  in __gfp_pfmemalloc_flags()
4461 		else if (oom_reserves_allowed(current))  in __gfp_pfmemalloc_flags()
4517 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in should_reclaim_retry()
4518 				ac->highest_zoneidx, ac->nodemask) {  in should_reclaim_retry()
4532 				ac->highest_zoneidx, alloc_flags, available);  in should_reclaim_retry()
4562 	 * current implementation of the WQ concurrency control doesn't  in should_reclaim_retry()
4567 	if (current->flags & PF_WQ_WORKER)  in should_reclaim_retry()
4583 	 * This assumes that for all allocations, ac->nodemask can come only  in check_retry_cpuset()
4588 	if (cpusets_enabled() && ac->nodemask &&  in check_retry_cpuset()
4589 			!cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {  in check_retry_cpuset()
4590 		ac->nodemask = NULL;  in check_retry_cpuset()
4647 	 * there was a cpuset modification and we are retrying - otherwise we  in __alloc_pages_slowpath()
4648 	 * could end up iterating over non-eligible zones endlessly.  in __alloc_pages_slowpath()
4650 	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,  in __alloc_pages_slowpath()
4651 					ac->highest_zoneidx, ac->nodemask);  in __alloc_pages_slowpath()
4652 	if (!ac->preferred_zoneref->zone)  in __alloc_pages_slowpath()
4668 	 * that we have enough base pages and don't need to reclaim. For non-  in __alloc_pages_slowpath()
4669 	 * movable high-order allocations, do that as well, as compaction will  in __alloc_pages_slowpath()
4677 			   (order > 0 && ac->migratetype != MIGRATE_MOVABLE))  in __alloc_pages_slowpath()
4699 			 *  - potentially very expensive because zones are far  in __alloc_pages_slowpath()
4702 			 *  - not guaranteed to help because isolate_freepages()  in __alloc_pages_slowpath()
4705 			 *  - unlikely to make entire pageblocks free on its  in __alloc_pages_slowpath()
4736 		ac->nodemask = NULL;  in __alloc_pages_slowpath()
4737 		ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,  in __alloc_pages_slowpath()
4738 					ac->highest_zoneidx, ac->nodemask);  in __alloc_pages_slowpath()
4751 	if (current->flags & PF_MEMALLOC)  in __alloc_pages_slowpath()
4782 	 * It doesn't make any sense to retry for the compaction if the order-0  in __alloc_pages_slowpath()
4783 	 * reclaim is not able to make any progress because the current  in __alloc_pages_slowpath()
4804 	if (tsk_is_oom_victim(current) &&  in __alloc_pages_slowpath()
4837 		WARN_ON_ONCE(current->flags & PF_MEMALLOC);  in __alloc_pages_slowpath()
4848 		 * Help non-failing allocations by giving them access to memory  in __alloc_pages_slowpath()
4861 	warn_alloc(gfp_mask, ac->nodemask,  in __alloc_pages_slowpath()
4872 	ac->highest_zoneidx = gfp_zone(gfp_mask);  in prepare_alloc_pages()
4873 	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);  in prepare_alloc_pages()
4874 	ac->nodemask = nodemask;  in prepare_alloc_pages()
4875 	ac->migratetype = gfp_migratetype(gfp_mask);  in prepare_alloc_pages()
4881 		 * to the current task context. It means that any node ok.  in prepare_alloc_pages()
4883 		if (!in_interrupt() && !ac->nodemask)  in prepare_alloc_pages()
4884 			ac->nodemask = &cpuset_current_mems_allowed;  in prepare_alloc_pages()
4900 	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);  in prepare_alloc_pages()
4907 	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,  in prepare_alloc_pages()
4908 					ac->highest_zoneidx, ac->nodemask);  in prepare_alloc_pages()
4943 	alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);  in __alloc_pages_nodemask()
4961 	 * &cpuset_current_mems_allowed to optimize the fast-path attempt.  in __alloc_pages_nodemask()
5015 		while (order-- > 0)  in __free_pages()
5032  *  An arbitrary-length arbitrary-offset area of memory which resides
5039  * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
5052 	nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;  in __page_frag_cache_refill()
5057 	nc->va = page ? page_address(page) : NULL;  in __page_frag_cache_refill()
5078 	if (unlikely(!nc->va)) {  in page_frag_alloc()
5086 		size = nc->size;  in page_frag_alloc()
5094 		nc->pfmemalloc = page_is_pfmemalloc(page);  in page_frag_alloc()
5095 		nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;  in page_frag_alloc()
5096 		nc->offset = size;  in page_frag_alloc()
5099 	offset = nc->offset - fragsz;  in page_frag_alloc()
5101 		page = virt_to_page(nc->va);  in page_frag_alloc()
5103 		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))  in page_frag_alloc()
5106 		if (unlikely(nc->pfmemalloc)) {  in page_frag_alloc()
5113 		size = nc->size;  in page_frag_alloc()
5119 		nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;  in page_frag_alloc()
5120 		offset = size - fragsz;  in page_frag_alloc()
5123 	nc->pagecnt_bias--;  in page_frag_alloc()
5124 	nc->offset = offset;  in page_frag_alloc()
5126 	return nc->va + offset;  in page_frag_alloc()
5159  * alloc_pages_exact - allocate an exact number physically-contiguous pages.
5165  * allocate memory in power-of-two pages.
5187  * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
5213  * free_pages_exact - release memory allocated via alloc_pages_exact()
5232  * nr_free_zone_pages - count number of pages beyond high watermark
5239  *     nr_free_zone_pages = managed_pages - high_pages
5257 			sum += size - high;  in nr_free_zone_pages()
5264  * nr_free_buffer_pages - count number of pages beyond high watermark
5304 	available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;  in si_mem_available()
5312 	pagecache -= min(pagecache / 2, wmark_low);  in si_mem_available()
5322 	available += reclaimable - min(reclaimable / 2, wmark_low);  in si_mem_available()
5332 	val->totalram = totalram_pages();  in si_meminfo()
5333 	val->sharedram = global_node_page_state(NR_SHMEM);  in si_meminfo()
5334 	val->freeram = global_zone_page_state(NR_FREE_PAGES);  in si_meminfo()
5335 	val->bufferram = nr_blockdev_pages();  in si_meminfo()
5336 	val->totalhigh = totalhigh_pages();  in si_meminfo()
5337 	val->freehigh = nr_free_highpages();  in si_meminfo()
5338 	val->mem_unit = PAGE_SIZE;  in si_meminfo()
5353 		managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);  in si_meminfo_node()
5354 	val->totalram = managed_pages;  in si_meminfo_node()
5355 	val->sharedram = node_page_state(pgdat, NR_SHMEM);  in si_meminfo_node()
5356 	val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);  in si_meminfo_node()
5359 		struct zone *zone = &pgdat->node_zones[zone_type];  in si_meminfo_node()
5366 	val->totalhigh = managed_highpages;  in si_meminfo_node()
5367 	val->freehigh = free_highpages;  in si_meminfo_node()
5369 	val->totalhigh = managed_highpages;  in si_meminfo_node()
5370 	val->freehigh = free_highpages;  in si_meminfo_node()
5372 	val->mem_unit = PAGE_SIZE;  in si_meminfo_node()
5386 	 * no node mask - aka implicit memory numa policy. Do not bother with  in show_mem_node_skip()
5387 	 * the synchronization - read_mems_allowed_begin - because we do not  in show_mem_node_skip()
5396 #define K(x) ((x) << (PAGE_SHIFT-10))
5426  * Show free area list (used inside shift_scroll-lock stuff)
5431  * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
5446 			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;  in show_free_areas()
5475 		if (show_mem_node_skip(filter, pgdat->node_id, nodemask))  in show_free_areas()
5502 			pgdat->node_id,  in show_free_areas()
5525 			pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?  in show_free_areas()
5537 			free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;  in show_free_areas()
5562 			zone->name,  in show_free_areas()
5567 			K(zone->nr_reserved_highatomic),  in show_free_areas()
5574 			K(zone->present_pages),  in show_free_areas()
5580 			K(this_cpu_read(zone->pageset->pcp.count)),  in show_free_areas()
5584 			printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);  in show_free_areas()
5596 		printk(KERN_CONT "%s: ", zone->name);  in show_free_areas()
5598 		spin_lock_irqsave(&zone->lock, flags);  in show_free_areas()
5600 			struct free_area *area = &zone->free_area[order];  in show_free_areas()
5603 			nr[order] = area->nr_free;  in show_free_areas()
5612 		spin_unlock_irqrestore(&zone->lock, flags);  in show_free_areas()
5631 	zoneref->zone = zone;  in zoneref_set_zone()
5632 	zoneref->zone_idx = zone_idx(zone);  in zoneref_set_zone()
5647 		zone_type--;  in build_zonerefs_node()
5648 		zone = pgdat->node_zones + zone_type;  in build_zonerefs_node()
5670 		return -EINVAL;  in __parse_numa_zonelist_order()
5693  * find_next_best_node - find the next node that should appear in a given node's fallback list
5754  * This results in maximum locality--normal zone overflows into local
5755  * DMA zone, if any--but risks exhausting DMA zone.
5763 	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;  in build_zonelists_in_node_order()
5773 	zonerefs->zone = NULL;  in build_zonelists_in_node_order()
5774 	zonerefs->zone_idx = 0;  in build_zonelists_in_node_order()
5785 	zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;  in build_thisnode_zonelists()
5788 	zonerefs->zone = NULL;  in build_thisnode_zonelists()
5789 	zonerefs->zone_idx = 0;  in build_thisnode_zonelists()
5806 	/* NUMA-aware ordering of nodes */  in build_zonelists()
5807 	local_node = pgdat->node_id;  in build_zonelists()
5816 		 * distance group to make it round-robin.  in build_zonelists()
5824 		load--;  in build_zonelists()
5845 	return zone_to_nid(z->zone);  in local_memory_node()
5859 	local_node = pgdat->node_id;  in build_zonelists()
5861 	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;  in build_zonelists()
5886 	zonerefs->zone = NULL;  in build_zonelists()
5887 	zonerefs->zone_idx = 0;  in build_zonelists()
5926 	 * building zonelists is fine - no need to touch other nodes.  in __build_all_zonelists()
5928 	if (self && !node_online(self->node_id)) {  in __build_all_zonelists()
5939 		 * We now know the "local memory node" for each node--  in __build_all_zonelists()
5941 		 * Set up numa_mem percpu variable for on-line cpus.  During  in __build_all_zonelists()
5942 		 * boot, only the boot cpu should be on-line;  we'll init the  in __build_all_zonelists()
5943 		 * secondary cpus' numa_mem as they come on-line.  During  in __build_all_zonelists()
5944 		 * node/memory hotplug, we'll fixup all on-line cpus.  in __build_all_zonelists()
5972 	 * (a chicken-egg dilemma).  in build_all_zonelists_init()
6002 	 * more accurate, but expensive to check per-zone. This check is  in build_all_zonelists()
6003 	 * made on memory-hotadd so a system can start with mobility  in build_all_zonelists()
6043  * Initially all pages are reserved - free ones are freed
6045  * done. Non-atomic initialization, single-pass.
6059 	if (highest_memmap_pfn < end_pfn - 1)  in memmap_init_zone()
6060 		highest_memmap_pfn = end_pfn - 1;  in memmap_init_zone()
6065 	 * memory. We limit the total number of pages to initialize to just  in memmap_init_zone()
6066 	 * those that might contain the memory mapping. We will defer the  in memmap_init_zone()
6074 		if (start_pfn == altmap->base_pfn)  in memmap_init_zone()
6075 			start_pfn += altmap->reserve;  in memmap_init_zone()
6076 		end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);  in memmap_init_zone()
6082 		 * There can be holes in boot-time mem_map[]s handed to this  in memmap_init_zone()
6117 	struct pglist_data *pgdat = zone->zone_pgdat;  in memmap_init_zone_device()
6121 	int nid = pgdat->node_id;  in memmap_init_zone_device()
6132 		start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);  in memmap_init_zone_device()
6133 		nr_pages = end_pfn - start_pfn;  in memmap_init_zone_device()
6145 		 * We can use the non-atomic __set_bit operation for setting  in memmap_init_zone_device()
6151 		 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer  in memmap_init_zone_device()
6153 		 * ever freed or placed on a driver-private list.  in memmap_init_zone_device()
6155 		page->pgmap = pgmap;  in memmap_init_zone_device()
6156 		page->zone_device_data = NULL;  in memmap_init_zone_device()
6162 		 * the address space during boot when many long-lived  in memmap_init_zone_device()
6175 		nr_pages, jiffies_to_msecs(jiffies - start));  in memmap_init_zone_device()
6183 		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);  in zone_init_free_lists()
6184 		zone->free_area[order].nr_free = 0;  in zone_init_free_lists()
6201 			size = end_pfn - start_pfn;  in memmap_init()
6214 	 * The per-cpu-pages pools are set to around 1000th of the  in zone_batchsize()
6226 	 * Clamp the batch to a 2^n - 1 value. Having a power  in zone_batchsize()
6235 	batch = rounddown_pow_of_two(batch + batch/2) - 1;  in zone_batchsize()
6251 	 * fragmented and becoming unavailable for high-order allocations.  in zone_batchsize()
6258  * pcp->high and pcp->batch values are related and dependent on one another:
6259  * ->batch must never be higher then ->high.
6263  * Any new users of pcp->batch and pcp->high should ensure they can cope with
6274 	pcp->batch = 1;  in pageset_update()
6278 	pcp->high = high;  in pageset_update()
6281 	pcp->batch = batch;  in pageset_update()
6287 	pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));  in pageset_set_batch()
6297 	pcp = &p->pcp;  in pageset_init()
6299 		INIT_LIST_HEAD(&pcp->lists[migratetype]);  in pageset_init()
6319 	pageset_update(&p->pcp, high, batch);  in pageset_set_high()
6335 	struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);  in zone_pageset_init()
6344 	zone->pageset = alloc_percpu(struct per_cpu_pageset);  in setup_zone_pageset()
6371 		memset(pcp->vm_numa_stat_diff, 0,  in setup_per_cpu_pageset()
6372 		       sizeof(pcp->vm_numa_stat_diff));  in setup_per_cpu_pageset()
6377 		pgdat->per_cpu_nodestats =  in setup_per_cpu_pageset()
6388 	zone->pageset = &boot_pageset;  in zone_pcp_init()
6392 			zone->name, zone->present_pages,  in zone_pcp_init()
6400 	struct pglist_data *pgdat = zone->zone_pgdat;  in init_currently_empty_zone()
6403 	if (zone_idx > pgdat->nr_zones)  in init_currently_empty_zone()
6404 		pgdat->nr_zones = zone_idx;  in init_currently_empty_zone()
6406 	zone->zone_start_pfn = zone_start_pfn;  in init_currently_empty_zone()
6409 			"Initialising map node %d zone %lu pfns %lu -> %lu\n",  in init_currently_empty_zone()
6410 			pgdat->node_id,  in init_currently_empty_zone()
6415 	zone->initialized = 1;  in init_currently_empty_zone()
6419  * get_pfn_range_for_nid - Return the start and end page frames for a node
6435 	*start_pfn = -1UL;  in get_pfn_range_for_nid()
6443 	if (*start_pfn == -1UL)  in get_pfn_range_for_nid()
6455 	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {  in find_usable_zone_for_movable()
6464 	VM_BUG_ON(zone_index == -1);  in find_usable_zone_for_movable()
6507  * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
6538 	return *zone_end_pfn - *zone_start_pfn;  in zone_spanned_pages_in_node()
6549 	unsigned long nr_absent = range_end_pfn - range_start_pfn;  in __absent_pages_in_range()
6556 		nr_absent -= end_pfn - start_pfn;  in __absent_pages_in_range()
6562  * absent_pages_in_range - Return number of page frames in holes within a range
6614 				nr_absent += end_pfn - start_pfn;  in zone_absent_pages_in_node()
6618 				nr_absent += end_pfn - start_pfn;  in zone_absent_pages_in_node()
6633 		struct zone *zone = pgdat->node_zones + i;  in calculate_node_totalpages()
6638 		spanned = zone_spanned_pages_in_node(pgdat->node_id, i,  in calculate_node_totalpages()
6643 		absent = zone_absent_pages_in_node(pgdat->node_id, i,  in calculate_node_totalpages()
6648 		real_size = size - absent;  in calculate_node_totalpages()
6651 			zone->zone_start_pfn = zone_start_pfn;  in calculate_node_totalpages()
6653 			zone->zone_start_pfn = 0;  in calculate_node_totalpages()
6654 		zone->spanned_pages = size;  in calculate_node_totalpages()
6655 		zone->present_pages = real_size;  in calculate_node_totalpages()
6661 	pgdat->node_spanned_pages = totalpages;  in calculate_node_totalpages()
6662 	pgdat->node_present_pages = realtotalpages;  in calculate_node_totalpages()
6663 	printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,  in calculate_node_totalpages()
6669  * Calculate the size of the zone->blockflags rounded to an unsigned long
6679 	zonesize += zone_start_pfn & (pageblock_nr_pages-1);  in usemap_size()
6694 	zone->pageblock_flags = NULL;  in setup_usemap()
6696 		zone->pageblock_flags =  in setup_usemap()
6698 					    pgdat->node_id);  in setup_usemap()
6699 		if (!zone->pageblock_flags)  in setup_usemap()
6701 			      usemapsize, zone->name, pgdat->node_id);  in setup_usemap()
6723 		order = MAX_ORDER - 1;  in set_pageblock_order()
6736  * is unused as pageblock_order is set at compile-time. See
6737  * include/linux/pageblock-flags.h for the values of pageblock_order based on
6769 	struct deferred_split *ds_queue = &pgdat->deferred_split_queue;  in pgdat_init_split_queue()
6771 	spin_lock_init(&ds_queue->split_queue_lock);  in pgdat_init_split_queue()
6772 	INIT_LIST_HEAD(&ds_queue->split_queue);  in pgdat_init_split_queue()
6773 	ds_queue->split_queue_len = 0;  in pgdat_init_split_queue()
6782 	init_waitqueue_head(&pgdat->kcompactd_wait);  in pgdat_init_kcompactd()
6795 	init_waitqueue_head(&pgdat->kswapd_wait);  in pgdat_init_internals()
6796 	init_waitqueue_head(&pgdat->pfmemalloc_wait);  in pgdat_init_internals()
6799 	spin_lock_init(&pgdat->lru_lock);  in pgdat_init_internals()
6800 	lruvec_init(&pgdat->__lruvec);  in pgdat_init_internals()
6806 	atomic_long_set(&zone->managed_pages, remaining_pages);  in zone_init_internals()
6808 	zone->name = zone_names[idx];  in zone_init_internals()
6809 	zone->zone_pgdat = NODE_DATA(nid);  in zone_init_internals()
6810 	spin_lock_init(&zone->lock);  in zone_init_internals()
6817  * - init pgdat internals
6818  * - init all zones belonging to this node
6830 		zone_init_internals(&pgdat->node_zones[z], z, nid, 0);  in free_area_init_core_hotplug()
6836  *   - mark all pages reserved
6837  *   - mark all memory queues empty
6838  *   - clear the memory bitmaps
6846 	int nid = pgdat->node_id;  in free_area_init_core()
6849 	pgdat->per_cpu_nodestats = &boot_nodestats;  in free_area_init_core()
6852 		struct zone *zone = pgdat->node_zones + j;  in free_area_init_core()
6854 		unsigned long zone_start_pfn = zone->zone_start_pfn;  in free_area_init_core()
6856 		size = zone->spanned_pages;  in free_area_init_core()
6857 		freesize = zone->present_pages;  in free_area_init_core()
6862 		 * and per-cpu initialisations  in free_area_init_core()
6867 				freesize -= memmap_pages;  in free_area_init_core()
6879 			freesize -= dma_reserve;  in free_area_init_core()
6886 		/* Charge for highmem memmap if there are enough kernel pages */  in free_area_init_core()
6888 			nr_kernel_pages -= memmap_pages;  in free_area_init_core()
6915 	if (!pgdat->node_spanned_pages)  in alloc_node_mem_map()
6918 	start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);  in alloc_node_mem_map()
6919 	offset = pgdat->node_start_pfn - start;  in alloc_node_mem_map()
6921 	if (!pgdat->node_mem_map) {  in alloc_node_mem_map()
6932 		size =  (end - start) * sizeof(struct page);  in alloc_node_mem_map()
6934 					  pgdat->node_id);  in alloc_node_mem_map()
6937 			      size, pgdat->node_id);  in alloc_node_mem_map()
6938 		pgdat->node_mem_map = map + offset;  in alloc_node_mem_map()
6941 				__func__, pgdat->node_id, (unsigned long)pgdat,  in alloc_node_mem_map()
6942 				(unsigned long)pgdat->node_mem_map);  in alloc_node_mem_map()
6948 		mem_map = NODE_DATA(0)->node_mem_map;  in alloc_node_mem_map()
6949 		if (page_to_pfn(mem_map) != pgdat->node_start_pfn)  in alloc_node_mem_map()
6950 			mem_map -= offset;  in alloc_node_mem_map()
6961 	pgdat->first_deferred_pfn = ULONG_MAX;  in pgdat_set_deferred_range()
6974 	WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);  in free_area_init_node()
6978 	pgdat->node_id = nid;  in free_area_init_node()
6979 	pgdat->node_start_pfn = start_pfn;  in free_area_init_node()
6980 	pgdat->per_cpu_nodestats = NULL;  in free_area_init_node()
6982 	pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,  in free_area_init_node()
6984 		end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);  in free_area_init_node()
7011 				+ pageblock_nr_pages - 1;  in init_unavailable_range()
7017 		 * get re-initialized via reserve_bootmem_region() later.  in init_unavailable_range()
7059 	 * section - see pfn_valid(). If the last section has holes at the  in init_unavailable_mem()
7094  * node_map_pfn_alignment - determine the maximum internode alignment
7101  * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
7105  * This is used to test whether pfn -> nid mapping of the chosen memory
7106  * model has fine enough granularity to avoid incorrect mapping for the
7127 		 * Start with a mask granular enough to pin-point to the  in node_map_pfn_alignment()
7128 		 * start pfn and tick off bits one-by-one until it becomes  in node_map_pfn_alignment()
7129 		 * too coarse to separate the current node from the last.  in node_map_pfn_alignment()
7131 		mask = ~((1 << __ffs(start)) - 1);  in node_map_pfn_alignment()
7144  * find_min_pfn_with_active_regions - Find the minimum PFN registered
7166 		unsigned long pages = end_pfn - start_pfn;  in early_calculate_totalpages()
7206 			usable_startpfn = PFN_DOWN(r->base);  in find_zone_movable_pfns_for_nodes()
7268 		 * Round-up so that ZONE_MOVABLE is at least as large as what  in find_zone_movable_pfns_for_nodes()
7274 		corepages = totalpages - required_movablecore;  in find_zone_movable_pfns_for_nodes()
7322 								- start_pfn;  in find_zone_movable_pfns_for_nodes()
7324 				kernelcore_remaining -= min(kernel_pages,  in find_zone_movable_pfns_for_nodes()
7326 				required_kernelcore -= min(kernel_pages,  in find_zone_movable_pfns_for_nodes()
7346 			 * start_pfn->end_pfn. Calculate size_pages as the  in find_zone_movable_pfns_for_nodes()
7349 			size_pages = end_pfn - start_pfn;  in find_zone_movable_pfns_for_nodes()
7359 			required_kernelcore -= min(required_kernelcore,  in find_zone_movable_pfns_for_nodes()
7361 			kernelcore_remaining -= size_pages;  in find_zone_movable_pfns_for_nodes()
7373 	usable_nodes--;  in find_zone_movable_pfns_for_nodes()
7393 	for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {  in check_for_memory()
7394 		struct zone *zone = &pgdat->node_zones[zone_type];  in check_for_memory()
7415  * free_area_init - Initialise all pg_data_t and zone data
7444 			zone = MAX_NR_ZONES - i - 1;  in free_area_init()
7467 		pr_info("  %-8s ", zone_names[i]);  in free_area_init()
7472 			pr_cont("[mem %#018Lx-%#018Lx]\n",  in free_area_init()
7476 					<< PAGE_SHIFT) - 1);  in free_area_init()
7489 	 * subsection-map relative to active online memory ranges to  in free_area_init()
7490 	 * enable future "sub-section" extensions of the memory map.  in free_area_init()
7494 		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,  in free_area_init()
7496 			((u64)end_pfn << PAGE_SHIFT) - 1);  in free_area_init()
7497 		subsection_map_init(start_pfn, end_pfn - start_pfn);  in free_area_init()
7509 		if (pgdat->node_present_pages)  in free_area_init()
7522 		return -EINVAL;  in cmdline_parse_core()
7573 	atomic_long_add(count, &page_zone(page)->managed_pages);  in adjust_managed_page_count()
7609 			s, pages << (PAGE_SHIFT - 10));  in free_reserved_area()
7619 	atomic_long_inc(&page_zone(page)->managed_pages);  in free_highmem_page()
7631 	codesize = _etext - _stext;  in mem_init_print_info()
7632 	datasize = _edata - _sdata;  in mem_init_print_info()
7633 	rosize = __end_rodata - __start_rodata;  in mem_init_print_info()
7634 	bss_size = __bss_stop - __bss_start;  in mem_init_print_info()
7635 	init_data_size = __init_end - __init_begin;  in mem_init_print_info()
7636 	init_code_size = _einittext - _sinittext;  in mem_init_print_info()
7648 			size -= adj; \  in mem_init_print_info()
7660 …(%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"  in mem_init_print_info()
7665 		nr_free_pages() << (PAGE_SHIFT - 10),  in mem_init_print_info()
7666 		physpages << (PAGE_SHIFT - 10),  in mem_init_print_info()
7669 		(physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),  in mem_init_print_info()
7670 		totalcma_pages << (PAGE_SHIFT - 10),  in mem_init_print_info()
7672 		totalhigh_pages() << (PAGE_SHIFT - 10),  in mem_init_print_info()
7678  * set_dma_reserve - set the specified number of pages reserved in the first zone
7681  * The per-cpu batchsize and zone watermarks are determined by managed_pages.
7686  * smaller per-cpu batchsize.
7701 	 * into the current processors event counters.  in page_alloc_cpu_dead()
7702 	 * This artificially elevates the count of the current  in page_alloc_cpu_dead()
7747  * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
7758 		pgdat->totalreserve_pages = 0;  in calculate_totalreserve_pages()
7761 			struct zone *zone = pgdat->node_zones + i;  in calculate_totalreserve_pages()
7767 				if (zone->lowmem_reserve[j] > max)  in calculate_totalreserve_pages()
7768 					max = zone->lowmem_reserve[j];  in calculate_totalreserve_pages()
7777 			pgdat->totalreserve_pages += max;  in calculate_totalreserve_pages()
7786  * setup_per_zone_lowmem_reserve - called whenever
7798 			struct zone *zone = pgdat->node_zones + j;  in setup_per_zone_lowmem_reserve()
7801 			zone->lowmem_reserve[j] = 0;  in setup_per_zone_lowmem_reserve()
7807 				idx--;  in setup_per_zone_lowmem_reserve()
7808 				lower_zone = pgdat->node_zones + idx;  in setup_per_zone_lowmem_reserve()
7812 					lower_zone->lowmem_reserve[j] = 0;  in setup_per_zone_lowmem_reserve()
7815 					lower_zone->lowmem_reserve[j] =  in setup_per_zone_lowmem_reserve()
7829 	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);  in __setup_per_zone_wmarks()
7843 		spin_lock_irqsave(&zone->lock, flags);  in __setup_per_zone_wmarks()
7852 			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)  in __setup_per_zone_wmarks()
7860 			zone->_watermark[WMARK_MIN] = min_pages;  in __setup_per_zone_wmarks()
7866 			zone->_watermark[WMARK_MIN] = tmp;  in __setup_per_zone_wmarks()
7878 		zone->watermark_boost = 0;  in __setup_per_zone_wmarks()
7879 		zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;  in __setup_per_zone_wmarks()
7880 		zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;  in __setup_per_zone_wmarks()
7882 		spin_unlock_irqrestore(&zone->lock, flags);  in __setup_per_zone_wmarks()
7890  * setup_per_zone_wmarks - called when min_free_kbytes changes
7891  * or when memory is hot-{added|removed}
7963  * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so  in postcore_initcall()
8005 		pgdat->min_unmapped_pages = 0;  in setup_min_unmapped_ratio()
8008 		zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *  in setup_min_unmapped_ratio()
8033 		pgdat->min_slab_pages = 0;  in setup_min_slab_ratio()
8036 		zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *  in setup_min_slab_ratio()
8056  * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
8086 				per_cpu_ptr(zone->pageset, cpu));  in __zone_pcp_update()
8090  * percpu_pagelist_fraction - changes the pcp->high for each zone on each
8112 		ret = -EINVAL;  in percpu_pagelist_fraction_sysctl_handler()
8144  * Because 32-bit systems cannot have large physical memory, where this scaling
8155  * - it is assumed that the hash table must contain an exact power-of-2
8157  * - limit is the number of hash buckets, not the total allocation size
8179 		numentries -= arch_reserved_kernel_pages();  in alloc_large_system_hash()
8195 		/* limit to 1 bucket per 2^scale bytes of low memory */  in alloc_large_system_hash()
8197 			numentries >>= (scale - PAGE_SHIFT);  in alloc_large_system_hash()
8199 			numentries <<= (PAGE_SHIFT - scale);  in alloc_large_system_hash()
8201 		/* Make sure we've got at least a 0-order allocation.. */  in alloc_large_system_hash()
8214 	/* limit allocation size to 1/16 total memory by default */  in alloc_large_system_hash()
8243 			 * If bucketsize is not a power-of-two, we may free  in alloc_large_system_hash()
8250 	} while (!table && size > PAGE_SIZE && --log2qty);  in alloc_large_system_hash()
8256 		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,  in alloc_large_system_hash()
8262 		*_hash_mask = (1 << log2qty) - 1;  in alloc_large_system_hash()
8272  * check without lock_page also may miss some movable non-lru pages at
8299 	for (; iter < pageblock_nr_pages - offset; iter++) {  in has_unmovable_pages()
8339 			skip_pages = compound_nr(head) - (page - head);  in has_unmovable_pages()
8340 			iter += skip_pages - 1;  in has_unmovable_pages()
8348 		 * because their page->_refcount is zero at all time.  in has_unmovable_pages()
8352 				iter += (1 << buddy_order(page)) - 1;  in has_unmovable_pages()
8393 			     pageblock_nr_pages) - 1);  in pfn_max_align_down()
8412 		.nid = zone_to_nid(cc->zone),  in __alloc_contig_migrate_range()
8418 	while (pfn < end || !list_empty(&cc->migratepages)) {  in __alloc_contig_migrate_range()
8419 		if (fatal_signal_pending(current)) {  in __alloc_contig_migrate_range()
8420 			ret = -EINTR;  in __alloc_contig_migrate_range()
8424 		if (list_empty(&cc->migratepages)) {  in __alloc_contig_migrate_range()
8425 			cc->nr_migratepages = 0;  in __alloc_contig_migrate_range()
8428 				ret = -EINTR;  in __alloc_contig_migrate_range()
8433 			ret = ret < 0 ? ret : -EBUSY;  in __alloc_contig_migrate_range()
8437 		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,  in __alloc_contig_migrate_range()
8438 							&cc->migratepages);  in __alloc_contig_migrate_range()
8439 		cc->nr_migratepages -= nr_reclaimed;  in __alloc_contig_migrate_range()
8441 		ret = migrate_pages(&cc->migratepages, alloc_migration_target,  in __alloc_contig_migrate_range()
8442 				NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE);  in __alloc_contig_migrate_range()
8445 		putback_movable_pages(&cc->migratepages);  in __alloc_contig_migrate_range()
8452  * alloc_contig_range() -- tries to allocate given range of pages
8454  * @end:	one-past-the-last PFN to allocate
8481 		.order = -1,  in alloc_contig_range()
8521 	 * In case of -EBUSY, we'd like to know which page causes problem.  in alloc_contig_range()
8528 	 * -EBUSY is not accidentally used or returned to caller.  in alloc_contig_range()
8531 	if (ret && ret != -EBUSY)  in alloc_contig_range()
8548 	 * We don't have to hold zone->lock here because the pages are  in alloc_contig_range()
8581 		ret = -EBUSY;  in alloc_contig_range()
8588 		ret = -EBUSY;  in alloc_contig_range()
8594 		free_contig_range(outer_start, start - outer_start);  in alloc_contig_range()
8596 		free_contig_range(end, outer_end - end);  in alloc_contig_range()
8643 	unsigned long last_pfn = start_pfn + nr_pages - 1;  in zone_spans_last_pfn()
8649  * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
8651  * @gfp_mask:	GFP mask to limit search and used during compaction
8680 		spin_lock_irqsave(&zone->lock, flags);  in alloc_contig_pages()
8682 		pfn = ALIGN(zone->zone_start_pfn, nr_pages);  in alloc_contig_pages()
8692 				spin_unlock_irqrestore(&zone->lock, flags);  in alloc_contig_pages()
8697 				spin_lock_irqsave(&zone->lock, flags);  in alloc_contig_pages()
8701 		spin_unlock_irqrestore(&zone->lock, flags);  in alloc_contig_pages()
8711 	for (; nr_pages--; pfn++) {  in free_contig_range()
8740 	if (zone->pageset != &boot_pageset) {  in zone_pcp_reset()
8742 			pset = per_cpu_ptr(zone->pageset, cpu);  in zone_pcp_reset()
8745 		free_percpu(zone->pageset);  in zone_pcp_reset()
8746 		zone->pageset = &boot_pageset;  in zone_pcp_reset()
8766 	spin_lock_irqsave(&zone->lock, flags);  in __offline_isolated_pages()
8794 	spin_unlock_irqrestore(&zone->lock, flags);  in __offline_isolated_pages()
8805 	spin_lock_irqsave(&zone->lock, flags);  in is_free_buddy_page()
8807 		struct page *page_head = page - (pfn & ((1 << order) - 1));  in is_free_buddy_page()
8812 	spin_unlock_irqrestore(&zone->lock, flags);  in is_free_buddy_page()
8819  * Break down a higher-order page in sub-pages, and keep our target out of
8830 		high--;  in break_down_buddy_pages()
8863 	spin_lock_irqsave(&zone->lock, flags);  in take_page_off_buddy()
8865 		struct page *page_head = page - (pfn & ((1 << order) - 1));  in take_page_off_buddy()
8882 	spin_unlock_irqrestore(&zone->lock, flags);  in take_page_off_buddy()