1443 					struct per_cpu_pages *pcp)  in free_pcppages_bulk()  argument
1449 	int prefetch_nr = READ_ONCE(pcp->batch);  in free_pcppages_bulk()
1458 	count = min(pcp->count, count);  in free_pcppages_bulk()
1473 			list = &pcp->lists[pindex];  in free_pcppages_bulk()
1513 	pcp->count -= nr_freed;  in free_pcppages_bulk()
3072 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)  in drain_zone_pages()  argument
3078 	batch = READ_ONCE(pcp->batch);  in drain_zone_pages()
3079 	to_drain = min(pcp->count, batch);  in drain_zone_pages()
3081 		free_pcppages_bulk(zone, to_drain, pcp);  in drain_zone_pages()
3096 	struct per_cpu_pages *pcp;  in drain_pages_zone()  local
3100 	pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in drain_pages_zone()
3101 	if (pcp->count)  in drain_pages_zone()
3102 		free_pcppages_bulk(zone, pcp->count, pcp);  in drain_pages_zone()
3202 		struct per_cpu_pages *pcp;  in __drain_all_pages()  local
3213 			pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __drain_all_pages()
3214 			if (pcp->count)  in __drain_all_pages()
3218 				pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);  in __drain_all_pages()
3219 				if (pcp->count) {  in __drain_all_pages()
3325 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch)  in nr_pcp_free()  argument
3341 	batch <<= pcp->free_factor;  in nr_pcp_free()
3343 		pcp->free_factor++;  in nr_pcp_free()
3349 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone)  in nr_pcp_high()  argument
3351 	int high = READ_ONCE(pcp->high);  in nr_pcp_high()
3363 	return min(READ_ONCE(pcp->batch) << 2, high);  in nr_pcp_high()
3370 	struct per_cpu_pages *pcp;  in free_unref_page_commit()  local
3375 	pcp = this_cpu_ptr(zone->per_cpu_pageset);  in free_unref_page_commit()
3377 	list_add(&page->lru, &pcp->lists[pindex]);  in free_unref_page_commit()
3378 	pcp->count += 1 << order;  in free_unref_page_commit()
3379 	high = nr_pcp_high(pcp, zone);  in free_unref_page_commit()
3380 	if (pcp->count >= high) {  in free_unref_page_commit()
3381 		int batch = READ_ONCE(pcp->batch);  in free_unref_page_commit()
3383 		free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp);  in free_unref_page_commit()
3605 			struct per_cpu_pages *pcp,  in __rmqueue_pcplist()  argument
3612 			int batch = READ_ONCE(pcp->batch);  in __rmqueue_pcplist()
3628 			pcp->count += alloced << order;  in __rmqueue_pcplist()
3635 		pcp->count -= 1 << order;  in __rmqueue_pcplist()
3647 	struct per_cpu_pages *pcp;  in rmqueue_pcplist()  local
3659 	pcp = this_cpu_ptr(zone->per_cpu_pageset);  in rmqueue_pcplist()
3660 	pcp->free_factor >>= 1;  in rmqueue_pcplist()
3661 	list = &pcp->lists[order_to_pindex(migratetype, order)];  in rmqueue_pcplist()
3662 	page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);  in rmqueue_pcplist()
5206 	struct per_cpu_pages *pcp;  in __alloc_pages_bulk()  local
5286 	pcp = this_cpu_ptr(zone->per_cpu_pageset);  in __alloc_pages_bulk()
5287 	pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)];  in __alloc_pages_bulk()
5298 								pcp, pcp_list);  in __alloc_pages_bulk()
6351 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
6874 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,  in pageset_update()  argument
6877 	WRITE_ONCE(pcp->batch, batch);  in pageset_update()
6878 	WRITE_ONCE(pcp->high, high);  in pageset_update()
6881 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats)  in per_cpu_pages_init()  argument
6885 	memset(pcp, 0, sizeof(*pcp));  in per_cpu_pages_init()
6889 		INIT_LIST_HEAD(&pcp->lists[pindex]);  in per_cpu_pages_init()
6897 	pcp->high = BOOT_PAGESET_HIGH;  in per_cpu_pages_init()
6898 	pcp->batch = BOOT_PAGESET_BATCH;  in per_cpu_pages_init()
6899 	pcp->free_factor = 0;  in per_cpu_pages_init()
6905 	struct per_cpu_pages *pcp;  in __zone_set_pageset_high_and_batch()  local
6909 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in __zone_set_pageset_high_and_batch()
6910 		pageset_update(pcp, high, batch);  in __zone_set_pageset_high_and_batch()
6945 		struct per_cpu_pages *pcp;  in setup_zone_pageset()  local
6948 		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);  in setup_zone_pageset()
6950 		per_cpu_pages_init(pcp, pzstats);  in setup_zone_pageset()