Lines Matching refs:pcp
1547 struct per_cpu_pages *pcp, in free_pcppages_bulk() argument
1560 count = min(pcp->count, count); in free_pcppages_bulk()
1577 list = &pcp->lists[pindex]; in free_pcppages_bulk()
1598 pcp->count -= nr_pages; in free_pcppages_bulk()
3175 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) in drain_zone_pages() argument
3179 batch = READ_ONCE(pcp->batch); in drain_zone_pages()
3180 to_drain = min(pcp->count, batch); in drain_zone_pages()
3189 spin_lock_irqsave(&pcp->lock, flags); in drain_zone_pages()
3190 free_pcppages_bulk(zone, to_drain, pcp, 0); in drain_zone_pages()
3191 spin_unlock_irqrestore(&pcp->lock, flags); in drain_zone_pages()
3201 struct per_cpu_pages *pcp; in drain_pages_zone() local
3203 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in drain_pages_zone()
3204 if (pcp->count) { in drain_pages_zone()
3208 spin_lock_irqsave(&pcp->lock, flags); in drain_pages_zone()
3209 free_pcppages_bulk(zone, pcp->count, pcp, 0); in drain_pages_zone()
3210 spin_unlock_irqrestore(&pcp->lock, flags); in drain_pages_zone()
3277 struct per_cpu_pages *pcp; in __drain_all_pages() local
3288 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __drain_all_pages()
3289 if (pcp->count) in __drain_all_pages()
3293 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); in __drain_all_pages()
3294 if (pcp->count) { in __drain_all_pages()
3395 static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch, in nr_pcp_free() argument
3402 return pcp->count; in nr_pcp_free()
3416 batch <<= pcp->free_factor; in nr_pcp_free()
3418 pcp->free_factor++; in nr_pcp_free()
3424 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, in nr_pcp_high() argument
3427 int high = READ_ONCE(pcp->high); in nr_pcp_high()
3439 return min(READ_ONCE(pcp->batch) << 2, high); in nr_pcp_high()
3442 static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, in free_unref_page_commit() argument
3452 list_add(&page->pcp_list, &pcp->lists[pindex]); in free_unref_page_commit()
3453 pcp->count += 1 << order; in free_unref_page_commit()
3461 free_high = (pcp->free_factor && order && order <= PAGE_ALLOC_COSTLY_ORDER); in free_unref_page_commit()
3463 high = nr_pcp_high(pcp, zone, free_high); in free_unref_page_commit()
3464 if (pcp->count >= high) { in free_unref_page_commit()
3465 int batch = READ_ONCE(pcp->batch); in free_unref_page_commit()
3467 free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch, free_high), pcp, pindex); in free_unref_page_commit()
3478 struct per_cpu_pages *pcp; in free_unref_page() local
3504 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); in free_unref_page()
3505 if (pcp) { in free_unref_page()
3506 free_unref_page_commit(zone, pcp, page, migratetype, order); in free_unref_page()
3507 pcp_spin_unlock_irqrestore(pcp, flags); in free_unref_page()
3520 struct per_cpu_pages *pcp = NULL; in free_unref_page_list() local
3551 if (pcp) in free_unref_page_list()
3552 pcp_spin_unlock_irqrestore(pcp, flags); in free_unref_page_list()
3555 pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); in free_unref_page_list()
3567 free_unref_page_commit(zone, pcp, page, migratetype, 0); in free_unref_page_list()
3574 pcp_spin_unlock_irqrestore(pcp, flags); in free_unref_page_list()
3576 pcp = pcp_spin_lock_irqsave(locked_zone->per_cpu_pageset, flags); in free_unref_page_list()
3580 if (pcp) in free_unref_page_list()
3581 pcp_spin_unlock_irqrestore(pcp, flags); in free_unref_page_list()
3738 struct per_cpu_pages *pcp, in __rmqueue_pcplist() argument
3745 int batch = READ_ONCE(pcp->batch); in __rmqueue_pcplist()
3761 pcp->count += alloced << order; in __rmqueue_pcplist()
3768 pcp->count -= 1 << order; in __rmqueue_pcplist()
3779 struct per_cpu_pages *pcp; in rmqueue_pcplist() local
3790 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); in rmqueue_pcplist()
3791 if (!pcp) { in rmqueue_pcplist()
3801 pcp->free_factor >>= 1; in rmqueue_pcplist()
3802 list = &pcp->lists[order_to_pindex(migratetype, order)]; in rmqueue_pcplist()
3803 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); in rmqueue_pcplist()
3804 pcp_spin_unlock_irqrestore(pcp, flags); in rmqueue_pcplist()
5379 struct per_cpu_pages *pcp; in __alloc_pages_bulk() local
5459 pcp = pcp_spin_trylock_irqsave(zone->per_cpu_pageset, flags); in __alloc_pages_bulk()
5460 if (!pcp) in __alloc_pages_bulk()
5464 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; in __alloc_pages_bulk()
5474 pcp, pcp_list); in __alloc_pages_bulk()
5478 pcp_spin_unlock_irqrestore(pcp, flags); in __alloc_pages_bulk()
5493 pcp_spin_unlock_irqrestore(pcp, flags); in __alloc_pages_bulk()
6585 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
7177 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, in pageset_update() argument
7180 WRITE_ONCE(pcp->batch, batch); in pageset_update()
7181 WRITE_ONCE(pcp->high, high); in pageset_update()
7184 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) in per_cpu_pages_init() argument
7188 memset(pcp, 0, sizeof(*pcp)); in per_cpu_pages_init()
7191 spin_lock_init(&pcp->lock); in per_cpu_pages_init()
7193 INIT_LIST_HEAD(&pcp->lists[pindex]); in per_cpu_pages_init()
7201 pcp->high = BOOT_PAGESET_HIGH; in per_cpu_pages_init()
7202 pcp->batch = BOOT_PAGESET_BATCH; in per_cpu_pages_init()
7203 pcp->free_factor = 0; in per_cpu_pages_init()
7209 struct per_cpu_pages *pcp; in __zone_set_pageset_high_and_batch() local
7213 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __zone_set_pageset_high_and_batch()
7214 pageset_update(pcp, high, batch); in __zone_set_pageset_high_and_batch()
7249 struct per_cpu_pages *pcp; in setup_zone_pageset() local
7252 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in setup_zone_pageset()
7254 per_cpu_pages_init(pcp, pzstats); in setup_zone_pageset()