42 	int (*evict)(struct z3fold_pool *pool, unsigned long handle);
73 	struct z3fold_pool *pool;  member
168 					struct z3fold_pool *pool)  in init_z3fold_page()  argument
187 	zhdr->pool = pool;  in init_z3fold_page()
251 	struct z3fold_pool *pool = zhdr->pool;  in __release_z3fold_page()  local
256 	spin_lock(&pool->lock);  in __release_z3fold_page()
259 	spin_unlock(&pool->lock);  in __release_z3fold_page()
262 	spin_lock(&pool->stale_lock);  in __release_z3fold_page()
263 	list_add(&zhdr->buddy, &pool->stale);  in __release_z3fold_page()
264 	queue_work(pool->release_wq, &pool->work);  in __release_z3fold_page()
265 	spin_unlock(&pool->stale_lock);  in __release_z3fold_page()
288 	spin_lock(&zhdr->pool->lock);  in release_z3fold_page_locked_list()
290 	spin_unlock(&zhdr->pool->lock);  in release_z3fold_page_locked_list()
298 	struct z3fold_pool *pool = container_of(w, struct z3fold_pool, work);  in free_pages_work()  local
300 	spin_lock(&pool->stale_lock);  in free_pages_work()
301 	while (!list_empty(&pool->stale)) {  in free_pages_work()
302 		struct z3fold_header *zhdr = list_first_entry(&pool->stale,  in free_pages_work()
309 		spin_unlock(&pool->stale_lock);  in free_pages_work()
313 		spin_lock(&pool->stale_lock);  in free_pages_work()
315 	spin_unlock(&pool->stale_lock);  in free_pages_work()
399 	struct z3fold_pool *pool = zhdr->pool;  in do_compact_page()  local
413 	spin_lock(&pool->lock);  in do_compact_page()
415 	spin_unlock(&pool->lock);  in do_compact_page()
418 		atomic64_dec(&pool->pages_nr);  in do_compact_page()
423 	unbuddied = get_cpu_ptr(pool->unbuddied);  in do_compact_page()
429 		spin_lock(&pool->lock);  in do_compact_page()
431 		spin_unlock(&pool->lock);  in do_compact_page()
434 	put_cpu_ptr(pool->unbuddied);  in do_compact_page()
463 	struct z3fold_pool *pool = NULL;  in z3fold_create_pool()  local
466 	pool = kzalloc(sizeof(struct z3fold_pool), gfp);  in z3fold_create_pool()
467 	if (!pool)  in z3fold_create_pool()
469 	spin_lock_init(&pool->lock);  in z3fold_create_pool()
470 	spin_lock_init(&pool->stale_lock);  in z3fold_create_pool()
471 	pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);  in z3fold_create_pool()
472 	if (!pool->unbuddied)  in z3fold_create_pool()
476 				per_cpu_ptr(pool->unbuddied, cpu);  in z3fold_create_pool()
480 	INIT_LIST_HEAD(&pool->lru);  in z3fold_create_pool()
481 	INIT_LIST_HEAD(&pool->stale);  in z3fold_create_pool()
482 	atomic64_set(&pool->pages_nr, 0);  in z3fold_create_pool()
483 	pool->name = name;  in z3fold_create_pool()
484 	pool->compact_wq = create_singlethread_workqueue(pool->name);  in z3fold_create_pool()
485 	if (!pool->compact_wq)  in z3fold_create_pool()
487 	pool->release_wq = create_singlethread_workqueue(pool->name);  in z3fold_create_pool()
488 	if (!pool->release_wq)  in z3fold_create_pool()
490 	INIT_WORK(&pool->work, free_pages_work);  in z3fold_create_pool()
491 	pool->ops = ops;  in z3fold_create_pool()
492 	return pool;  in z3fold_create_pool()
495 	destroy_workqueue(pool->compact_wq);  in z3fold_create_pool()
497 	free_percpu(pool->unbuddied);  in z3fold_create_pool()
499 	kfree(pool);  in z3fold_create_pool()
510 static void z3fold_destroy_pool(struct z3fold_pool *pool)  in z3fold_destroy_pool()  argument
512 	destroy_workqueue(pool->release_wq);  in z3fold_destroy_pool()
513 	destroy_workqueue(pool->compact_wq);  in z3fold_destroy_pool()
514 	kfree(pool);  in z3fold_destroy_pool()
536 static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp,  in z3fold_alloc()  argument
559 		unbuddied = get_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
570 			spin_lock(&pool->lock);  in z3fold_alloc()
575 				spin_unlock(&pool->lock);  in z3fold_alloc()
576 				put_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
581 			spin_unlock(&pool->lock);  in z3fold_alloc()
587 				put_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
602 		put_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
618 					atomic64_dec(&pool->pages_nr);  in z3fold_alloc()
632 		spin_lock(&pool->stale_lock);  in z3fold_alloc()
633 		zhdr = list_first_entry_or_null(&pool->stale,  in z3fold_alloc()
642 			spin_unlock(&pool->stale_lock);  in z3fold_alloc()
646 			spin_unlock(&pool->stale_lock);  in z3fold_alloc()
655 	atomic64_inc(&pool->pages_nr);  in z3fold_alloc()
656 	zhdr = init_z3fold_page(page, pool);  in z3fold_alloc()
676 		struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
680 		spin_lock(&pool->lock);  in z3fold_alloc()
682 		spin_unlock(&pool->lock);  in z3fold_alloc()
684 		put_cpu_ptr(pool->unbuddied);  in z3fold_alloc()
688 	spin_lock(&pool->lock);  in z3fold_alloc()
693 	list_add(&page->lru, &pool->lru);  in z3fold_alloc()
696 	spin_unlock(&pool->lock);  in z3fold_alloc()
713 static void z3fold_free(struct z3fold_pool *pool, unsigned long handle)  in z3fold_free()  argument
749 		spin_lock(&pool->lock);  in z3fold_free()
751 		spin_unlock(&pool->lock);  in z3fold_free()
753 		atomic64_dec(&pool->pages_nr);  in z3fold_free()
758 		atomic64_dec(&pool->pages_nr);  in z3fold_free()
770 		spin_lock(&pool->lock);  in z3fold_free()
772 		spin_unlock(&pool->lock);  in z3fold_free()
779 	queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);  in z3fold_free()
819 static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)  in z3fold_reclaim_page()  argument
827 	spin_lock(&pool->lock);  in z3fold_reclaim_page()
828 	if (!pool->ops || !pool->ops->evict || retries == 0) {  in z3fold_reclaim_page()
829 		spin_unlock(&pool->lock);  in z3fold_reclaim_page()
833 		if (list_empty(&pool->lru)) {  in z3fold_reclaim_page()
834 			spin_unlock(&pool->lock);  in z3fold_reclaim_page()
837 		list_for_each_prev(pos, &pool->lru) {  in z3fold_reclaim_page()
854 		spin_unlock(&pool->lock);  in z3fold_reclaim_page()
883 			ret = pool->ops->evict(pool, middle_handle);  in z3fold_reclaim_page()
888 			ret = pool->ops->evict(pool, first_handle);  in z3fold_reclaim_page()
893 			ret = pool->ops->evict(pool, last_handle);  in z3fold_reclaim_page()
903 			spin_lock(&pool->lock);  in z3fold_reclaim_page()
904 			list_add(&page->lru, &pool->lru);  in z3fold_reclaim_page()
905 			spin_unlock(&pool->lock);  in z3fold_reclaim_page()
911 				atomic64_dec(&pool->pages_nr);  in z3fold_reclaim_page()
919 			spin_lock(&pool->lock);  in z3fold_reclaim_page()
920 			list_add(&page->lru, &pool->lru);  in z3fold_reclaim_page()
921 			spin_unlock(&pool->lock);  in z3fold_reclaim_page()
926 		spin_lock(&pool->lock);  in z3fold_reclaim_page()
928 	spin_unlock(&pool->lock);  in z3fold_reclaim_page()
942 static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle)  in z3fold_map()  argument
986 static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle)  in z3fold_unmap()  argument
1011 static u64 z3fold_get_pool_size(struct z3fold_pool *pool)  in z3fold_get_pool_size()  argument
1013 	return atomic64_read(&pool->pages_nr);  in z3fold_get_pool_size()
1020 static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle)  in z3fold_zpool_evict()  argument
1022 	if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict)  in z3fold_zpool_evict()
1023 		return pool->zpool_ops->evict(pool->zpool, handle);  in z3fold_zpool_evict()
1036 	struct z3fold_pool *pool;  in z3fold_zpool_create()  local
1038 	pool = z3fold_create_pool(name, gfp,  in z3fold_zpool_create()
1040 	if (pool) {  in z3fold_zpool_create()
1041 		pool->zpool = zpool;  in z3fold_zpool_create()
1042 		pool->zpool_ops = zpool_ops;  in z3fold_zpool_create()
1044 	return pool;  in z3fold_zpool_create()
1047 static void z3fold_zpool_destroy(void *pool)  in z3fold_zpool_destroy()  argument
1049 	z3fold_destroy_pool(pool);  in z3fold_zpool_destroy()
1052 static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp,  in z3fold_zpool_malloc()  argument
1055 	return z3fold_alloc(pool, size, gfp, handle);  in z3fold_zpool_malloc()
1057 static void z3fold_zpool_free(void *pool, unsigned long handle)  in z3fold_zpool_free()  argument
1059 	z3fold_free(pool, handle);  in z3fold_zpool_free()
1062 static int z3fold_zpool_shrink(void *pool, unsigned int pages,  in z3fold_zpool_shrink()  argument
1069 		ret = z3fold_reclaim_page(pool, 8);  in z3fold_zpool_shrink()
1081 static void *z3fold_zpool_map(void *pool, unsigned long handle,  in z3fold_zpool_map()  argument
1084 	return z3fold_map(pool, handle);  in z3fold_zpool_map()
1086 static void z3fold_zpool_unmap(void *pool, unsigned long handle)  in z3fold_zpool_unmap()  argument
1088 	z3fold_unmap(pool, handle);  in z3fold_zpool_unmap()
1091 static u64 z3fold_zpool_total_size(void *pool)  in z3fold_zpool_total_size()  argument
1093 	return z3fold_get_pool_size(pool) * PAGE_SIZE;  in z3fold_zpool_total_size()