165 	struct zswap_pool *pool;  member
212 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
213 static int zswap_pool_get(struct zswap_pool *pool);
214 static void zswap_pool_put(struct zswap_pool *pool);
228 	struct zswap_pool *pool;  in zswap_update_total_size()  local
233 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_update_total_size()
234 		total += zpool_get_total_size(pool->zpool);  in zswap_update_total_size()
337 		zpool_free(entry->pool->zpool, entry->handle);  in zswap_free_entry()
338 		zswap_pool_put(entry->pool);  in zswap_free_entry()
409 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_prepare()  local
412 	if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))  in zswap_cpu_comp_prepare()
415 	tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);  in zswap_cpu_comp_prepare()
418 		       pool->tfm_name, PTR_ERR(tfm));  in zswap_cpu_comp_prepare()
421 	*per_cpu_ptr(pool->tfm, cpu) = tfm;  in zswap_cpu_comp_prepare()
427 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_dead()  local
430 	tfm = *per_cpu_ptr(pool->tfm, cpu);  in zswap_cpu_comp_dead()
433 	*per_cpu_ptr(pool->tfm, cpu) = NULL;  in zswap_cpu_comp_dead()
443 	struct zswap_pool *pool;  in __zswap_pool_current()  local
445 	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);  in __zswap_pool_current()
446 	WARN_ONCE(!pool && zswap_has_pool,  in __zswap_pool_current()
449 	return pool;  in __zswap_pool_current()
461 	struct zswap_pool *pool;  in zswap_pool_current_get()  local
465 	pool = __zswap_pool_current();  in zswap_pool_current_get()
466 	if (!zswap_pool_get(pool))  in zswap_pool_current_get()
467 		pool = NULL;  in zswap_pool_current_get()
471 	return pool;  in zswap_pool_current_get()
476 	struct zswap_pool *pool, *last = NULL;  in zswap_pool_last_get()  local
480 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_pool_last_get()
481 		last = pool;  in zswap_pool_last_get()
495 	struct zswap_pool *pool;  in zswap_pool_find_get()  local
499 	list_for_each_entry_rcu(pool, &zswap_pools, list) {  in zswap_pool_find_get()
500 		if (strcmp(pool->tfm_name, compressor))  in zswap_pool_find_get()
502 		if (strcmp(zpool_get_type(pool->zpool), type))  in zswap_pool_find_get()
505 		if (!zswap_pool_get(pool))  in zswap_pool_find_get()
507 		return pool;  in zswap_pool_find_get()
515 	struct zswap_pool *pool;  in zswap_pool_create()  local
531 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in zswap_pool_create()
532 	if (!pool)  in zswap_pool_create()
538 	pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);  in zswap_pool_create()
539 	if (!pool->zpool) {  in zswap_pool_create()
543 	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));  in zswap_pool_create()
545 	strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));  in zswap_pool_create()
546 	pool->tfm = alloc_percpu(struct crypto_comp *);  in zswap_pool_create()
547 	if (!pool->tfm) {  in zswap_pool_create()
553 				       &pool->node);  in zswap_pool_create()
556 	pr_debug("using %s compressor\n", pool->tfm_name);  in zswap_pool_create()
561 	kref_init(&pool->kref);  in zswap_pool_create()
562 	INIT_LIST_HEAD(&pool->list);  in zswap_pool_create()
564 	zswap_pool_debug("created", pool);  in zswap_pool_create()
566 	return pool;  in zswap_pool_create()
569 	free_percpu(pool->tfm);  in zswap_pool_create()
570 	if (pool->zpool)  in zswap_pool_create()
571 		zpool_destroy_pool(pool->zpool);  in zswap_pool_create()
572 	kfree(pool);  in zswap_pool_create()
616 static void zswap_pool_destroy(struct zswap_pool *pool)  in zswap_pool_destroy()  argument
618 	zswap_pool_debug("destroying", pool);  in zswap_pool_destroy()
620 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);  in zswap_pool_destroy()
621 	free_percpu(pool->tfm);  in zswap_pool_destroy()
622 	zpool_destroy_pool(pool->zpool);  in zswap_pool_destroy()
623 	kfree(pool);  in zswap_pool_destroy()
626 static int __must_check zswap_pool_get(struct zswap_pool *pool)  in zswap_pool_get()  argument
628 	if (!pool)  in zswap_pool_get()
631 	return kref_get_unless_zero(&pool->kref);  in zswap_pool_get()
636 	struct zswap_pool *pool = container_of(work, typeof(*pool), work);  in __zswap_pool_release()  local
641 	WARN_ON(kref_get_unless_zero(&pool->kref));  in __zswap_pool_release()
644 	zswap_pool_destroy(pool);  in __zswap_pool_release()
649 	struct zswap_pool *pool;  in __zswap_pool_empty()  local
651 	pool = container_of(kref, typeof(*pool), kref);  in __zswap_pool_empty()
655 	WARN_ON(pool == zswap_pool_current());  in __zswap_pool_empty()
657 	list_del_rcu(&pool->list);  in __zswap_pool_empty()
659 	INIT_WORK(&pool->work, __zswap_pool_release);  in __zswap_pool_empty()
660 	schedule_work(&pool->work);  in __zswap_pool_empty()
665 static void zswap_pool_put(struct zswap_pool *pool)  in zswap_pool_put()  argument
667 	kref_put(&pool->kref, __zswap_pool_empty);  in zswap_pool_put()
678 	struct zswap_pool *pool, *put_pool = NULL;  in __zswap_param_set()  local
716 	pool = zswap_pool_find_get(type, compressor);  in __zswap_param_set()
717 	if (pool) {  in __zswap_param_set()
718 		zswap_pool_debug("using existing", pool);  in __zswap_param_set()
719 		WARN_ON(pool == zswap_pool_current());  in __zswap_param_set()
720 		list_del_rcu(&pool->list);  in __zswap_param_set()
725 	if (!pool)  in __zswap_param_set()
726 		pool = zswap_pool_create(type, compressor);  in __zswap_param_set()
728 	if (pool)  in __zswap_param_set()
737 		list_add_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
739 	} else if (pool) {  in __zswap_param_set()
744 		list_add_tail_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
745 		put_pool = pool;  in __zswap_param_set()
750 	if (!zswap_has_pool && !pool) {  in __zswap_param_set()
849 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)  in zswap_writeback_entry()  argument
866 	zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);  in zswap_writeback_entry()
868 	zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
898 		src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,  in zswap_writeback_entry()
901 		tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_writeback_entry()
904 		put_cpu_ptr(entry->pool->tfm);  in zswap_writeback_entry()
906 		zpool_unmap_handle(entry->pool->zpool, entry->handle);  in zswap_writeback_entry()
957 	struct zswap_pool *pool;  in zswap_shrink()  local
960 	pool = zswap_pool_last_get();  in zswap_shrink()
961 	if (!pool)  in zswap_shrink()
964 	ret = zpool_shrink(pool->zpool, 1, NULL);  in zswap_shrink()
966 	zswap_pool_put(pool);  in zswap_shrink()
1062 	entry->pool = zswap_pool_current_get();  in zswap_frontswap_store()
1063 	if (!entry->pool) {  in zswap_frontswap_store()
1070 	tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_store()
1074 	put_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_store()
1081 	hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;  in zswap_frontswap_store()
1082 	ret = zpool_malloc(entry->pool->zpool, hlen + dlen,  in zswap_frontswap_store()
1093 	buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);  in zswap_frontswap_store()
1096 	zpool_unmap_handle(entry->pool->zpool, handle);  in zswap_frontswap_store()
1126 	zswap_pool_put(entry->pool);  in zswap_frontswap_store()
1166 	src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);  in zswap_frontswap_load()
1167 	if (zpool_evictable(entry->pool->zpool))  in zswap_frontswap_load()
1170 	tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_load()
1172 	put_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_load()
1174 	zpool_unmap_handle(entry->pool->zpool, entry->handle);  in zswap_frontswap_load()
1310 	struct zswap_pool *pool;  in init_zswap()  local
1334 	pool = __zswap_pool_create_fallback();  in init_zswap()
1335 	if (pool) {  in init_zswap()
1336 		pr_info("loaded using pool %s/%s\n", pool->tfm_name,  in init_zswap()
1337 			zpool_get_type(pool->zpool));  in init_zswap()
1338 		list_add(&pool->list, &zswap_pools);  in init_zswap()