166 	struct zswap_pool *pool;  member
213 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
214 static int zswap_pool_get(struct zswap_pool *pool);
215 static void zswap_pool_put(struct zswap_pool *pool);
236 	struct zswap_pool *pool;  in zswap_update_total_size()  local
241 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_update_total_size()
242 		total += zpool_get_total_size(pool->zpool);  in zswap_update_total_size()
345 		zpool_free(entry->pool->zpool, entry->handle);  in zswap_free_entry()
346 		zswap_pool_put(entry->pool);  in zswap_free_entry()
417 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_prepare()  local
420 	if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))  in zswap_cpu_comp_prepare()
423 	tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);  in zswap_cpu_comp_prepare()
426 		       pool->tfm_name, PTR_ERR(tfm));  in zswap_cpu_comp_prepare()
429 	*per_cpu_ptr(pool->tfm, cpu) = tfm;  in zswap_cpu_comp_prepare()
435 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_dead()  local
438 	tfm = *per_cpu_ptr(pool->tfm, cpu);  in zswap_cpu_comp_dead()
441 	*per_cpu_ptr(pool->tfm, cpu) = NULL;  in zswap_cpu_comp_dead()
451 	struct zswap_pool *pool;  in __zswap_pool_current()  local
453 	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);  in __zswap_pool_current()
454 	WARN_ONCE(!pool && zswap_has_pool,  in __zswap_pool_current()
457 	return pool;  in __zswap_pool_current()
469 	struct zswap_pool *pool;  in zswap_pool_current_get()  local
473 	pool = __zswap_pool_current();  in zswap_pool_current_get()
474 	if (!zswap_pool_get(pool))  in zswap_pool_current_get()
475 		pool = NULL;  in zswap_pool_current_get()
479 	return pool;  in zswap_pool_current_get()
484 	struct zswap_pool *pool, *last = NULL;  in zswap_pool_last_get()  local
488 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_pool_last_get()
489 		last = pool;  in zswap_pool_last_get()
503 	struct zswap_pool *pool;  in zswap_pool_find_get()  local
507 	list_for_each_entry_rcu(pool, &zswap_pools, list) {  in zswap_pool_find_get()
508 		if (strcmp(pool->tfm_name, compressor))  in zswap_pool_find_get()
510 		if (strcmp(zpool_get_type(pool->zpool), type))  in zswap_pool_find_get()
513 		if (!zswap_pool_get(pool))  in zswap_pool_find_get()
515 		return pool;  in zswap_pool_find_get()
523 	struct zswap_pool *pool = container_of(w, typeof(*pool),  in shrink_worker()  local
526 	if (zpool_shrink(pool->zpool, 1, NULL))  in shrink_worker()
528 	zswap_pool_put(pool);  in shrink_worker()
533 	struct zswap_pool *pool;  in zswap_pool_create()  local
549 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in zswap_pool_create()
550 	if (!pool)  in zswap_pool_create()
556 	pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);  in zswap_pool_create()
557 	if (!pool->zpool) {  in zswap_pool_create()
561 	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));  in zswap_pool_create()
563 	strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));  in zswap_pool_create()
564 	pool->tfm = alloc_percpu(struct crypto_comp *);  in zswap_pool_create()
565 	if (!pool->tfm) {  in zswap_pool_create()
571 				       &pool->node);  in zswap_pool_create()
574 	pr_debug("using %s compressor\n", pool->tfm_name);  in zswap_pool_create()
579 	kref_init(&pool->kref);  in zswap_pool_create()
580 	INIT_LIST_HEAD(&pool->list);  in zswap_pool_create()
581 	INIT_WORK(&pool->shrink_work, shrink_worker);  in zswap_pool_create()
583 	zswap_pool_debug("created", pool);  in zswap_pool_create()
585 	return pool;  in zswap_pool_create()
588 	free_percpu(pool->tfm);  in zswap_pool_create()
589 	if (pool->zpool)  in zswap_pool_create()
590 		zpool_destroy_pool(pool->zpool);  in zswap_pool_create()
591 	kfree(pool);  in zswap_pool_create()
637 static void zswap_pool_destroy(struct zswap_pool *pool)  in zswap_pool_destroy()  argument
639 	zswap_pool_debug("destroying", pool);  in zswap_pool_destroy()
641 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);  in zswap_pool_destroy()
642 	free_percpu(pool->tfm);  in zswap_pool_destroy()
643 	zpool_destroy_pool(pool->zpool);  in zswap_pool_destroy()
644 	kfree(pool);  in zswap_pool_destroy()
647 static int __must_check zswap_pool_get(struct zswap_pool *pool)  in zswap_pool_get()  argument
649 	if (!pool)  in zswap_pool_get()
652 	return kref_get_unless_zero(&pool->kref);  in zswap_pool_get()
657 	struct zswap_pool *pool = container_of(work, typeof(*pool),  in __zswap_pool_release()  local
663 	WARN_ON(kref_get_unless_zero(&pool->kref));  in __zswap_pool_release()
666 	zswap_pool_destroy(pool);  in __zswap_pool_release()
671 	struct zswap_pool *pool;  in __zswap_pool_empty()  local
673 	pool = container_of(kref, typeof(*pool), kref);  in __zswap_pool_empty()
677 	WARN_ON(pool == zswap_pool_current());  in __zswap_pool_empty()
679 	list_del_rcu(&pool->list);  in __zswap_pool_empty()
681 	INIT_WORK(&pool->release_work, __zswap_pool_release);  in __zswap_pool_empty()
682 	schedule_work(&pool->release_work);  in __zswap_pool_empty()
687 static void zswap_pool_put(struct zswap_pool *pool)  in zswap_pool_put()  argument
689 	kref_put(&pool->kref, __zswap_pool_empty);  in zswap_pool_put()
700 	struct zswap_pool *pool, *put_pool = NULL;  in __zswap_param_set()  local
738 	pool = zswap_pool_find_get(type, compressor);  in __zswap_param_set()
739 	if (pool) {  in __zswap_param_set()
740 		zswap_pool_debug("using existing", pool);  in __zswap_param_set()
741 		WARN_ON(pool == zswap_pool_current());  in __zswap_param_set()
742 		list_del_rcu(&pool->list);  in __zswap_param_set()
747 	if (!pool)  in __zswap_param_set()
748 		pool = zswap_pool_create(type, compressor);  in __zswap_param_set()
750 	if (pool)  in __zswap_param_set()
759 		list_add_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
761 	} else if (pool) {  in __zswap_param_set()
766 		list_add_tail_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
767 		put_pool = pool;  in __zswap_param_set()
772 	if (!zswap_has_pool && !pool) {  in __zswap_param_set()
871 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)  in zswap_writeback_entry()  argument
888 	zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);  in zswap_writeback_entry()
899 		zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
922 		tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_writeback_entry()
925 		put_cpu_ptr(entry->pool->tfm);  in zswap_writeback_entry()
972 	zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
1029 		struct zswap_pool *pool;  in zswap_frontswap_store()  local
1033 		pool = zswap_pool_last_get();  in zswap_frontswap_store()
1034 		if (pool)  in zswap_frontswap_store()
1035 			queue_work(shrink_wq, &pool->shrink_work);  in zswap_frontswap_store()
1070 	entry->pool = zswap_pool_current_get();  in zswap_frontswap_store()
1071 	if (!entry->pool) {  in zswap_frontswap_store()
1078 	tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_store()
1082 	put_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_store()
1089 	hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;  in zswap_frontswap_store()
1091 	if (zpool_malloc_support_movable(entry->pool->zpool))  in zswap_frontswap_store()
1093 	ret = zpool_malloc(entry->pool->zpool, hlen + dlen, gfp, &handle);  in zswap_frontswap_store()
1102 	buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);  in zswap_frontswap_store()
1105 	zpool_unmap_handle(entry->pool->zpool, handle);  in zswap_frontswap_store()
1135 	zswap_pool_put(entry->pool);  in zswap_frontswap_store()
1175 	src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);  in zswap_frontswap_load()
1176 	if (zpool_evictable(entry->pool->zpool))  in zswap_frontswap_load()
1179 	tfm = *get_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_load()
1181 	put_cpu_ptr(entry->pool->tfm);  in zswap_frontswap_load()
1183 	zpool_unmap_handle(entry->pool->zpool, entry->handle);  in zswap_frontswap_load()
1317 	struct zswap_pool *pool;  in init_zswap()  local
1341 	pool = __zswap_pool_create_fallback();  in init_zswap()
1342 	if (pool) {  in init_zswap()
1343 		pr_info("loaded using pool %s/%s\n", pool->tfm_name,  in init_zswap()
1344 			zpool_get_type(pool->zpool));  in init_zswap()
1345 		list_add(&pool->list, &zswap_pools);  in init_zswap()
1362 	if (pool)  in init_zswap()
1363 		zswap_pool_destroy(pool);  in init_zswap()