176 	struct zswap_pool *pool;  member
223 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
224 static int zswap_pool_get(struct zswap_pool *pool);
225 static void zswap_pool_put(struct zswap_pool *pool);
246 	struct zswap_pool *pool;  in zswap_update_total_size()  local
251 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_update_total_size()
252 		total += zpool_get_total_size(pool->zpool);  in zswap_update_total_size()
355 		zpool_free(entry->pool->zpool, entry->handle);  in zswap_free_entry()
356 		zswap_pool_put(entry->pool);  in zswap_free_entry()
447 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_prepare()  local
448 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);  in zswap_cpu_comp_prepare()
452 	acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));  in zswap_cpu_comp_prepare()
455 				pool->tfm_name, PTR_ERR(acomp));  in zswap_cpu_comp_prepare()
463 		       pool->tfm_name);  in zswap_cpu_comp_prepare()
486 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);  in zswap_cpu_comp_dead()  local
487 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);  in zswap_cpu_comp_dead()
505 	struct zswap_pool *pool;  in __zswap_pool_current()  local
507 	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);  in __zswap_pool_current()
508 	WARN_ONCE(!pool && zswap_has_pool,  in __zswap_pool_current()
511 	return pool;  in __zswap_pool_current()
523 	struct zswap_pool *pool;  in zswap_pool_current_get()  local
527 	pool = __zswap_pool_current();  in zswap_pool_current_get()
528 	if (!zswap_pool_get(pool))  in zswap_pool_current_get()
529 		pool = NULL;  in zswap_pool_current_get()
533 	return pool;  in zswap_pool_current_get()
538 	struct zswap_pool *pool, *last = NULL;  in zswap_pool_last_get()  local
542 	list_for_each_entry_rcu(pool, &zswap_pools, list)  in zswap_pool_last_get()
543 		last = pool;  in zswap_pool_last_get()
557 	struct zswap_pool *pool;  in zswap_pool_find_get()  local
561 	list_for_each_entry_rcu(pool, &zswap_pools, list) {  in zswap_pool_find_get()
562 		if (strcmp(pool->tfm_name, compressor))  in zswap_pool_find_get()
564 		if (strcmp(zpool_get_type(pool->zpool), type))  in zswap_pool_find_get()
567 		if (!zswap_pool_get(pool))  in zswap_pool_find_get()
569 		return pool;  in zswap_pool_find_get()
577 	struct zswap_pool *pool = container_of(w, typeof(*pool),  in shrink_worker()  local
580 	if (zpool_shrink(pool->zpool, 1, NULL))  in shrink_worker()
582 	zswap_pool_put(pool);  in shrink_worker()
587 	struct zswap_pool *pool;  in zswap_pool_create()  local
603 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in zswap_pool_create()
604 	if (!pool)  in zswap_pool_create()
610 	pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);  in zswap_pool_create()
611 	if (!pool->zpool) {  in zswap_pool_create()
615 	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));  in zswap_pool_create()
617 	strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));  in zswap_pool_create()
619 	pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);  in zswap_pool_create()
620 	if (!pool->acomp_ctx) {  in zswap_pool_create()
626 				       &pool->node);  in zswap_pool_create()
629 	pr_debug("using %s compressor\n", pool->tfm_name);  in zswap_pool_create()
634 	kref_init(&pool->kref);  in zswap_pool_create()
635 	INIT_LIST_HEAD(&pool->list);  in zswap_pool_create()
636 	INIT_WORK(&pool->shrink_work, shrink_worker);  in zswap_pool_create()
638 	zswap_pool_debug("created", pool);  in zswap_pool_create()
640 	return pool;  in zswap_pool_create()
643 	if (pool->acomp_ctx)  in zswap_pool_create()
644 		free_percpu(pool->acomp_ctx);  in zswap_pool_create()
645 	if (pool->zpool)  in zswap_pool_create()
646 		zpool_destroy_pool(pool->zpool);  in zswap_pool_create()
647 	kfree(pool);  in zswap_pool_create()
693 static void zswap_pool_destroy(struct zswap_pool *pool)  in zswap_pool_destroy()  argument
695 	zswap_pool_debug("destroying", pool);  in zswap_pool_destroy()
697 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);  in zswap_pool_destroy()
698 	free_percpu(pool->acomp_ctx);  in zswap_pool_destroy()
699 	zpool_destroy_pool(pool->zpool);  in zswap_pool_destroy()
700 	kfree(pool);  in zswap_pool_destroy()
703 static int __must_check zswap_pool_get(struct zswap_pool *pool)  in zswap_pool_get()  argument
705 	if (!pool)  in zswap_pool_get()
708 	return kref_get_unless_zero(&pool->kref);  in zswap_pool_get()
713 	struct zswap_pool *pool = container_of(work, typeof(*pool),  in __zswap_pool_release()  local
719 	WARN_ON(kref_get_unless_zero(&pool->kref));  in __zswap_pool_release()
722 	zswap_pool_destroy(pool);  in __zswap_pool_release()
727 	struct zswap_pool *pool;  in __zswap_pool_empty()  local
729 	pool = container_of(kref, typeof(*pool), kref);  in __zswap_pool_empty()
733 	WARN_ON(pool == zswap_pool_current());  in __zswap_pool_empty()
735 	list_del_rcu(&pool->list);  in __zswap_pool_empty()
737 	INIT_WORK(&pool->release_work, __zswap_pool_release);  in __zswap_pool_empty()
738 	schedule_work(&pool->release_work);  in __zswap_pool_empty()
743 static void zswap_pool_put(struct zswap_pool *pool)  in zswap_pool_put()  argument
745 	kref_put(&pool->kref, __zswap_pool_empty);  in zswap_pool_put()
756 	struct zswap_pool *pool, *put_pool = NULL;  in __zswap_param_set()  local
794 	pool = zswap_pool_find_get(type, compressor);  in __zswap_param_set()
795 	if (pool) {  in __zswap_param_set()
796 		zswap_pool_debug("using existing", pool);  in __zswap_param_set()
797 		WARN_ON(pool == zswap_pool_current());  in __zswap_param_set()
798 		list_del_rcu(&pool->list);  in __zswap_param_set()
803 	if (!pool)  in __zswap_param_set()
804 		pool = zswap_pool_create(type, compressor);  in __zswap_param_set()
806 	if (pool)  in __zswap_param_set()
815 		list_add_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
817 	} else if (pool) {  in __zswap_param_set()
822 		list_add_tail_rcu(&pool->list, &zswap_pools);  in __zswap_param_set()
823 		put_pool = pool;  in __zswap_param_set()
828 	if (!zswap_has_pool && !pool) {  in __zswap_param_set()
927 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)  in zswap_writeback_entry()  argument
945 	if (!zpool_can_sleep_mapped(pool)) {  in zswap_writeback_entry()
952 	zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);  in zswap_writeback_entry()
963 		zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
971 	if (!zpool_can_sleep_mapped(pool)) {  in zswap_writeback_entry()
974 		zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
991 		acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);  in zswap_writeback_entry()
1048 	if (zpool_can_sleep_mapped(pool))  in zswap_writeback_entry()
1049 		zpool_unmap_handle(pool, handle);  in zswap_writeback_entry()
1110 		struct zswap_pool *pool;  in zswap_frontswap_store()  local
1114 		pool = zswap_pool_last_get();  in zswap_frontswap_store()
1115 		if (pool)  in zswap_frontswap_store()
1116 			queue_work(shrink_wq, &pool->shrink_work);  in zswap_frontswap_store()
1151 	entry->pool = zswap_pool_current_get();  in zswap_frontswap_store()
1152 	if (!entry->pool) {  in zswap_frontswap_store()
1158 	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);  in zswap_frontswap_store()
1190 	hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;  in zswap_frontswap_store()
1192 	if (zpool_malloc_support_movable(entry->pool->zpool))  in zswap_frontswap_store()
1194 	ret = zpool_malloc(entry->pool->zpool, hlen + dlen, gfp, &handle);  in zswap_frontswap_store()
1203 	buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_WO);  in zswap_frontswap_store()
1206 	zpool_unmap_handle(entry->pool->zpool, handle);  in zswap_frontswap_store()
1236 	zswap_pool_put(entry->pool);  in zswap_frontswap_store()
1276 	if (!zpool_can_sleep_mapped(entry->pool->zpool)) {  in zswap_frontswap_load()
1287 	src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);  in zswap_frontswap_load()
1288 	if (zpool_evictable(entry->pool->zpool))  in zswap_frontswap_load()
1291 	if (!zpool_can_sleep_mapped(entry->pool->zpool)) {  in zswap_frontswap_load()
1296 		zpool_unmap_handle(entry->pool->zpool, entry->handle);  in zswap_frontswap_load()
1299 	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);  in zswap_frontswap_load()
1308 	if (zpool_can_sleep_mapped(entry->pool->zpool))  in zswap_frontswap_load()
1309 		zpool_unmap_handle(entry->pool->zpool, entry->handle);  in zswap_frontswap_load()
1439 	struct zswap_pool *pool;  in init_zswap()  local
1463 	pool = __zswap_pool_create_fallback();  in init_zswap()
1464 	if (pool) {  in init_zswap()
1465 		pr_info("loaded using pool %s/%s\n", pool->tfm_name,  in init_zswap()
1466 			zpool_get_type(pool->zpool));  in init_zswap()
1467 		list_add(&pool->list, &zswap_pools);  in init_zswap()
1484 	if (pool)  in init_zswap()
1485 		zswap_pool_destroy(pool);  in init_zswap()