279 struct zspage {  struct
308 static void migrate_lock_init(struct zspage *zspage);  argument
309 static void migrate_read_lock(struct zspage *zspage);
310 static void migrate_read_unlock(struct zspage *zspage);
313 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
319 static void migrate_lock_init(struct zspage *zspage) {}  in migrate_lock_init()  argument
320 static void migrate_read_lock(struct zspage *zspage) {}  in migrate_read_lock()  argument
321 static void migrate_read_unlock(struct zspage *zspage) {}  in migrate_read_unlock()  argument
324 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
334 	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in create_cache()
362 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)  in cache_alloc_zspage()
368 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)  in cache_free_zspage()  argument
370 	kmem_cache_free(pool->zspage_cachep, zspage);  in cache_free_zspage()
464 static bool is_zspage_isolated(struct zspage *zspage)  in is_zspage_isolated()  argument
466 	return zspage->isolated;  in is_zspage_isolated()
475 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
477 	return zspage->inuse;  in get_zspage_inuse()
481 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
483 	zspage->inuse += val;  in mod_zspage_inuse()
486 static inline struct page *get_first_page(struct zspage *zspage)  in get_first_page()  argument
488 	struct page *first_page = zspage->first_page;  in get_first_page()
504 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
506 	return zspage->freeobj;  in get_freeobj()
509 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
511 	zspage->freeobj = obj;  in set_freeobj()
514 static void get_zspage_mapping(struct zspage *zspage,  in get_zspage_mapping()  argument
518 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage_mapping()
520 	*fullness = zspage->fullness;  in get_zspage_mapping()
521 	*class_idx = zspage->class;  in get_zspage_mapping()
524 static void set_zspage_mapping(struct zspage *zspage,  in set_zspage_mapping()  argument
528 	zspage->class = class_idx;  in set_zspage_mapping()
529 	zspage->fullness = fullness;  in set_zspage_mapping()
694 						struct zspage *zspage)  in get_fullness_group()  argument
699 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
721 				struct zspage *zspage,  in insert_zspage()  argument
724 	struct zspage *head;  in insert_zspage()
728 					struct zspage, list);  in insert_zspage()
734 		if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {  in insert_zspage()
735 			list_add(&zspage->list, &head->list);  in insert_zspage()
739 	list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
747 				struct zspage *zspage,  in remove_zspage()  argument
751 	VM_BUG_ON(is_zspage_isolated(zspage));  in remove_zspage()
753 	list_del_init(&zspage->list);  in remove_zspage()
767 						struct zspage *zspage)  in fix_fullness_group()  argument
772 	get_zspage_mapping(zspage, &class_idx, &currfg);  in fix_fullness_group()
773 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
777 	if (!is_zspage_isolated(zspage)) {  in fix_fullness_group()
778 		remove_zspage(class, zspage, currfg);  in fix_fullness_group()
779 		insert_zspage(class, zspage, newfg);  in fix_fullness_group()
782 	set_zspage_mapping(zspage, class_idx, newfg);  in fix_fullness_group()
824 static struct zspage *get_zspage(struct page *page)  in get_zspage()
826 	struct zspage *zspage = (struct zspage *)page->private;  in get_zspage()  local
828 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
829 	return zspage;  in get_zspage()
914 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
918 	for (cursor = get_first_page(zspage); cursor != NULL; cursor =  in trylock_zspage()
928 	for (cursor = get_first_page(zspage); cursor != fail; cursor =  in trylock_zspage()
936 				struct zspage *zspage)  in __free_zspage()  argument
942 	get_zspage_mapping(zspage, &class_idx, &fg);  in __free_zspage()
946 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
949 	next = page = get_first_page(zspage);  in __free_zspage()
960 	cache_free_zspage(pool, zspage);  in __free_zspage()
968 				struct zspage *zspage)  in free_zspage()  argument
970 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
971 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
973 	if (!trylock_zspage(zspage)) {  in free_zspage()
978 	remove_zspage(class, zspage, ZS_EMPTY);  in free_zspage()
979 	__free_zspage(pool, class, zspage);  in free_zspage()
983 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
987 	struct page *page = get_first_page(zspage);  in init_zspage()
1024 	set_freeobj(zspage, 0);  in init_zspage()
1027 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
1045 		set_page_private(page, (unsigned long)zspage);  in create_page_chain()
1048 			zspage->first_page = page;  in create_page_chain()
1063 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
1069 	struct zspage *zspage = cache_alloc_zspage(pool, gfp);  in alloc_zspage()  local
1071 	if (!zspage)  in alloc_zspage()
1074 	memset(zspage, 0, sizeof(struct zspage));  in alloc_zspage()
1075 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
1076 	migrate_lock_init(zspage);  in alloc_zspage()
1087 			cache_free_zspage(pool, zspage);  in alloc_zspage()
1095 	create_page_chain(class, zspage, pages);  in alloc_zspage()
1096 	init_zspage(class, zspage);  in alloc_zspage()
1098 	return zspage;  in alloc_zspage()
1101 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
1104 	struct zspage *zspage;  in find_get_zspage()  local
1107 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
1108 				struct zspage, list);  in find_get_zspage()
1109 		if (zspage)  in find_get_zspage()
1113 	return zspage;  in find_get_zspage()
1265 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1267 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1294 	struct zspage *zspage;  in zs_map_object()  local
1318 	zspage = get_zspage(page);  in zs_map_object()
1321 	migrate_read_lock(zspage);  in zs_map_object()
1323 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_map_object()
1352 	struct zspage *zspage;  in zs_unmap_object()  local
1364 	zspage = get_zspage(page);  in zs_unmap_object()
1365 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_unmap_object()
1383 	migrate_read_unlock(zspage);  in zs_unmap_object()
1408 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1419 	obj = get_freeobj(zspage);  in obj_malloc()
1424 	m_page = get_first_page(zspage);  in obj_malloc()
1431 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1437 		zspage->first_page->index = handle;  in obj_malloc()
1440 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1464 	struct zspage *zspage;  in zs_malloc()  local
1478 	zspage = find_get_zspage(class);  in zs_malloc()
1479 	if (likely(zspage)) {  in zs_malloc()
1480 		obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1482 		fix_fullness_group(class, zspage);  in zs_malloc()
1491 	zspage = alloc_zspage(pool, class, gfp);  in zs_malloc()
1492 	if (!zspage) {  in zs_malloc()
1498 	obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1499 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1500 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1501 	set_zspage_mapping(zspage, class->index, newfg);  in zs_malloc()
1508 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1518 	struct zspage *zspage;  in obj_free()  local
1527 	zspage = get_zspage(f_page);  in obj_free()
1533 	link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1535 	set_freeobj(zspage, f_objidx);  in obj_free()
1536 	mod_zspage_inuse(zspage, -1);  in obj_free()
1542 	struct zspage *zspage;  in zs_free()  local
1557 	zspage = get_zspage(f_page);  in zs_free()
1559 	migrate_read_lock(zspage);  in zs_free()
1561 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_free()
1566 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1568 		migrate_read_unlock(zspage);  in zs_free()
1572 	isolated = is_zspage_isolated(zspage);  in zs_free()
1573 	migrate_read_unlock(zspage);  in zs_free()
1576 		free_zspage(pool, class, zspage);  in zs_free()
1745 static struct zspage *isolate_zspage(struct size_class *class, bool source)  in isolate_zspage()
1748 	struct zspage *zspage;  in isolate_zspage()  local
1757 		zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],  in isolate_zspage()
1758 							struct zspage, list);  in isolate_zspage()
1759 		if (zspage) {  in isolate_zspage()
1760 			VM_BUG_ON(is_zspage_isolated(zspage));  in isolate_zspage()
1761 			remove_zspage(class, zspage, fg[i]);  in isolate_zspage()
1762 			return zspage;  in isolate_zspage()
1766 	return zspage;  in isolate_zspage()
1777 			struct zspage *zspage)  in putback_zspage()  argument
1781 	VM_BUG_ON(is_zspage_isolated(zspage));  in putback_zspage()
1783 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1784 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1785 	set_zspage_mapping(zspage, class->index, fullness);  in putback_zspage()
1795 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1797 	struct page *page = get_first_page(zspage);  in lock_zspage()
1831 static void migrate_lock_init(struct zspage *zspage)  in migrate_lock_init()  argument
1833 	rwlock_init(&zspage->lock);  in migrate_lock_init()
1836 static void migrate_read_lock(struct zspage *zspage)  in migrate_read_lock()  argument
1838 	read_lock(&zspage->lock);  in migrate_read_lock()
1841 static void migrate_read_unlock(struct zspage *zspage)  in migrate_read_unlock()  argument
1843 	read_unlock(&zspage->lock);  in migrate_read_unlock()
1846 static void migrate_write_lock(struct zspage *zspage)  in migrate_write_lock()  argument
1848 	write_lock(&zspage->lock);  in migrate_write_lock()
1851 static void migrate_write_unlock(struct zspage *zspage)  in migrate_write_unlock()  argument
1853 	write_unlock(&zspage->lock);  in migrate_write_unlock()
1857 static void inc_zspage_isolation(struct zspage *zspage)  in inc_zspage_isolation()  argument
1859 	zspage->isolated++;  in inc_zspage_isolation()
1862 static void dec_zspage_isolation(struct zspage *zspage)  in dec_zspage_isolation()  argument
1864 	zspage->isolated--;  in dec_zspage_isolation()
1869 				    struct zspage *zspage)  in putback_zspage_deferred()  argument
1873 	fg = putback_zspage(class, zspage);  in putback_zspage_deferred()
1892 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1899 	page = get_first_page(zspage);  in replace_sub_page()
1908 	create_page_chain(class, zspage, pages);  in replace_sub_page()
1921 	struct zspage *zspage;  in zs_page_isolate()  local
1931 	zspage = get_zspage(page);  in zs_page_isolate()
1938 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1944 	if (get_zspage_inuse(zspage) == 0) {  in zs_page_isolate()
1950 	if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1959 	if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1960 		get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1962 		remove_zspage(class, zspage, fullness);  in zs_page_isolate()
1965 	inc_zspage_isolation(zspage);  in zs_page_isolate()
1978 	struct zspage *zspage;  in zs_page_migrate()  local
1998 	zspage = get_zspage(page);  in zs_page_migrate()
2001 	migrate_write_lock(zspage);  in zs_page_migrate()
2002 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_migrate()
2008 	if (!get_zspage_inuse(zspage)) {  in zs_page_migrate()
2052 	replace_sub_page(class, zspage, newpage, page);  in zs_page_migrate()
2055 	dec_zspage_isolation(zspage);  in zs_page_migrate()
2061 	if (!is_zspage_isolated(zspage)) {  in zs_page_migrate()
2068 		putback_zspage_deferred(pool, class, zspage);  in zs_page_migrate()
2090 	migrate_write_unlock(zspage);  in zs_page_migrate()
2102 	struct zspage *zspage;  in zs_page_putback()  local
2107 	zspage = get_zspage(page);  in zs_page_putback()
2108 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_page_putback()
2114 	dec_zspage_isolation(zspage);  in zs_page_putback()
2115 	if (!is_zspage_isolated(zspage)) {  in zs_page_putback()
2120 		putback_zspage_deferred(pool, class, zspage);  in zs_page_putback()
2189 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
2205 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
2206 		list_del(&zspage->list);  in async_free_zspage()
2207 		lock_zspage(zspage);  in async_free_zspage()
2209 		get_zspage_mapping(zspage, &class_idx, &fullness);  in async_free_zspage()
2213 		__free_zspage(pool, pool->size_class[class_idx], zspage);  in async_free_zspage()
2228 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
2230 	struct page *page = get_first_page(zspage);  in SetZsPageMovable()
2263 	struct zspage *src_zspage;  in __zs_compact()
2264 	struct zspage *dst_zspage = NULL;  in __zs_compact()