273 struct zspage {  struct
302 static void migrate_lock_init(struct zspage *zspage);  argument
303 static void migrate_read_lock(struct zspage *zspage);
304 static void migrate_read_unlock(struct zspage *zspage);
307 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
313 static void migrate_lock_init(struct zspage *zspage) {}  in migrate_lock_init()  argument
314 static void migrate_read_lock(struct zspage *zspage) {}  in migrate_read_lock()  argument
315 static void migrate_read_unlock(struct zspage *zspage) {}  in migrate_read_unlock()  argument
318 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
328 	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in create_cache()
356 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)  in cache_alloc_zspage()
362 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)  in cache_free_zspage()  argument
364 	kmem_cache_free(pool->zspage_cachep, zspage);  in cache_free_zspage()
457 static bool is_zspage_isolated(struct zspage *zspage)  in is_zspage_isolated()  argument
459 	return zspage->isolated;  in is_zspage_isolated()
468 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
470 	return zspage->inuse;  in get_zspage_inuse()
473 static inline void set_zspage_inuse(struct zspage *zspage, int val)  in set_zspage_inuse()  argument
475 	zspage->inuse = val;  in set_zspage_inuse()
478 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
480 	zspage->inuse += val;  in mod_zspage_inuse()
483 static inline struct page *get_first_page(struct zspage *zspage)  in get_first_page()  argument
485 	struct page *first_page = zspage->first_page;  in get_first_page()
501 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
503 	return zspage->freeobj;  in get_freeobj()
506 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
508 	zspage->freeobj = obj;  in set_freeobj()
511 static void get_zspage_mapping(struct zspage *zspage,  in get_zspage_mapping()  argument
515 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage_mapping()
517 	*fullness = zspage->fullness;  in get_zspage_mapping()
518 	*class_idx = zspage->class;  in get_zspage_mapping()
521 static void set_zspage_mapping(struct zspage *zspage,  in set_zspage_mapping()  argument
525 	zspage->class = class_idx;  in set_zspage_mapping()
526 	zspage->fullness = fullness;  in set_zspage_mapping()
707 						struct zspage *zspage)  in get_fullness_group()  argument
712 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
734 				struct zspage *zspage,  in insert_zspage()  argument
737 	struct zspage *head;  in insert_zspage()
741 					struct zspage, list);  in insert_zspage()
747 		if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {  in insert_zspage()
748 			list_add(&zspage->list, &head->list);  in insert_zspage()
752 	list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
760 				struct zspage *zspage,  in remove_zspage()  argument
764 	VM_BUG_ON(is_zspage_isolated(zspage));  in remove_zspage()
766 	list_del_init(&zspage->list);  in remove_zspage()
780 						struct zspage *zspage)  in fix_fullness_group()  argument
785 	get_zspage_mapping(zspage, &class_idx, &currfg);  in fix_fullness_group()
786 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
790 	if (!is_zspage_isolated(zspage)) {  in fix_fullness_group()
791 		remove_zspage(class, zspage, currfg);  in fix_fullness_group()
792 		insert_zspage(class, zspage, newfg);  in fix_fullness_group()
795 	set_zspage_mapping(zspage, class_idx, newfg);  in fix_fullness_group()
837 static struct zspage *get_zspage(struct page *page)  in get_zspage()
839 	struct zspage *zspage = (struct zspage *)page->private;  in get_zspage()  local
841 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
842 	return zspage;  in get_zspage()
927 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
931 	for (cursor = get_first_page(zspage); cursor != NULL; cursor =  in trylock_zspage()
941 	for (cursor = get_first_page(zspage); cursor != fail; cursor =  in trylock_zspage()
949 				struct zspage *zspage)  in __free_zspage()  argument
955 	get_zspage_mapping(zspage, &class_idx, &fg);  in __free_zspage()
959 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
962 	next = page = get_first_page(zspage);  in __free_zspage()
973 	cache_free_zspage(pool, zspage);  in __free_zspage()
981 				struct zspage *zspage)  in free_zspage()  argument
983 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
984 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
986 	if (!trylock_zspage(zspage)) {  in free_zspage()
991 	remove_zspage(class, zspage, ZS_EMPTY);  in free_zspage()
992 	__free_zspage(pool, class, zspage);  in free_zspage()
996 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
1000 	struct page *page = get_first_page(zspage);  in init_zspage()
1037 	set_freeobj(zspage, 0);  in init_zspage()
1040 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
1058 		set_page_private(page, (unsigned long)zspage);  in create_page_chain()
1061 			zspage->first_page = page;  in create_page_chain()
1076 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
1082 	struct zspage *zspage = cache_alloc_zspage(pool, gfp);  in alloc_zspage()  local
1084 	if (!zspage)  in alloc_zspage()
1087 	memset(zspage, 0, sizeof(struct zspage));  in alloc_zspage()
1088 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
1089 	migrate_lock_init(zspage);  in alloc_zspage()
1100 			cache_free_zspage(pool, zspage);  in alloc_zspage()
1108 	create_page_chain(class, zspage, pages);  in alloc_zspage()
1109 	init_zspage(class, zspage);  in alloc_zspage()
1111 	return zspage;  in alloc_zspage()
1114 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
1117 	struct zspage *zspage;  in find_get_zspage()  local
1120 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
1121 				struct zspage, list);  in find_get_zspage()
1122 		if (zspage)  in find_get_zspage()
1126 	return zspage;  in find_get_zspage()
1278 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1280 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1307 	struct zspage *zspage;  in zs_map_object()  local
1331 	zspage = get_zspage(page);  in zs_map_object()
1334 	migrate_read_lock(zspage);  in zs_map_object()
1336 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_map_object()
1365 	struct zspage *zspage;  in zs_unmap_object()  local
1377 	zspage = get_zspage(page);  in zs_unmap_object()
1378 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_unmap_object()
1396 	migrate_read_unlock(zspage);  in zs_unmap_object()
1421 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1432 	obj = get_freeobj(zspage);  in obj_malloc()
1437 	m_page = get_first_page(zspage);  in obj_malloc()
1444 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1450 		zspage->first_page->index = handle;  in obj_malloc()
1453 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1477 	struct zspage *zspage;  in zs_malloc()  local
1491 	zspage = find_get_zspage(class);  in zs_malloc()
1492 	if (likely(zspage)) {  in zs_malloc()
1493 		obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1495 		fix_fullness_group(class, zspage);  in zs_malloc()
1504 	zspage = alloc_zspage(pool, class, gfp);  in zs_malloc()
1505 	if (!zspage) {  in zs_malloc()
1511 	obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1512 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1513 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1514 	set_zspage_mapping(zspage, class->index, newfg);  in zs_malloc()
1521 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1531 	struct zspage *zspage;  in obj_free()  local
1540 	zspage = get_zspage(f_page);  in obj_free()
1546 	link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1548 	set_freeobj(zspage, f_objidx);  in obj_free()
1549 	mod_zspage_inuse(zspage, -1);  in obj_free()
1555 	struct zspage *zspage;  in zs_free()  local
1570 	zspage = get_zspage(f_page);  in zs_free()
1572 	migrate_read_lock(zspage);  in zs_free()
1574 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_free()
1579 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1581 		migrate_read_unlock(zspage);  in zs_free()
1585 	isolated = is_zspage_isolated(zspage);  in zs_free()
1586 	migrate_read_unlock(zspage);  in zs_free()
1589 		free_zspage(pool, class, zspage);  in zs_free()
1758 static struct zspage *isolate_zspage(struct size_class *class, bool source)  in isolate_zspage()
1761 	struct zspage *zspage;  in isolate_zspage()  local
1770 		zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],  in isolate_zspage()
1771 							struct zspage, list);  in isolate_zspage()
1772 		if (zspage) {  in isolate_zspage()
1773 			VM_BUG_ON(is_zspage_isolated(zspage));  in isolate_zspage()
1774 			remove_zspage(class, zspage, fg[i]);  in isolate_zspage()
1775 			return zspage;  in isolate_zspage()
1779 	return zspage;  in isolate_zspage()
1790 			struct zspage *zspage)  in putback_zspage()  argument
1794 	VM_BUG_ON(is_zspage_isolated(zspage));  in putback_zspage()
1796 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1797 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1798 	set_zspage_mapping(zspage, class->index, fullness);  in putback_zspage()
1808 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1810 	struct page *page = get_first_page(zspage);  in lock_zspage()
1849 static void migrate_lock_init(struct zspage *zspage)  in migrate_lock_init()  argument
1851 	rwlock_init(&zspage->lock);  in migrate_lock_init()
1854 static void migrate_read_lock(struct zspage *zspage)  in migrate_read_lock()  argument
1856 	read_lock(&zspage->lock);  in migrate_read_lock()
1859 static void migrate_read_unlock(struct zspage *zspage)  in migrate_read_unlock()  argument
1861 	read_unlock(&zspage->lock);  in migrate_read_unlock()
1864 static void migrate_write_lock(struct zspage *zspage)  in migrate_write_lock()  argument
1866 	write_lock(&zspage->lock);  in migrate_write_lock()
1869 static void migrate_write_unlock(struct zspage *zspage)  in migrate_write_unlock()  argument
1871 	write_unlock(&zspage->lock);  in migrate_write_unlock()
1875 static void inc_zspage_isolation(struct zspage *zspage)  in inc_zspage_isolation()  argument
1877 	zspage->isolated++;  in inc_zspage_isolation()
1880 static void dec_zspage_isolation(struct zspage *zspage)  in dec_zspage_isolation()  argument
1882 	zspage->isolated--;  in dec_zspage_isolation()
1885 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1892 	page = get_first_page(zspage);  in replace_sub_page()
1901 	create_page_chain(class, zspage, pages);  in replace_sub_page()
1914 	struct zspage *zspage;  in zs_page_isolate()  local
1924 	zspage = get_zspage(page);  in zs_page_isolate()
1931 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1937 	if (get_zspage_inuse(zspage) == 0) {  in zs_page_isolate()
1943 	if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1952 	if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1953 		get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1954 		remove_zspage(class, zspage, fullness);  in zs_page_isolate()
1957 	inc_zspage_isolation(zspage);  in zs_page_isolate()
1970 	struct zspage *zspage;  in zs_page_migrate()  local
1990 	zspage = get_zspage(page);  in zs_page_migrate()
1993 	migrate_write_lock(zspage);  in zs_page_migrate()
1994 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_migrate()
2000 	if (!get_zspage_inuse(zspage)) {  in zs_page_migrate()
2044 	replace_sub_page(class, zspage, newpage, page);  in zs_page_migrate()
2047 	dec_zspage_isolation(zspage);  in zs_page_migrate()
2053 	if (!is_zspage_isolated(zspage))  in zs_page_migrate()
2054 		putback_zspage(class, zspage);  in zs_page_migrate()
2074 	migrate_write_unlock(zspage);  in zs_page_migrate()
2086 	struct zspage *zspage;  in zs_page_putback()  local
2091 	zspage = get_zspage(page);  in zs_page_putback()
2092 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_page_putback()
2098 	dec_zspage_isolation(zspage);  in zs_page_putback()
2099 	if (!is_zspage_isolated(zspage)) {  in zs_page_putback()
2100 		fg = putback_zspage(class, zspage);  in zs_page_putback()
2146 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
2162 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
2163 		list_del(&zspage->list);  in async_free_zspage()
2164 		lock_zspage(zspage);  in async_free_zspage()
2166 		get_zspage_mapping(zspage, &class_idx, &fullness);  in async_free_zspage()
2170 		__free_zspage(pool, pool->size_class[class_idx], zspage);  in async_free_zspage()
2185 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
2187 	struct page *page = get_first_page(zspage);  in SetZsPageMovable()
2220 	struct zspage *src_zspage;  in __zs_compact()
2221 	struct zspage *dst_zspage = NULL;  in __zs_compact()