244 struct zspage {  struct
268 static void SetZsHugePage(struct zspage *zspage)  in SetZsHugePage()  argument
270 	zspage->huge = 1;  in SetZsHugePage()
273 static bool ZsHugePage(struct zspage *zspage)  in ZsHugePage()  argument
275 	return zspage->huge;  in ZsHugePage()
278 static void migrate_lock_init(struct zspage *zspage);
279 static void migrate_read_lock(struct zspage *zspage);
280 static void migrate_read_unlock(struct zspage *zspage);
283 static void migrate_write_lock(struct zspage *zspage);
284 static void migrate_write_lock_nested(struct zspage *zspage);
285 static void migrate_write_unlock(struct zspage *zspage);
288 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
290 static void migrate_write_lock(struct zspage *zspage) {}  in migrate_write_lock()  argument
291 static void migrate_write_lock_nested(struct zspage *zspage) {}  in migrate_write_lock_nested()  argument
292 static void migrate_write_unlock(struct zspage *zspage) {}  in migrate_write_unlock()  argument
295 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
305 	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in create_cache()
333 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)  in cache_alloc_zspage()
339 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)  in cache_free_zspage()  argument
341 	kmem_cache_free(pool->zspage_cachep, zspage);  in cache_free_zspage()
440 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
442 	return zspage->inuse;  in get_zspage_inuse()
446 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
448 	zspage->inuse += val;  in mod_zspage_inuse()
451 static inline struct page *get_first_page(struct zspage *zspage)  in get_first_page()  argument
453 	struct page *first_page = zspage->first_page;  in get_first_page()
469 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
471 	return zspage->freeobj;  in get_freeobj()
474 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
476 	zspage->freeobj = obj;  in set_freeobj()
479 static void get_zspage_mapping(struct zspage *zspage,  in get_zspage_mapping()  argument
483 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage_mapping()
485 	*fullness = zspage->fullness;  in get_zspage_mapping()
486 	*class_idx = zspage->class;  in get_zspage_mapping()
490 				       struct zspage *zspage)  in zspage_class()  argument
492 	return pool->size_class[zspage->class];  in zspage_class()
495 static void set_zspage_mapping(struct zspage *zspage,  in set_zspage_mapping()  argument
499 	zspage->class = class_idx;  in set_zspage_mapping()
500 	zspage->fullness = fullness;  in set_zspage_mapping()
664 static int get_fullness_group(struct size_class *class, struct zspage *zspage)  in get_fullness_group()  argument
668 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
692 				struct zspage *zspage,  in insert_zspage()  argument
696 	list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
704 				struct zspage *zspage,  in remove_zspage()  argument
709 	list_del_init(&zspage->list);  in remove_zspage()
722 static int fix_fullness_group(struct size_class *class, struct zspage *zspage)  in fix_fullness_group()  argument
727 	get_zspage_mapping(zspage, &class_idx, &currfg);  in fix_fullness_group()
728 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
732 	remove_zspage(class, zspage, currfg);  in fix_fullness_group()
733 	insert_zspage(class, zspage, newfg);  in fix_fullness_group()
734 	set_zspage_mapping(zspage, class_idx, newfg);  in fix_fullness_group()
739 static struct zspage *get_zspage(struct page *page)  in get_zspage()
741 	struct zspage *zspage = (struct zspage *)page_private(page);  in get_zspage()  local
743 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
744 	return zspage;  in get_zspage()
749 	struct zspage *zspage = get_zspage(page);  in get_next_page()  local
751 	if (unlikely(ZsHugePage(zspage)))  in get_next_page()
802 	struct zspage *zspage = get_zspage(page);  in obj_allocated()  local
804 	if (unlikely(ZsHugePage(zspage))) {  in obj_allocated()
827 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
831 	for (cursor = get_first_page(zspage); cursor != NULL; cursor =  in trylock_zspage()
841 	for (cursor = get_first_page(zspage); cursor != fail; cursor =  in trylock_zspage()
849 				struct zspage *zspage)  in __free_zspage()  argument
855 	get_zspage_mapping(zspage, &class_idx, &fg);  in __free_zspage()
859 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
862 	next = page = get_first_page(zspage);  in __free_zspage()
873 	cache_free_zspage(pool, zspage);  in __free_zspage()
880 				struct zspage *zspage)  in free_zspage()  argument
882 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
883 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
890 	if (!trylock_zspage(zspage)) {  in free_zspage()
895 	remove_zspage(class, zspage, ZS_INUSE_RATIO_0);  in free_zspage()
896 	__free_zspage(pool, class, zspage);  in free_zspage()
900 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
904 	struct page *page = get_first_page(zspage);  in init_zspage()
941 	set_freeobj(zspage, 0);  in init_zspage()
944 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
962 		set_page_private(page, (unsigned long)zspage);  in create_page_chain()
965 			zspage->first_page = page;  in create_page_chain()
969 				SetZsHugePage(zspage);  in create_page_chain()
980 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
986 	struct zspage *zspage = cache_alloc_zspage(pool, gfp);  in alloc_zspage()  local
988 	if (!zspage)  in alloc_zspage()
991 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
992 	migrate_lock_init(zspage);  in alloc_zspage()
1003 			cache_free_zspage(pool, zspage);  in alloc_zspage()
1011 	create_page_chain(class, zspage, pages);  in alloc_zspage()
1012 	init_zspage(class, zspage);  in alloc_zspage()
1013 	zspage->pool = pool;  in alloc_zspage()
1015 	return zspage;  in alloc_zspage()
1018 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
1021 	struct zspage *zspage;  in find_get_zspage()  local
1024 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
1025 						  struct zspage, list);  in find_get_zspage()
1026 		if (zspage)  in find_get_zspage()
1030 	return zspage;  in find_get_zspage()
1140 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1142 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1145 static bool zspage_empty(struct zspage *zspage)  in zspage_empty()  argument
1147 	return get_zspage_inuse(zspage) == 0;  in zspage_empty()
1195 	struct zspage *zspage;  in zs_map_object()  local
1216 	zspage = get_zspage(page);  in zs_map_object()
1224 	migrate_read_lock(zspage);  in zs_map_object()
1227 	class = zspage_class(pool, zspage);  in zs_map_object()
1247 	if (likely(!ZsHugePage(zspage)))  in zs_map_object()
1256 	struct zspage *zspage;  in zs_unmap_object()  local
1266 	zspage = get_zspage(page);  in zs_unmap_object()
1267 	class = zspage_class(pool, zspage);  in zs_unmap_object()
1284 	migrate_read_unlock(zspage);  in zs_unmap_object()
1308 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1319 	class = pool->size_class[zspage->class];  in obj_malloc()
1321 	obj = get_freeobj(zspage);  in obj_malloc()
1326 	m_page = get_first_page(zspage);  in obj_malloc()
1333 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1334 	if (likely(!ZsHugePage(zspage)))  in obj_malloc()
1339 		zspage->first_page->index = handle;  in obj_malloc()
1342 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1365 	struct zspage *zspage;  in zs_malloc()  local
1380 	zspage = find_get_zspage(class);  in zs_malloc()
1381 	if (likely(zspage)) {  in zs_malloc()
1382 		obj = obj_malloc(pool, zspage, handle);  in zs_malloc()
1384 		fix_fullness_group(class, zspage);  in zs_malloc()
1393 	zspage = alloc_zspage(pool, class, gfp);  in zs_malloc()
1394 	if (!zspage) {  in zs_malloc()
1400 	obj = obj_malloc(pool, zspage, handle);  in zs_malloc()
1401 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1402 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1403 	set_zspage_mapping(zspage, class->index, newfg);  in zs_malloc()
1410 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1421 	struct zspage *zspage;  in obj_free()  local
1429 	zspage = get_zspage(f_page);  in obj_free()
1435 	if (likely(!ZsHugePage(zspage)))  in obj_free()
1436 		link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1439 	set_freeobj(zspage, f_objidx);  in obj_free()
1442 	mod_zspage_inuse(zspage, -1);  in obj_free()
1447 	struct zspage *zspage;  in zs_free()  local
1463 	zspage = get_zspage(f_page);  in zs_free()
1464 	class = zspage_class(pool, zspage);  in zs_free()
1469 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1471 		free_zspage(pool, class, zspage);  in zs_free()
1578 static void migrate_zspage(struct zs_pool *pool, struct zspage *src_zspage,  in migrate_zspage()
1579 			   struct zspage *dst_zspage)  in migrate_zspage()
1614 static struct zspage *isolate_src_zspage(struct size_class *class)  in isolate_src_zspage()
1616 	struct zspage *zspage;  in isolate_src_zspage()  local
1620 		zspage = list_first_entry_or_null(&class->fullness_list[fg],  in isolate_src_zspage()
1621 						  struct zspage, list);  in isolate_src_zspage()
1622 		if (zspage) {  in isolate_src_zspage()
1623 			remove_zspage(class, zspage, fg);  in isolate_src_zspage()
1624 			return zspage;  in isolate_src_zspage()
1628 	return zspage;  in isolate_src_zspage()
1631 static struct zspage *isolate_dst_zspage(struct size_class *class)  in isolate_dst_zspage()
1633 	struct zspage *zspage;  in isolate_dst_zspage()  local
1637 		zspage = list_first_entry_or_null(&class->fullness_list[fg],  in isolate_dst_zspage()
1638 						  struct zspage, list);  in isolate_dst_zspage()
1639 		if (zspage) {  in isolate_dst_zspage()
1640 			remove_zspage(class, zspage, fg);  in isolate_dst_zspage()
1641 			return zspage;  in isolate_dst_zspage()
1645 	return zspage;  in isolate_dst_zspage()
1655 static int putback_zspage(struct size_class *class, struct zspage *zspage)  in putback_zspage()  argument
1659 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1660 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1661 	set_zspage_mapping(zspage, class->index, fullness);  in putback_zspage()
1671 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1684 		migrate_read_lock(zspage);  in lock_zspage()
1685 		page = get_first_page(zspage);  in lock_zspage()
1689 		migrate_read_unlock(zspage);  in lock_zspage()
1700 			migrate_read_unlock(zspage);  in lock_zspage()
1703 			migrate_read_lock(zspage);  in lock_zspage()
1706 	migrate_read_unlock(zspage);  in lock_zspage()
1710 static void migrate_lock_init(struct zspage *zspage)  in migrate_lock_init()  argument
1712 	rwlock_init(&zspage->lock);  in migrate_lock_init()
1715 static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock)  in migrate_read_lock()  argument
1717 	read_lock(&zspage->lock);  in migrate_read_lock()
1720 static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock)  in migrate_read_unlock()  argument
1722 	read_unlock(&zspage->lock);  in migrate_read_unlock()
1726 static void migrate_write_lock(struct zspage *zspage)  in migrate_write_lock()  argument
1728 	write_lock(&zspage->lock);  in migrate_write_lock()
1731 static void migrate_write_lock_nested(struct zspage *zspage)  in migrate_write_lock_nested()  argument
1733 	write_lock_nested(&zspage->lock, SINGLE_DEPTH_NESTING);  in migrate_write_lock_nested()
1736 static void migrate_write_unlock(struct zspage *zspage)  in migrate_write_unlock()  argument
1738 	write_unlock(&zspage->lock);  in migrate_write_unlock()
1742 static void inc_zspage_isolation(struct zspage *zspage)  in inc_zspage_isolation()  argument
1744 	zspage->isolated++;  in inc_zspage_isolation()
1747 static void dec_zspage_isolation(struct zspage *zspage)  in dec_zspage_isolation()  argument
1749 	VM_BUG_ON(zspage->isolated == 0);  in dec_zspage_isolation()
1750 	zspage->isolated--;  in dec_zspage_isolation()
1755 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1762 	page = get_first_page(zspage);  in replace_sub_page()
1771 	create_page_chain(class, zspage, pages);  in replace_sub_page()
1773 	if (unlikely(ZsHugePage(zspage)))  in replace_sub_page()
1781 	struct zspage *zspage;  in zs_page_isolate()  local
1789 	zspage = get_zspage(page);  in zs_page_isolate()
1790 	pool = zspage->pool;  in zs_page_isolate()
1792 	inc_zspage_isolation(zspage);  in zs_page_isolate()
1803 	struct zspage *zspage;  in zs_page_migrate()  local
1822 	zspage = get_zspage(page);  in zs_page_migrate()
1823 	pool = zspage->pool;  in zs_page_migrate()
1830 	class = zspage_class(pool, zspage);  in zs_page_migrate()
1833 	migrate_write_lock(zspage);  in zs_page_migrate()
1858 	replace_sub_page(class, zspage, newpage, page);  in zs_page_migrate()
1859 	dec_zspage_isolation(zspage);  in zs_page_migrate()
1865 	migrate_write_unlock(zspage);  in zs_page_migrate()
1882 	struct zspage *zspage;  in zs_page_putback()  local
1886 	zspage = get_zspage(page);  in zs_page_putback()
1887 	pool = zspage->pool;  in zs_page_putback()
1889 	dec_zspage_isolation(zspage);  in zs_page_putback()
1909 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
1925 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
1926 		list_del(&zspage->list);  in async_free_zspage()
1927 		lock_zspage(zspage);  in async_free_zspage()
1929 		get_zspage_mapping(zspage, &class_idx, &fullness);  in async_free_zspage()
1933 		__free_zspage(pool, class, zspage);  in async_free_zspage()
1953 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
1955 	struct page *page = get_first_page(zspage);  in SetZsPageMovable()
1990 	struct zspage *src_zspage = NULL;  in __zs_compact()
1991 	struct zspage *dst_zspage = NULL;  in __zs_compact()