254 struct zspage {  struct
280 static void SetZsHugePage(struct zspage *zspage)  in SetZsHugePage()  argument
282 	zspage->huge = 1;  in SetZsHugePage()
285 static bool ZsHugePage(struct zspage *zspage)  in ZsHugePage()  argument
287 	return zspage->huge;  in ZsHugePage()
291 static void migrate_lock_init(struct zspage *zspage);
292 static void migrate_read_lock(struct zspage *zspage);
293 static void migrate_read_unlock(struct zspage *zspage);
294 static void migrate_write_lock(struct zspage *zspage);
295 static void migrate_write_lock_nested(struct zspage *zspage);
296 static void migrate_write_unlock(struct zspage *zspage);
299 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
301 static void migrate_lock_init(struct zspage *zspage) {}  in migrate_lock_init()  argument
302 static void migrate_read_lock(struct zspage *zspage) {}  in migrate_read_lock()  argument
303 static void migrate_read_unlock(struct zspage *zspage) {}  in migrate_read_unlock()  argument
304 static void migrate_write_lock(struct zspage *zspage) {}  in migrate_write_lock()  argument
305 static void migrate_write_lock_nested(struct zspage *zspage) {}  in migrate_write_lock_nested()  argument
306 static void migrate_write_unlock(struct zspage *zspage) {}  in migrate_write_unlock()  argument
309 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
319 	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in create_cache()
347 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)  in cache_alloc_zspage()
353 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)  in cache_free_zspage()  argument
355 	kmem_cache_free(pool->zspage_cachep, zspage);  in cache_free_zspage()
456 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
458 	return zspage->inuse;  in get_zspage_inuse()
462 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
464 	zspage->inuse += val;  in mod_zspage_inuse()
467 static inline struct page *get_first_page(struct zspage *zspage)  in get_first_page()  argument
469 	struct page *first_page = zspage->first_page;  in get_first_page()
485 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
487 	return zspage->freeobj;  in get_freeobj()
490 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
492 	zspage->freeobj = obj;  in set_freeobj()
495 static void get_zspage_mapping(struct zspage *zspage,  in get_zspage_mapping()  argument
499 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage_mapping()
501 	*fullness = zspage->fullness;  in get_zspage_mapping()
502 	*class_idx = zspage->class;  in get_zspage_mapping()
506 					     struct zspage *zspage)  in zspage_class()  argument
508 	return pool->size_class[zspage->class];  in zspage_class()
511 static void set_zspage_mapping(struct zspage *zspage,  in set_zspage_mapping()  argument
515 	zspage->class = class_idx;  in set_zspage_mapping()
516 	zspage->fullness = fullness;  in set_zspage_mapping()
681 						struct zspage *zspage)  in get_fullness_group()  argument
686 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
708 				struct zspage *zspage,  in insert_zspage()  argument
711 	struct zspage *head;  in insert_zspage()
715 					struct zspage, list);  in insert_zspage()
720 	if (head && get_zspage_inuse(zspage) < get_zspage_inuse(head))  in insert_zspage()
721 		list_add(&zspage->list, &head->list);  in insert_zspage()
723 		list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
731 				struct zspage *zspage,  in remove_zspage()  argument
736 	list_del_init(&zspage->list);  in remove_zspage()
750 						struct zspage *zspage)  in fix_fullness_group()  argument
755 	get_zspage_mapping(zspage, &class_idx, &currfg);  in fix_fullness_group()
756 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
760 	remove_zspage(class, zspage, currfg);  in fix_fullness_group()
761 	insert_zspage(class, zspage, newfg);  in fix_fullness_group()
762 	set_zspage_mapping(zspage, class_idx, newfg);  in fix_fullness_group()
803 static struct zspage *get_zspage(struct page *page)  in get_zspage()
805 	struct zspage *zspage = (struct zspage *)page_private(page);  in get_zspage()  local
807 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
808 	return zspage;  in get_zspage()
813 	struct zspage *zspage = get_zspage(page);  in get_next_page()  local
815 	if (unlikely(ZsHugePage(zspage)))  in get_next_page()
865 	struct zspage *zspage = get_zspage(page);  in obj_allocated()  local
867 	if (unlikely(ZsHugePage(zspage))) {  in obj_allocated()
889 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
893 	for (cursor = get_first_page(zspage); cursor != NULL; cursor =  in trylock_zspage()
903 	for (cursor = get_first_page(zspage); cursor != fail; cursor =  in trylock_zspage()
911 				struct zspage *zspage)  in __free_zspage()  argument
917 	get_zspage_mapping(zspage, &class_idx, &fg);  in __free_zspage()
921 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
924 	next = page = get_first_page(zspage);  in __free_zspage()
935 	cache_free_zspage(pool, zspage);  in __free_zspage()
943 				struct zspage *zspage)  in free_zspage()  argument
945 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
946 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
953 	if (!trylock_zspage(zspage)) {  in free_zspage()
958 	remove_zspage(class, zspage, ZS_EMPTY);  in free_zspage()
959 	__free_zspage(pool, class, zspage);  in free_zspage()
963 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
967 	struct page *page = get_first_page(zspage);  in init_zspage()
1004 	set_freeobj(zspage, 0);  in init_zspage()
1007 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
1025 		set_page_private(page, (unsigned long)zspage);  in create_page_chain()
1028 			zspage->first_page = page;  in create_page_chain()
1032 				SetZsHugePage(zspage);  in create_page_chain()
1043 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
1049 	struct zspage *zspage = cache_alloc_zspage(pool, gfp);  in alloc_zspage()  local
1051 	if (!zspage)  in alloc_zspage()
1054 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
1055 	migrate_lock_init(zspage);  in alloc_zspage()
1066 			cache_free_zspage(pool, zspage);  in alloc_zspage()
1074 	create_page_chain(class, zspage, pages);  in alloc_zspage()
1075 	init_zspage(class, zspage);  in alloc_zspage()
1076 	zspage->pool = pool;  in alloc_zspage()
1078 	return zspage;  in alloc_zspage()
1081 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
1084 	struct zspage *zspage;  in find_get_zspage()  local
1087 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
1088 				struct zspage, list);  in find_get_zspage()
1089 		if (zspage)  in find_get_zspage()
1093 	return zspage;  in find_get_zspage()
1203 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1205 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1232 	struct zspage *zspage;  in zs_map_object()  local
1253 	zspage = get_zspage(page);  in zs_map_object()
1261 	migrate_read_lock(zspage);  in zs_map_object()
1264 	class = zspage_class(pool, zspage);  in zs_map_object()
1284 	if (likely(!ZsHugePage(zspage)))  in zs_map_object()
1293 	struct zspage *zspage;  in zs_unmap_object()  local
1303 	zspage = get_zspage(page);  in zs_unmap_object()
1304 	class = zspage_class(pool, zspage);  in zs_unmap_object()
1321 	migrate_read_unlock(zspage);  in zs_unmap_object()
1345 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1356 	class = pool->size_class[zspage->class];  in obj_malloc()
1358 	obj = get_freeobj(zspage);  in obj_malloc()
1363 	m_page = get_first_page(zspage);  in obj_malloc()
1370 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1371 	if (likely(!ZsHugePage(zspage)))  in obj_malloc()
1376 		zspage->first_page->index = handle;  in obj_malloc()
1379 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1402 	struct zspage *zspage;  in zs_malloc()  local
1417 	zspage = find_get_zspage(class);  in zs_malloc()
1418 	if (likely(zspage)) {  in zs_malloc()
1419 		obj = obj_malloc(pool, zspage, handle);  in zs_malloc()
1421 		fix_fullness_group(class, zspage);  in zs_malloc()
1431 	zspage = alloc_zspage(pool, class, gfp);  in zs_malloc()
1432 	if (!zspage) {  in zs_malloc()
1438 	obj = obj_malloc(pool, zspage, handle);  in zs_malloc()
1439 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1440 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1441 	set_zspage_mapping(zspage, class->index, newfg);  in zs_malloc()
1449 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1459 	struct zspage *zspage;  in obj_free()  local
1467 	zspage = get_zspage(f_page);  in obj_free()
1473 	if (likely(!ZsHugePage(zspage)))  in obj_free()
1474 		link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1478 	set_freeobj(zspage, f_objidx);  in obj_free()
1479 	mod_zspage_inuse(zspage, -1);  in obj_free()
1484 	struct zspage *zspage;  in zs_free()  local
1500 	zspage = get_zspage(f_page);  in zs_free()
1501 	class = zspage_class(pool, zspage);  in zs_free()
1507 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1511 	free_zspage(pool, class, zspage);  in zs_free()
1670 static struct zspage *isolate_zspage(struct size_class *class, bool source)  in isolate_zspage()
1673 	struct zspage *zspage;  in isolate_zspage()  local
1682 		zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],  in isolate_zspage()
1683 							struct zspage, list);  in isolate_zspage()
1684 		if (zspage) {  in isolate_zspage()
1685 			remove_zspage(class, zspage, fg[i]);  in isolate_zspage()
1686 			return zspage;  in isolate_zspage()
1690 	return zspage;  in isolate_zspage()
1701 			struct zspage *zspage)  in putback_zspage()  argument
1705 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1706 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1707 	set_zspage_mapping(zspage, class->index, fullness);  in putback_zspage()
1717 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1730 		migrate_read_lock(zspage);  in lock_zspage()
1731 		page = get_first_page(zspage);  in lock_zspage()
1735 		migrate_read_unlock(zspage);  in lock_zspage()
1746 			migrate_read_unlock(zspage);  in lock_zspage()
1749 			migrate_read_lock(zspage);  in lock_zspage()
1752 	migrate_read_unlock(zspage);  in lock_zspage()
1755 static void migrate_lock_init(struct zspage *zspage)  in migrate_lock_init()  argument
1757 	rwlock_init(&zspage->lock);  in migrate_lock_init()
1760 static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock)  in migrate_read_lock()  argument
1762 	read_lock(&zspage->lock);  in migrate_read_lock()
1765 static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock)  in migrate_read_unlock()  argument
1767 	read_unlock(&zspage->lock);  in migrate_read_unlock()
1770 static void migrate_write_lock(struct zspage *zspage)  in migrate_write_lock()  argument
1772 	write_lock(&zspage->lock);  in migrate_write_lock()
1775 static void migrate_write_lock_nested(struct zspage *zspage)  in migrate_write_lock_nested()  argument
1777 	write_lock_nested(&zspage->lock, SINGLE_DEPTH_NESTING);  in migrate_write_lock_nested()
1780 static void migrate_write_unlock(struct zspage *zspage)  in migrate_write_unlock()  argument
1782 	write_unlock(&zspage->lock);  in migrate_write_unlock()
1786 static void inc_zspage_isolation(struct zspage *zspage)  in inc_zspage_isolation()  argument
1788 	zspage->isolated++;  in inc_zspage_isolation()
1791 static void dec_zspage_isolation(struct zspage *zspage)  in dec_zspage_isolation()  argument
1793 	VM_BUG_ON(zspage->isolated == 0);  in dec_zspage_isolation()
1794 	zspage->isolated--;  in dec_zspage_isolation()
1799 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1806 	page = get_first_page(zspage);  in replace_sub_page()
1815 	create_page_chain(class, zspage, pages);  in replace_sub_page()
1817 	if (unlikely(ZsHugePage(zspage)))  in replace_sub_page()
1824 	struct zspage *zspage;  in zs_page_isolate()  local
1833 	zspage = get_zspage(page);  in zs_page_isolate()
1834 	migrate_write_lock(zspage);  in zs_page_isolate()
1835 	inc_zspage_isolation(zspage);  in zs_page_isolate()
1836 	migrate_write_unlock(zspage);  in zs_page_isolate()
1846 	struct zspage *zspage;  in zs_page_migrate()  local
1866 	zspage = get_zspage(page);  in zs_page_migrate()
1867 	pool = zspage->pool;  in zs_page_migrate()
1874 	class = zspage_class(pool, zspage);  in zs_page_migrate()
1881 	migrate_write_lock(zspage);  in zs_page_migrate()
1906 	replace_sub_page(class, zspage, newpage, page);  in zs_page_migrate()
1913 	dec_zspage_isolation(zspage);  in zs_page_migrate()
1914 	migrate_write_unlock(zspage);  in zs_page_migrate()
1930 	struct zspage *zspage;  in zs_page_putback()  local
1935 	zspage = get_zspage(page);  in zs_page_putback()
1936 	migrate_write_lock(zspage);  in zs_page_putback()
1937 	dec_zspage_isolation(zspage);  in zs_page_putback()
1938 	migrate_write_unlock(zspage);  in zs_page_putback()
1957 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
1972 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
1973 		list_del(&zspage->list);  in async_free_zspage()
1974 		lock_zspage(zspage);  in async_free_zspage()
1976 		get_zspage_mapping(zspage, &class_idx, &fullness);  in async_free_zspage()
1980 		__free_zspage(pool, class, zspage);  in async_free_zspage()
2000 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
2002 	struct page *page = get_first_page(zspage);  in SetZsPageMovable()
2038 	struct zspage *src_zspage;  in __zs_compact()
2039 	struct zspage *dst_zspage = NULL;  in __zs_compact()