301 	struct z3fold_header *zhdr = page_address(page);  in init_z3fold_page()  local
311 		return zhdr;  in init_z3fold_page()
317 	spin_lock_init(&zhdr->page_lock);  in init_z3fold_page()
318 	kref_init(&zhdr->refcount);  in init_z3fold_page()
319 	zhdr->first_chunks = 0;  in init_z3fold_page()
320 	zhdr->middle_chunks = 0;  in init_z3fold_page()
321 	zhdr->last_chunks = 0;  in init_z3fold_page()
322 	zhdr->first_num = 0;  in init_z3fold_page()
323 	zhdr->start_middle = 0;  in init_z3fold_page()
324 	zhdr->cpu = -1;  in init_z3fold_page()
325 	zhdr->slots = slots;  in init_z3fold_page()
326 	zhdr->pool = pool;  in init_z3fold_page()
327 	INIT_LIST_HEAD(&zhdr->buddy);  in init_z3fold_page()
328 	INIT_WORK(&zhdr->work, compact_page_work);  in init_z3fold_page()
329 	return zhdr;  in init_z3fold_page()
345 static inline void z3fold_page_lock(struct z3fold_header *zhdr)  in z3fold_page_lock()  argument
347 	spin_lock(&zhdr->page_lock);  in z3fold_page_lock()
351 static inline int z3fold_page_trylock(struct z3fold_header *zhdr)  in z3fold_page_trylock()  argument
353 	return spin_trylock(&zhdr->page_lock);  in z3fold_page_trylock()
357 static inline void z3fold_page_unlock(struct z3fold_header *zhdr)  in z3fold_page_unlock()  argument
359 	spin_unlock(&zhdr->page_lock);  in z3fold_page_unlock()
363 static inline int __idx(struct z3fold_header *zhdr, enum buddy bud)  in __idx()  argument
365 	return (bud + zhdr->first_num) & BUDDY_MASK;  in __idx()
372 static unsigned long __encode_handle(struct z3fold_header *zhdr,  in __encode_handle()  argument
376 	unsigned long h = (unsigned long)zhdr;  in __encode_handle()
387 	idx = __idx(zhdr, bud);  in __encode_handle()
390 		h |= (zhdr->last_chunks << BUDDY_SHIFT);  in __encode_handle()
396 static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud)  in encode_handle()  argument
398 	return __encode_handle(zhdr, zhdr->slots, bud);  in encode_handle()
427 	struct z3fold_header *zhdr;  in handle_to_buddy()  local
432 	zhdr = (struct z3fold_header *)(addr & PAGE_MASK);  in handle_to_buddy()
433 	return (addr - zhdr->first_num) & BUDDY_MASK;  in handle_to_buddy()
436 static inline struct z3fold_pool *zhdr_to_pool(struct z3fold_header *zhdr)  in zhdr_to_pool()  argument
438 	return zhdr->pool;  in zhdr_to_pool()
441 static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked)  in __release_z3fold_page()  argument
443 	struct page *page = virt_to_page(zhdr);  in __release_z3fold_page()
444 	struct z3fold_pool *pool = zhdr_to_pool(zhdr);  in __release_z3fold_page()
446 	WARN_ON(!list_empty(&zhdr->buddy));  in __release_z3fold_page()
454 		z3fold_page_unlock(zhdr);  in __release_z3fold_page()
456 	list_add(&zhdr->buddy, &pool->stale);  in __release_z3fold_page()
464 	struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,  in release_z3fold_page()  local
466 	__release_z3fold_page(zhdr, false);  in release_z3fold_page()
471 	struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,  in release_z3fold_page_locked()  local
473 	WARN_ON(z3fold_page_trylock(zhdr));  in release_z3fold_page_locked()
474 	__release_z3fold_page(zhdr, true);  in release_z3fold_page_locked()
479 	struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,  in release_z3fold_page_locked_list()  local
481 	struct z3fold_pool *pool = zhdr_to_pool(zhdr);  in release_z3fold_page_locked_list()
483 	list_del_init(&zhdr->buddy);  in release_z3fold_page_locked_list()
486 	WARN_ON(z3fold_page_trylock(zhdr));  in release_z3fold_page_locked_list()
487 	__release_z3fold_page(zhdr, true);  in release_z3fold_page_locked_list()
496 		struct z3fold_header *zhdr = list_first_entry(&pool->stale,  in free_pages_work()  local
498 		struct page *page = virt_to_page(zhdr);  in free_pages_work()
500 		list_del(&zhdr->buddy);  in free_pages_work()
504 		cancel_work_sync(&zhdr->work);  in free_pages_work()
516 static int num_free_chunks(struct z3fold_header *zhdr)  in num_free_chunks()  argument
524 	if (zhdr->middle_chunks != 0) {  in num_free_chunks()
525 		int nfree_before = zhdr->first_chunks ?  in num_free_chunks()
526 			0 : zhdr->start_middle - ZHDR_CHUNKS;  in num_free_chunks()
527 		int nfree_after = zhdr->last_chunks ?  in num_free_chunks()
529 				(zhdr->start_middle + zhdr->middle_chunks);  in num_free_chunks()
532 		nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks;  in num_free_chunks()
538 				struct z3fold_header *zhdr)  in add_to_unbuddied()  argument
540 	if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 ||  in add_to_unbuddied()
541 			zhdr->middle_chunks == 0) {  in add_to_unbuddied()
544 		int freechunks = num_free_chunks(zhdr);  in add_to_unbuddied()
546 		list_add(&zhdr->buddy, &unbuddied[freechunks]);  in add_to_unbuddied()
548 		zhdr->cpu = smp_processor_id();  in add_to_unbuddied()
553 static inline void *mchunk_memmove(struct z3fold_header *zhdr,  in mchunk_memmove()  argument
556 	void *beg = zhdr;  in mchunk_memmove()
558 		       beg + (zhdr->start_middle << CHUNK_SHIFT),  in mchunk_memmove()
559 		       zhdr->middle_chunks << CHUNK_SHIFT);  in mchunk_memmove()
564 static int z3fold_compact_page(struct z3fold_header *zhdr)  in z3fold_compact_page()  argument
566 	struct page *page = virt_to_page(zhdr);  in z3fold_compact_page()
574 	if (zhdr->middle_chunks == 0)  in z3fold_compact_page()
577 	if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {  in z3fold_compact_page()
579 		mchunk_memmove(zhdr, ZHDR_CHUNKS);  in z3fold_compact_page()
580 		zhdr->first_chunks = zhdr->middle_chunks;  in z3fold_compact_page()
581 		zhdr->middle_chunks = 0;  in z3fold_compact_page()
582 		zhdr->start_middle = 0;  in z3fold_compact_page()
583 		zhdr->first_num++;  in z3fold_compact_page()
591 	if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 &&  in z3fold_compact_page()
592 	    zhdr->start_middle - (zhdr->first_chunks + ZHDR_CHUNKS) >=  in z3fold_compact_page()
594 		mchunk_memmove(zhdr, zhdr->first_chunks + ZHDR_CHUNKS);  in z3fold_compact_page()
595 		zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS;  in z3fold_compact_page()
597 	} else if (zhdr->last_chunks != 0 && zhdr->first_chunks == 0 &&  in z3fold_compact_page()
598 		   TOTAL_CHUNKS - (zhdr->last_chunks + zhdr->start_middle  in z3fold_compact_page()
599 					+ zhdr->middle_chunks) >=  in z3fold_compact_page()
601 		unsigned short new_start = TOTAL_CHUNKS - zhdr->last_chunks -  in z3fold_compact_page()
602 			zhdr->middle_chunks;  in z3fold_compact_page()
603 		mchunk_memmove(zhdr, new_start);  in z3fold_compact_page()
604 		zhdr->start_middle = new_start;  in z3fold_compact_page()
611 static void do_compact_page(struct z3fold_header *zhdr, bool locked)  in do_compact_page()  argument
613 	struct z3fold_pool *pool = zhdr_to_pool(zhdr);  in do_compact_page()
616 	page = virt_to_page(zhdr);  in do_compact_page()
618 		WARN_ON(z3fold_page_trylock(zhdr));  in do_compact_page()
620 		z3fold_page_lock(zhdr);  in do_compact_page()
622 		z3fold_page_unlock(zhdr);  in do_compact_page()
626 	list_del_init(&zhdr->buddy);  in do_compact_page()
629 	if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {  in do_compact_page()
637 		z3fold_page_unlock(zhdr);  in do_compact_page()
641 	z3fold_compact_page(zhdr);  in do_compact_page()
642 	add_to_unbuddied(pool, zhdr);  in do_compact_page()
643 	z3fold_page_unlock(zhdr);  in do_compact_page()
648 	struct z3fold_header *zhdr = container_of(w, struct z3fold_header,  in compact_page_work()  local
651 	do_compact_page(zhdr, false);  in compact_page_work()
658 	struct z3fold_header *zhdr = NULL;  in __z3fold_alloc()  local
669 		zhdr = list_first_entry_or_null(READ_ONCE(l),  in __z3fold_alloc()
672 		if (!zhdr)  in __z3fold_alloc()
678 		if (unlikely(zhdr != list_first_entry(READ_ONCE(l),  in __z3fold_alloc()
680 		    !z3fold_page_trylock(zhdr)) {  in __z3fold_alloc()
682 			zhdr = NULL;  in __z3fold_alloc()
688 		list_del_init(&zhdr->buddy);  in __z3fold_alloc()
689 		zhdr->cpu = -1;  in __z3fold_alloc()
692 		page = virt_to_page(zhdr);  in __z3fold_alloc()
694 			z3fold_page_unlock(zhdr);  in __z3fold_alloc()
695 			zhdr = NULL;  in __z3fold_alloc()
708 		kref_get(&zhdr->refcount);  in __z3fold_alloc()
713 	if (!zhdr) {  in __z3fold_alloc()
724 			zhdr = list_first_entry_or_null(READ_ONCE(l),  in __z3fold_alloc()
727 			if (!zhdr || !z3fold_page_trylock(zhdr)) {  in __z3fold_alloc()
729 				zhdr = NULL;  in __z3fold_alloc()
732 			list_del_init(&zhdr->buddy);  in __z3fold_alloc()
733 			zhdr->cpu = -1;  in __z3fold_alloc()
736 			page = virt_to_page(zhdr);  in __z3fold_alloc()
738 				z3fold_page_unlock(zhdr);  in __z3fold_alloc()
739 				zhdr = NULL;  in __z3fold_alloc()
744 			kref_get(&zhdr->refcount);  in __z3fold_alloc()
749 	return zhdr;  in __z3fold_alloc()
868 	struct z3fold_header *zhdr = NULL;  in z3fold_alloc()  local
883 		zhdr = __z3fold_alloc(pool, size, can_sleep);  in z3fold_alloc()
884 		if (zhdr) {  in z3fold_alloc()
885 			if (zhdr->first_chunks == 0) {  in z3fold_alloc()
886 				if (zhdr->middle_chunks != 0 &&  in z3fold_alloc()
887 				    chunks >= zhdr->start_middle)  in z3fold_alloc()
891 			} else if (zhdr->last_chunks == 0)  in z3fold_alloc()
893 			else if (zhdr->middle_chunks == 0)  in z3fold_alloc()
896 				if (kref_put(&zhdr->refcount,  in z3fold_alloc()
900 					z3fold_page_unlock(zhdr);  in z3fold_alloc()
905 			page = virt_to_page(zhdr);  in z3fold_alloc()
914 		zhdr = list_first_entry_or_null(&pool->stale,  in z3fold_alloc()
921 		if (zhdr) {  in z3fold_alloc()
922 			list_del(&zhdr->buddy);  in z3fold_alloc()
924 			cancel_work_sync(&zhdr->work);  in z3fold_alloc()
925 			page = virt_to_page(zhdr);  in z3fold_alloc()
936 	zhdr = init_z3fold_page(page, bud == HEADLESS, pool, gfp);  in z3fold_alloc()
937 	if (!zhdr) {  in z3fold_alloc()
957 	z3fold_page_lock(zhdr);  in z3fold_alloc()
961 		zhdr->first_chunks = chunks;  in z3fold_alloc()
963 		zhdr->last_chunks = chunks;  in z3fold_alloc()
965 		zhdr->middle_chunks = chunks;  in z3fold_alloc()
966 		zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS;  in z3fold_alloc()
968 	add_to_unbuddied(pool, zhdr);  in z3fold_alloc()
978 	*handle = encode_handle(zhdr, bud);  in z3fold_alloc()
981 		z3fold_page_unlock(zhdr);  in z3fold_alloc()
998 	struct z3fold_header *zhdr;  in z3fold_free()  local
1003 	zhdr = handle_to_z3fold_header(handle);  in z3fold_free()
1004 	page = virt_to_page(zhdr);  in z3fold_free()
1024 	z3fold_page_lock(zhdr);  in z3fold_free()
1029 		zhdr->first_chunks = 0;  in z3fold_free()
1032 		zhdr->middle_chunks = 0;  in z3fold_free()
1035 		zhdr->last_chunks = 0;  in z3fold_free()
1040 		z3fold_page_unlock(zhdr);  in z3fold_free()
1045 	if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) {  in z3fold_free()
1051 		z3fold_page_unlock(zhdr);  in z3fold_free()
1056 		z3fold_page_unlock(zhdr);  in z3fold_free()
1060 	if (zhdr->cpu < 0 || !cpu_online(zhdr->cpu)) {  in z3fold_free()
1062 		list_del_init(&zhdr->buddy);  in z3fold_free()
1064 		zhdr->cpu = -1;  in z3fold_free()
1065 		kref_get(&zhdr->refcount);  in z3fold_free()
1066 		do_compact_page(zhdr, true);  in z3fold_free()
1070 	kref_get(&zhdr->refcount);  in z3fold_free()
1071 	queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);  in z3fold_free()
1073 	z3fold_page_unlock(zhdr);  in z3fold_free()
1115 	struct z3fold_header *zhdr = NULL;  in z3fold_reclaim_page()  local
1147 			zhdr = page_address(page);  in z3fold_reclaim_page()
1151 			if (!z3fold_page_trylock(zhdr)) {  in z3fold_reclaim_page()
1153 				zhdr = NULL;  in z3fold_reclaim_page()
1156 			kref_get(&zhdr->refcount);  in z3fold_reclaim_page()
1157 			list_del_init(&zhdr->buddy);  in z3fold_reclaim_page()
1158 			zhdr->cpu = -1;  in z3fold_reclaim_page()
1162 		if (!zhdr)  in z3fold_reclaim_page()
1178 			if (zhdr->first_chunks)  in z3fold_reclaim_page()
1179 				first_handle = __encode_handle(zhdr, &slots,  in z3fold_reclaim_page()
1181 			if (zhdr->middle_chunks)  in z3fold_reclaim_page()
1182 				middle_handle = __encode_handle(zhdr, &slots,  in z3fold_reclaim_page()
1184 			if (zhdr->last_chunks)  in z3fold_reclaim_page()
1185 				last_handle = __encode_handle(zhdr, &slots,  in z3fold_reclaim_page()
1191 			z3fold_page_unlock(zhdr);  in z3fold_reclaim_page()
1193 			first_handle = __encode_handle(zhdr, &slots, HEADLESS);  in z3fold_reclaim_page()
1225 			z3fold_page_lock(zhdr);  in z3fold_reclaim_page()
1226 			if (kref_put(&zhdr->refcount,  in z3fold_reclaim_page()
1239 			z3fold_page_unlock(zhdr);  in z3fold_reclaim_page()
1262 	struct z3fold_header *zhdr;  in z3fold_map()  local
1267 	zhdr = handle_to_z3fold_header(handle);  in z3fold_map()
1268 	addr = zhdr;  in z3fold_map()
1269 	page = virt_to_page(zhdr);  in z3fold_map()
1274 	z3fold_page_lock(zhdr);  in z3fold_map()
1281 		addr += zhdr->start_middle << CHUNK_SHIFT;  in z3fold_map()
1295 		zhdr->mapped_count++;  in z3fold_map()
1296 	z3fold_page_unlock(zhdr);  in z3fold_map()
1308 	struct z3fold_header *zhdr;  in z3fold_unmap()  local
1312 	zhdr = handle_to_z3fold_header(handle);  in z3fold_unmap()
1313 	page = virt_to_page(zhdr);  in z3fold_unmap()
1318 	z3fold_page_lock(zhdr);  in z3fold_unmap()
1322 	zhdr->mapped_count--;  in z3fold_unmap()
1323 	z3fold_page_unlock(zhdr);  in z3fold_unmap()
1339 	struct z3fold_header *zhdr;  in z3fold_page_isolate()  local
1349 	zhdr = page_address(page);  in z3fold_page_isolate()
1350 	z3fold_page_lock(zhdr);  in z3fold_page_isolate()
1355 	pool = zhdr_to_pool(zhdr);  in z3fold_page_isolate()
1357 	if (zhdr->mapped_count == 0) {  in z3fold_page_isolate()
1358 		kref_get(&zhdr->refcount);  in z3fold_page_isolate()
1359 		if (!list_empty(&zhdr->buddy))  in z3fold_page_isolate()
1360 			list_del_init(&zhdr->buddy);  in z3fold_page_isolate()
1365 		z3fold_page_unlock(zhdr);  in z3fold_page_isolate()
1369 	z3fold_page_unlock(zhdr);  in z3fold_page_isolate()
1376 	struct z3fold_header *zhdr, *new_zhdr;  in z3fold_page_migrate()  local
1384 	zhdr = page_address(page);  in z3fold_page_migrate()
1385 	pool = zhdr_to_pool(zhdr);  in z3fold_page_migrate()
1387 	if (!z3fold_page_trylock(zhdr)) {  in z3fold_page_migrate()
1390 	if (zhdr->mapped_count != 0) {  in z3fold_page_migrate()
1391 		z3fold_page_unlock(zhdr);  in z3fold_page_migrate()
1394 	if (work_pending(&zhdr->work)) {  in z3fold_page_migrate()
1395 		z3fold_page_unlock(zhdr);  in z3fold_page_migrate()
1399 	memcpy(new_zhdr, zhdr, PAGE_SIZE);  in z3fold_page_migrate()
1402 	z3fold_page_unlock(zhdr);  in z3fold_page_migrate()
1439 	struct z3fold_header *zhdr;  in z3fold_page_putback()  local
1442 	zhdr = page_address(page);  in z3fold_page_putback()
1443 	pool = zhdr_to_pool(zhdr);  in z3fold_page_putback()
1445 	z3fold_page_lock(zhdr);  in z3fold_page_putback()
1446 	if (!list_empty(&zhdr->buddy))  in z3fold_page_putback()
1447 		list_del_init(&zhdr->buddy);  in z3fold_page_putback()
1449 	if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {  in z3fold_page_putback()
1456 	z3fold_page_unlock(zhdr);  in z3fold_page_putback()