69 		struct z_erofs_pcluster *pcl;  in z_erofs_alloc_pcluster()  local
74 		pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS);  in z_erofs_alloc_pcluster()
75 		if (!pcl)  in z_erofs_alloc_pcluster()
77 		pcl->pclusterpages = nrpages;  in z_erofs_alloc_pcluster()
78 		return pcl;  in z_erofs_alloc_pcluster()
83 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)  in z_erofs_free_pcluster()  argument
90 		if (pcl->pclusterpages > pcs->maxpages)  in z_erofs_free_pcluster()
93 		kmem_cache_free(pcs->slab, pcl);  in z_erofs_free_pcluster()
204 	struct z_erofs_pcluster *pcl, *tailpcl;  member
241 	struct z_erofs_pcluster *pcl = clt->pcl;  in preload_compressed_pages()  local
251 	pages = pcl->compressed_pages;  in preload_compressed_pages()
252 	index = pcl->obj.index;  in preload_compressed_pages()
253 	for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) {  in preload_compressed_pages()
310 	struct z_erofs_pcluster *const pcl =  in erofs_try_to_free_all_cached_pages()  local
318 	for (i = 0; i < pcl->pclusterpages; ++i) {  in erofs_try_to_free_all_cached_pages()
319 		struct page *page = pcl->compressed_pages[i];  in erofs_try_to_free_all_cached_pages()
332 		WRITE_ONCE(pcl->compressed_pages[i], NULL);  in erofs_try_to_free_all_cached_pages()
341 	struct z_erofs_pcluster *const pcl = (void *)page_private(page);  in erofs_try_to_free_cached_page()  local
344 	if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {  in erofs_try_to_free_cached_page()
347 		for (i = 0; i < pcl->pclusterpages; ++i) {  in erofs_try_to_free_cached_page()
348 			if (pcl->compressed_pages[i] == page) {  in erofs_try_to_free_cached_page()
349 				WRITE_ONCE(pcl->compressed_pages[i], NULL);  in erofs_try_to_free_cached_page()
354 		erofs_workgroup_unfreeze(&pcl->obj, 1);  in erofs_try_to_free_cached_page()
366 	struct z_erofs_pcluster *const pcl = clt->pcl;  in z_erofs_try_inplace_io()  local
368 	while (clt->icpage_ptr > pcl->compressed_pages)  in z_erofs_try_inplace_io()
395 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_try_to_claim_pcluster()  local
399 	if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,  in z_erofs_try_to_claim_pcluster()
401 		*owned_head = &pcl->next;  in z_erofs_try_to_claim_pcluster()
411 	if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,  in z_erofs_try_to_claim_pcluster()
426 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_lookup_collection()  local
431 	if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {  in z_erofs_lookup_collection()
436 	cl = z_erofs_primarycollection(pcl);  in z_erofs_lookup_collection()
442 	length = READ_ONCE(pcl->length);  in z_erofs_lookup_collection()
455 		       length != cmpxchg_relaxed(&pcl->length, length, llen)) {  in z_erofs_lookup_collection()
457 			length = READ_ONCE(pcl->length);  in z_erofs_lookup_collection()
463 		clt->tailpcl = pcl;  in z_erofs_lookup_collection()
474 	struct z_erofs_pcluster *pcl;  in z_erofs_register_collection()  local
480 	pcl = z_erofs_alloc_pcluster(map->m_plen >> PAGE_SHIFT);  in z_erofs_register_collection()
481 	if (IS_ERR(pcl))  in z_erofs_register_collection()
482 		return PTR_ERR(pcl);  in z_erofs_register_collection()
484 	atomic_set(&pcl->obj.refcount, 1);  in z_erofs_register_collection()
485 	pcl->obj.index = map->m_pa >> PAGE_SHIFT;  in z_erofs_register_collection()
487 	pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |  in z_erofs_register_collection()
492 		pcl->algorithmformat = Z_EROFS_COMPRESSION_LZ4;  in z_erofs_register_collection()
494 		pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;  in z_erofs_register_collection()
497 	pcl->next = clt->owned_head;  in z_erofs_register_collection()
500 	cl = z_erofs_primarycollection(pcl);  in z_erofs_register_collection()
510 	grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);  in z_erofs_register_collection()
516 	if (grp != &pcl->obj) {  in z_erofs_register_collection()
517 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_register_collection()
523 		clt->tailpcl = pcl;  in z_erofs_register_collection()
524 	clt->owned_head = &pcl->next;  in z_erofs_register_collection()
525 	clt->pcl = pcl;  in z_erofs_register_collection()
531 	z_erofs_free_pcluster(pcl);  in z_erofs_register_collection()
555 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_collector_begin()
567 		erofs_workgroup_put(&clt->pcl->obj);  in z_erofs_collector_begin()
576 	clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages;  in z_erofs_collector_begin()
595 	struct z_erofs_pcluster *const pcl =  in erofs_workgroup_free_rcu()  local
597 	struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl);  in erofs_workgroup_free_rcu()
604 	struct z_erofs_pcluster *const pcl =  in z_erofs_collection_put()  local
607 	erofs_workgroup_put(&pcl->obj);  in z_erofs_collection_put()
838 				       struct z_erofs_pcluster *pcl,  in z_erofs_decompress_pcluster()  argument
853 	cl = z_erofs_primarycollection(pcl);  in z_erofs_decompress_pcluster()
921 	compressed_pages = pcl->compressed_pages;  in z_erofs_decompress_pcluster()
923 	for (i = 0; i < pcl->pclusterpages; ++i) {  in z_erofs_decompress_pcluster()
967 	llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;  in z_erofs_decompress_pcluster()
970 		partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);  in z_erofs_decompress_pcluster()
976 	inputsize = pcl->pclusterpages * PAGE_SIZE;  in z_erofs_decompress_pcluster()
984 					.alg = pcl->algorithmformat,  in z_erofs_decompress_pcluster()
991 	for (i = 0; i < pcl->pclusterpages; ++i) {  in z_erofs_decompress_pcluster()
1029 	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);  in z_erofs_decompress_pcluster()
1044 		struct z_erofs_pcluster *pcl;  in z_erofs_decompress_queue()  local
1052 		pcl = container_of(owned, struct z_erofs_pcluster, next);  in z_erofs_decompress_queue()
1053 		owned = READ_ONCE(pcl->next);  in z_erofs_decompress_queue()
1055 		z_erofs_decompress_pcluster(io->sb, pcl, pagepool);  in z_erofs_decompress_queue()
1072 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,  in pickup_page_for_submission()  argument
1078 	const pgoff_t index = pcl->obj.index;  in pickup_page_for_submission()
1088 	page = READ_ONCE(pcl->compressed_pages[nr]);  in pickup_page_for_submission()
1113 		WRITE_ONCE(pcl->compressed_pages[nr], page);  in pickup_page_for_submission()
1139 		WRITE_ONCE(pcl->compressed_pages[nr], page);  in pickup_page_for_submission()
1151 			set_page_private(page, (unsigned long)pcl);  in pickup_page_for_submission()
1175 	if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {  in pickup_page_for_submission()
1186 	attach_page_private(page, pcl);  in pickup_page_for_submission()
1239 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,  in move_to_bypass_jobqueue()  argument
1250 	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);  in move_to_bypass_jobqueue()
1253 	WRITE_ONCE(*bypass_qtail, &pcl->next);  in move_to_bypass_jobqueue()
1255 	qtail[JQ_BYPASS] = &pcl->next;  in move_to_bypass_jobqueue()
1282 		struct z_erofs_pcluster *pcl;  in z_erofs_submit_queue()  local
1291 		pcl = container_of(owned_head, struct z_erofs_pcluster, next);  in z_erofs_submit_queue()
1293 		cur = pcl->obj.index;  in z_erofs_submit_queue()
1294 		end = cur + pcl->pclusterpages;  in z_erofs_submit_queue()
1297 		owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,  in z_erofs_submit_queue()
1303 			page = pickup_page_for_submission(pcl, i++, pagepool,  in z_erofs_submit_queue()
1337 			qtail[JQ_SUBMIT] = &pcl->next;  in z_erofs_submit_queue()
1339 			move_to_bypass_jobqueue(pcl, qtail, owned_head);  in z_erofs_submit_queue()