59 	struct z_erofs_pcluster *pcl = ptr;  in z_erofs_pcluster_init_once()  local
60 	struct z_erofs_collection *cl = z_erofs_primarycollection(pcl);  in z_erofs_pcluster_init_once()
67 		pcl->compressed_pages[i] = NULL;  in z_erofs_pcluster_init_once()
124 	struct z_erofs_pcluster *pcl, *tailpcl;  member
159 	const struct z_erofs_pcluster *pcl = clt->pcl;  in preload_compressed_pages()  local
160 	const unsigned int clusterpages = BIT(pcl->clusterbits);  in preload_compressed_pages()
162 	pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages);  in preload_compressed_pages()
168 	for (; pages < pcl->compressed_pages + clusterpages; ++pages) {  in preload_compressed_pages()
204 	struct z_erofs_pcluster *const pcl =  in erofs_try_to_free_all_cached_pages()  local
207 	const unsigned int clusterpages = BIT(pcl->clusterbits);  in erofs_try_to_free_all_cached_pages()
215 		struct page *page = pcl->compressed_pages[i];  in erofs_try_to_free_all_cached_pages()
228 		WRITE_ONCE(pcl->compressed_pages[i], NULL);  in erofs_try_to_free_all_cached_pages()
241 	struct z_erofs_pcluster *const pcl = (void *)page_private(page);  in erofs_try_to_free_cached_page()  local
242 	const unsigned int clusterpages = BIT(pcl->clusterbits);  in erofs_try_to_free_cached_page()
245 	if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {  in erofs_try_to_free_cached_page()
249 			if (pcl->compressed_pages[i] == page) {  in erofs_try_to_free_cached_page()
250 				WRITE_ONCE(pcl->compressed_pages[i], NULL);  in erofs_try_to_free_cached_page()
255 		erofs_workgroup_unfreeze(&pcl->obj, 1);  in erofs_try_to_free_cached_page()
269 	struct z_erofs_pcluster *const pcl = clt->pcl;  in z_erofs_try_inplace_io()  local
270 	const unsigned int clusterpages = BIT(pcl->clusterbits);  in z_erofs_try_inplace_io()
272 	while (clt->compressedpages < pcl->compressed_pages + clusterpages) {  in z_erofs_try_inplace_io()
301 try_to_claim_pcluster(struct z_erofs_pcluster *pcl,  in try_to_claim_pcluster()  argument
306 	if (pcl->next == Z_EROFS_PCLUSTER_NIL) {  in try_to_claim_pcluster()
308 		if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,  in try_to_claim_pcluster()
312 		*owned_head = &pcl->next;  in try_to_claim_pcluster()
315 	} else if (pcl->next == Z_EROFS_PCLUSTER_TAIL) {  in try_to_claim_pcluster()
321 		if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,  in try_to_claim_pcluster()
334 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_lookup_collection()  local
339 	if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {  in z_erofs_lookup_collection()
344 	cl = z_erofs_primarycollection(pcl);  in z_erofs_lookup_collection()
350 	length = READ_ONCE(pcl->length);  in z_erofs_lookup_collection()
363 		       length != cmpxchg_relaxed(&pcl->length, length, llen)) {  in z_erofs_lookup_collection()
365 			length = READ_ONCE(pcl->length);  in z_erofs_lookup_collection()
371 		clt->tailpcl = pcl;  in z_erofs_lookup_collection()
372 	clt->mode = try_to_claim_pcluster(pcl, &clt->owned_head);  in z_erofs_lookup_collection()
384 	struct z_erofs_pcluster *pcl;  in z_erofs_register_collection()  local
390 	pcl = kmem_cache_alloc(pcluster_cachep, GFP_NOFS);  in z_erofs_register_collection()
391 	if (!pcl)  in z_erofs_register_collection()
394 	atomic_set(&pcl->obj.refcount, 1);  in z_erofs_register_collection()
395 	pcl->obj.index = map->m_pa >> PAGE_SHIFT;  in z_erofs_register_collection()
397 	pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |  in z_erofs_register_collection()
402 		pcl->algorithmformat = Z_EROFS_COMPRESSION_LZ4;  in z_erofs_register_collection()
404 		pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;  in z_erofs_register_collection()
406 	pcl->clusterbits = EROFS_I(inode)->z_physical_clusterbits[0];  in z_erofs_register_collection()
407 	pcl->clusterbits -= PAGE_SHIFT;  in z_erofs_register_collection()
410 	pcl->next = clt->owned_head;  in z_erofs_register_collection()
413 	cl = z_erofs_primarycollection(pcl);  in z_erofs_register_collection()
427 	grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);  in z_erofs_register_collection()
433 	if (grp != &pcl->obj) {  in z_erofs_register_collection()
434 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_register_collection()
440 		clt->tailpcl = pcl;  in z_erofs_register_collection()
441 	clt->owned_head = &pcl->next;  in z_erofs_register_collection()
442 	clt->pcl = pcl;  in z_erofs_register_collection()
448 	kmem_cache_free(pcluster_cachep, pcl);  in z_erofs_register_collection()
472 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_collector_begin()
484 		erofs_workgroup_put(&clt->pcl->obj);  in z_erofs_collector_begin()
492 	clt->compressedpages = clt->pcl->compressed_pages;  in z_erofs_collector_begin()
514 	struct z_erofs_pcluster *const pcl =  in erofs_workgroup_free_rcu()  local
516 	struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl);  in erofs_workgroup_free_rcu()
523 	struct z_erofs_pcluster *const pcl =  in z_erofs_collection_put()  local
526 	erofs_workgroup_put(&pcl->obj);  in z_erofs_collection_put()
741 				       struct z_erofs_pcluster *pcl,  in z_erofs_decompress_pcluster()  argument
745 	const unsigned int clusterpages = BIT(pcl->clusterbits);  in z_erofs_decompress_pcluster()
757 	cl = z_erofs_primarycollection(pcl);  in z_erofs_decompress_pcluster()
825 	compressed_pages = pcl->compressed_pages;  in z_erofs_decompress_pcluster()
871 	llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;  in z_erofs_decompress_pcluster()
874 		partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);  in z_erofs_decompress_pcluster()
887 					.alg = pcl->algorithmformat,  in z_erofs_decompress_pcluster()
932 	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);  in z_erofs_decompress_pcluster()
947 		struct z_erofs_pcluster *pcl;  in z_erofs_decompress_queue()  local
955 		pcl = container_of(owned, struct z_erofs_pcluster, next);  in z_erofs_decompress_queue()
956 		owned = READ_ONCE(pcl->next);  in z_erofs_decompress_queue()
958 		z_erofs_decompress_pcluster(io->sb, pcl, pagepool);  in z_erofs_decompress_queue()
975 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,  in pickup_page_for_submission()  argument
981 	const pgoff_t index = pcl->obj.index;  in pickup_page_for_submission()
991 	page = READ_ONCE(pcl->compressed_pages[nr]);  in pickup_page_for_submission()
1028 		WRITE_ONCE(pcl->compressed_pages[nr], page);  in pickup_page_for_submission()
1040 			set_page_private(page, (unsigned long)pcl);  in pickup_page_for_submission()
1070 	if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {  in pickup_page_for_submission()
1083 		set_page_private(page, (unsigned long)pcl);  in pickup_page_for_submission()
1135 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,  in move_to_bypass_jobqueue()  argument
1146 	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);  in move_to_bypass_jobqueue()
1149 	WRITE_ONCE(*bypass_qtail, &pcl->next);  in move_to_bypass_jobqueue()
1151 	qtail[JQ_BYPASS] = &pcl->next;  in move_to_bypass_jobqueue()
1178 		struct z_erofs_pcluster *pcl;  in z_erofs_submit_queue()  local
1187 		pcl = container_of(owned_head, struct z_erofs_pcluster, next);  in z_erofs_submit_queue()
1189 		cur = pcl->obj.index;  in z_erofs_submit_queue()
1190 		end = cur + BIT(pcl->clusterbits);  in z_erofs_submit_queue()
1193 		owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,  in z_erofs_submit_queue()
1199 			page = pickup_page_for_submission(pcl, i++, pagepool,  in z_erofs_submit_queue()
1233 			qtail[JQ_SUBMIT] = &pcl->next;  in z_erofs_submit_queue()
1235 			move_to_bypass_jobqueue(pcl, qtail, owned_head);  in z_erofs_submit_queue()