135 	struct z_erofs_collector clt;  member
149 	.inode = __i, .clt = COLLECTOR_INIT(), \
155 static void preload_compressed_pages(struct z_erofs_collector *clt,  in preload_compressed_pages()  argument
159 	const struct z_erofs_pcluster *pcl = clt->pcl;  in preload_compressed_pages()
161 	struct page **pages = clt->compressedpages;  in preload_compressed_pages()
165 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED)  in preload_compressed_pages()
184 				clt->compressedpages = pages;  in preload_compressed_pages()
197 		clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;  in preload_compressed_pages()
266 static inline bool z_erofs_try_inplace_io(struct z_erofs_collector *clt,  in z_erofs_try_inplace_io()  argument
269 	struct z_erofs_pcluster *const pcl = clt->pcl;  in z_erofs_try_inplace_io()
272 	while (clt->compressedpages < pcl->compressed_pages + clusterpages) {  in z_erofs_try_inplace_io()
273 		if (!cmpxchg(clt->compressedpages++, NULL, page))  in z_erofs_try_inplace_io()
280 static int z_erofs_attach_page(struct z_erofs_collector *clt,  in z_erofs_attach_page()  argument
288 	if (clt->mode >= COLLECT_PRIMARY &&  in z_erofs_attach_page()
290 	    z_erofs_try_inplace_io(clt, page))  in z_erofs_attach_page()
293 	ret = z_erofs_pagevec_enqueue(&clt->vector,  in z_erofs_attach_page()
295 	clt->cl->vcnt += (unsigned int)ret;  in z_erofs_attach_page()
330 static int z_erofs_lookup_collection(struct z_erofs_collector *clt,  in z_erofs_lookup_collection()  argument
334 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_lookup_collection()
339 	if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {  in z_erofs_lookup_collection()
370 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_lookup_collection()
371 		clt->tailpcl = pcl;  in z_erofs_lookup_collection()
372 	clt->mode = try_to_claim_pcluster(pcl, &clt->owned_head);  in z_erofs_lookup_collection()
374 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_lookup_collection()
375 		clt->tailpcl = NULL;  in z_erofs_lookup_collection()
376 	clt->cl = cl;  in z_erofs_lookup_collection()
380 static int z_erofs_register_collection(struct z_erofs_collector *clt,  in z_erofs_register_collection()  argument
410 	pcl->next = clt->owned_head;  in z_erofs_register_collection()
411 	clt->mode = COLLECT_PRIMARY_FOLLOWED;  in z_erofs_register_collection()
434 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_register_collection()
439 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_register_collection()
440 		clt->tailpcl = pcl;  in z_erofs_register_collection()
441 	clt->owned_head = &pcl->next;  in z_erofs_register_collection()
442 	clt->pcl = pcl;  in z_erofs_register_collection()
443 	clt->cl = cl;  in z_erofs_register_collection()
452 static int z_erofs_collector_begin(struct z_erofs_collector *clt,  in z_erofs_collector_begin()  argument
459 	DBG_BUGON(clt->cl);  in z_erofs_collector_begin()
462 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);  in z_erofs_collector_begin()
463 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);  in z_erofs_collector_begin()
472 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_collector_begin()
474 		ret = z_erofs_register_collection(clt, inode, map);  in z_erofs_collector_begin()
482 	ret = z_erofs_lookup_collection(clt, inode, map);  in z_erofs_collector_begin()
484 		erofs_workgroup_put(&clt->pcl->obj);  in z_erofs_collector_begin()
489 	z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,  in z_erofs_collector_begin()
490 				  clt->cl->pagevec, clt->cl->vcnt);  in z_erofs_collector_begin()
492 	clt->compressedpages = clt->pcl->compressed_pages;  in z_erofs_collector_begin()
493 	if (clt->mode <= COLLECT_PRIMARY) /* cannot do in-place I/O */  in z_erofs_collector_begin()
494 		clt->compressedpages += Z_EROFS_CLUSTER_MAX_PAGES;  in z_erofs_collector_begin()
529 static bool z_erofs_collector_end(struct z_erofs_collector *clt)  in z_erofs_collector_end()  argument
531 	struct z_erofs_collection *cl = clt->cl;  in z_erofs_collector_end()
536 	z_erofs_pagevec_ctor_exit(&clt->vector, false);  in z_erofs_collector_end()
543 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)  in z_erofs_collector_end()
546 	clt->cl = NULL;  in z_erofs_collector_end()
570 	struct z_erofs_collector *const clt = &fe->clt;  in z_erofs_do_read_page()  local
591 		if (!clt->cl)  in z_erofs_do_read_page()
599 	if (z_erofs_collector_end(clt))  in z_erofs_do_read_page()
612 	err = z_erofs_collector_begin(clt, inode, map);  in z_erofs_do_read_page()
622 	preload_compressed_pages(clt, MNGD_MAPPING(sbi), cache_strategy);  in z_erofs_do_read_page()
631 	tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED &&  in z_erofs_do_read_page()
632 		  clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);  in z_erofs_do_read_page()
647 		tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);  in z_erofs_do_read_page()
650 	err = z_erofs_attach_page(clt, page, page_type);  in z_erofs_do_read_page()
657 		err = z_erofs_attach_page(clt, newpage,  in z_erofs_do_read_page()
673 	clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1);  in z_erofs_do_read_page()
1164 	z_erofs_next_pcluster_t owned_head = f->clt.owned_head;  in z_erofs_submit_queue()
1258 	if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_runqueue()
1288 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readpage()
1349 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readahead()