216 	struct z_erofs_collector clt;  member
230 	.inode = __i, .clt = COLLECTOR_INIT(), \
236 static void preload_compressed_pages(struct z_erofs_collector *clt,  in preload_compressed_pages()  argument
241 	struct z_erofs_pcluster *pcl = clt->pcl;  in preload_compressed_pages()
248 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED)  in preload_compressed_pages()
303 		clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;  in preload_compressed_pages()
363 static bool z_erofs_try_inplace_io(struct z_erofs_collector *clt,  in z_erofs_try_inplace_io()  argument
366 	struct z_erofs_pcluster *const pcl = clt->pcl;  in z_erofs_try_inplace_io()
368 	while (clt->icpage_ptr > pcl->compressed_pages)  in z_erofs_try_inplace_io()
369 		if (!cmpxchg(--clt->icpage_ptr, NULL, page))  in z_erofs_try_inplace_io()
375 static int z_erofs_attach_page(struct z_erofs_collector *clt,  in z_erofs_attach_page()  argument
382 	if (clt->mode >= COLLECT_PRIMARY &&  in z_erofs_attach_page()
384 	    z_erofs_try_inplace_io(clt, page))  in z_erofs_attach_page()
387 	ret = z_erofs_pagevec_enqueue(&clt->vector, page, type);  in z_erofs_attach_page()
388 	clt->cl->vcnt += (unsigned int)ret;  in z_erofs_attach_page()
393 static void z_erofs_try_to_claim_pcluster(struct z_erofs_collector *clt)  in z_erofs_try_to_claim_pcluster()  argument
395 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_try_to_claim_pcluster()
396 	z_erofs_next_pcluster_t *owned_head = &clt->owned_head;  in z_erofs_try_to_claim_pcluster()
403 		clt->mode = COLLECT_PRIMARY_FOLLOWED;  in z_erofs_try_to_claim_pcluster()
414 		clt->mode = COLLECT_PRIMARY_HOOKED;  in z_erofs_try_to_claim_pcluster()
415 		clt->tailpcl = NULL;  in z_erofs_try_to_claim_pcluster()
419 	clt->mode = COLLECT_PRIMARY;  in z_erofs_try_to_claim_pcluster()
422 static int z_erofs_lookup_collection(struct z_erofs_collector *clt,  in z_erofs_lookup_collection()  argument
426 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_lookup_collection()
431 	if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {  in z_erofs_lookup_collection()
462 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_lookup_collection()
463 		clt->tailpcl = pcl;  in z_erofs_lookup_collection()
465 	z_erofs_try_to_claim_pcluster(clt);  in z_erofs_lookup_collection()
466 	clt->cl = cl;  in z_erofs_lookup_collection()
470 static int z_erofs_register_collection(struct z_erofs_collector *clt,  in z_erofs_register_collection()  argument
497 	pcl->next = clt->owned_head;  in z_erofs_register_collection()
498 	clt->mode = COLLECT_PRIMARY_FOLLOWED;  in z_erofs_register_collection()
517 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_register_collection()
522 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_register_collection()
523 		clt->tailpcl = pcl;  in z_erofs_register_collection()
524 	clt->owned_head = &pcl->next;  in z_erofs_register_collection()
525 	clt->pcl = pcl;  in z_erofs_register_collection()
526 	clt->cl = cl;  in z_erofs_register_collection()
535 static int z_erofs_collector_begin(struct z_erofs_collector *clt,  in z_erofs_collector_begin()  argument
542 	DBG_BUGON(clt->cl);  in z_erofs_collector_begin()
545 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);  in z_erofs_collector_begin()
546 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);  in z_erofs_collector_begin()
555 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_collector_begin()
557 		ret = z_erofs_register_collection(clt, inode, map);  in z_erofs_collector_begin()
565 	ret = z_erofs_lookup_collection(clt, inode, map);  in z_erofs_collector_begin()
567 		erofs_workgroup_put(&clt->pcl->obj);  in z_erofs_collector_begin()
572 	z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,  in z_erofs_collector_begin()
573 				  clt->cl->pagevec, clt->cl->vcnt);  in z_erofs_collector_begin()
576 	clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages;  in z_erofs_collector_begin()
610 static bool z_erofs_collector_end(struct z_erofs_collector *clt)  in z_erofs_collector_end()  argument
612 	struct z_erofs_collection *cl = clt->cl;  in z_erofs_collector_end()
617 	z_erofs_pagevec_ctor_exit(&clt->vector, false);  in z_erofs_collector_end()
624 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)  in z_erofs_collector_end()
627 	clt->cl = NULL;  in z_erofs_collector_end()
651 	struct z_erofs_collector *const clt = &fe->clt;  in z_erofs_do_read_page()  local
672 		if (!clt->cl)  in z_erofs_do_read_page()
680 	if (z_erofs_collector_end(clt))  in z_erofs_do_read_page()
693 	err = z_erofs_collector_begin(clt, inode, map);  in z_erofs_do_read_page()
703 	preload_compressed_pages(clt, MNGD_MAPPING(sbi),  in z_erofs_do_read_page()
713 	tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED &&  in z_erofs_do_read_page()
714 		  clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);  in z_erofs_do_read_page()
729 		tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);  in z_erofs_do_read_page()
732 	err = z_erofs_attach_page(clt, page, page_type);  in z_erofs_do_read_page()
739 		err = z_erofs_attach_page(clt, newpage,  in z_erofs_do_read_page()
755 	clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1);  in z_erofs_do_read_page()
1268 	z_erofs_next_pcluster_t owned_head = f->clt.owned_head;  in z_erofs_submit_queue()
1362 	if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_runqueue()
1392 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readpage()
1454 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readahead()