Lines Matching refs:pcl
147 struct z_erofs_pcluster *pcl; in z_erofs_alloc_pcluster() local
152 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); in z_erofs_alloc_pcluster()
153 if (!pcl) in z_erofs_alloc_pcluster()
155 pcl->pclusterpages = nrpages; in z_erofs_alloc_pcluster()
156 return pcl; in z_erofs_alloc_pcluster()
161 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) in z_erofs_free_pcluster() argument
163 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_free_pcluster()
172 kmem_cache_free(pcs->slab, pcl); in z_erofs_free_pcluster()
278 struct z_erofs_pcluster *pcl, *tailpcl; member
300 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_bind_cache() local
313 for (i = 0; i < pcl->pclusterpages; ++i) { in z_erofs_bind_cache()
319 if (READ_ONCE(pcl->compressed_bvecs[i].page)) in z_erofs_bind_cache()
322 page = find_get_page(mc, pcl->obj.index + i); in z_erofs_bind_cache()
343 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, in z_erofs_bind_cache()
365 struct z_erofs_pcluster *const pcl = in erofs_try_to_free_all_cached_pages() local
369 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in erofs_try_to_free_all_cached_pages()
374 for (i = 0; i < pcl->pclusterpages; ++i) { in erofs_try_to_free_all_cached_pages()
375 struct page *page = pcl->compressed_bvecs[i].page; in erofs_try_to_free_all_cached_pages()
388 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in erofs_try_to_free_all_cached_pages()
397 struct z_erofs_pcluster *const pcl = (void *)page_private(page); in erofs_try_to_free_cached_page() local
400 if (!erofs_workgroup_try_to_freeze(&pcl->obj, 1)) in erofs_try_to_free_cached_page()
404 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in erofs_try_to_free_cached_page()
405 for (i = 0; i < pcl->pclusterpages; ++i) { in erofs_try_to_free_cached_page()
406 if (pcl->compressed_bvecs[i].page == page) { in erofs_try_to_free_cached_page()
407 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in erofs_try_to_free_cached_page()
412 erofs_workgroup_unfreeze(&pcl->obj, 1); in erofs_try_to_free_cached_page()
421 struct z_erofs_pcluster *const pcl = fe->pcl; in z_erofs_try_inplace_io() local
424 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page, in z_erofs_try_inplace_io()
426 pcl->compressed_bvecs[fe->icur] = *bvec; in z_erofs_try_inplace_io()
449 fe->pcl->vcnt += (ret >= 0); in z_erofs_attach_page()
455 struct z_erofs_pcluster *pcl = f->pcl; in z_erofs_try_to_claim_pcluster() local
459 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, in z_erofs_try_to_claim_pcluster()
461 *owned_head = &pcl->next; in z_erofs_try_to_claim_pcluster()
471 if (*owned_head != &pcl->next && pcl != f->tailpcl && in z_erofs_try_to_claim_pcluster()
472 cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, in z_erofs_try_to_claim_pcluster()
487 struct z_erofs_pcluster *pcl; in z_erofs_register_pcluster() local
497 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : in z_erofs_register_pcluster()
499 if (IS_ERR(pcl)) in z_erofs_register_pcluster()
500 return PTR_ERR(pcl); in z_erofs_register_pcluster()
502 atomic_set(&pcl->obj.refcount, 1); in z_erofs_register_pcluster()
503 pcl->algorithmformat = map->m_algorithmformat; in z_erofs_register_pcluster()
504 pcl->length = 0; in z_erofs_register_pcluster()
505 pcl->partial = true; in z_erofs_register_pcluster()
508 pcl->next = fe->owned_head; in z_erofs_register_pcluster()
509 pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_register_pcluster()
516 mutex_init(&pcl->lock); in z_erofs_register_pcluster()
517 DBG_BUGON(!mutex_trylock(&pcl->lock)); in z_erofs_register_pcluster()
520 pcl->obj.index = 0; /* which indicates ztailpacking */ in z_erofs_register_pcluster()
521 pcl->pageofs_in = erofs_blkoff(map->m_pa); in z_erofs_register_pcluster()
522 pcl->tailpacking_size = map->m_plen; in z_erofs_register_pcluster()
524 pcl->obj.index = map->m_pa >> PAGE_SHIFT; in z_erofs_register_pcluster()
526 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj); in z_erofs_register_pcluster()
532 if (grp != &pcl->obj) { in z_erofs_register_pcluster()
533 fe->pcl = container_of(grp, in z_erofs_register_pcluster()
541 fe->tailpcl = pcl; in z_erofs_register_pcluster()
542 fe->owned_head = &pcl->next; in z_erofs_register_pcluster()
543 fe->pcl = pcl; in z_erofs_register_pcluster()
547 mutex_unlock(&pcl->lock); in z_erofs_register_pcluster()
548 z_erofs_free_pcluster(pcl); in z_erofs_register_pcluster()
558 DBG_BUGON(fe->pcl); in z_erofs_collector_begin()
573 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); in z_erofs_collector_begin()
580 mutex_lock(&fe->pcl->lock); in z_erofs_collector_begin()
583 fe->tailpcl = fe->pcl; in z_erofs_collector_begin()
589 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, in z_erofs_collector_begin()
590 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); in z_erofs_collector_begin()
592 fe->icur = z_erofs_pclusterpages(fe->pcl); in z_erofs_collector_begin()
608 struct z_erofs_pcluster *const pcl = in erofs_workgroup_free_rcu() local
611 call_rcu(&pcl->rcu, z_erofs_rcu_callback); in erofs_workgroup_free_rcu()
616 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_collector_end() local
618 if (!pcl) in z_erofs_collector_end()
622 mutex_unlock(&pcl->lock); in z_erofs_collector_end()
634 erofs_workgroup_put(&pcl->obj); in z_erofs_collector_end()
636 fe->pcl = NULL; in z_erofs_collector_end()
719 if (fe->pcl) in z_erofs_do_read_page()
732 if (z_erofs_is_inline_pcluster(fe->pcl)) { in z_erofs_do_read_page()
744 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, in z_erofs_do_read_page()
817 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) in z_erofs_do_read_page()
818 fe->pcl->multibases = true; in z_erofs_do_read_page()
819 if (fe->pcl->length < offset + end - map->m_la) { in z_erofs_do_read_page()
820 fe->pcl->length = offset + end - map->m_la; in z_erofs_do_read_page()
821 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_do_read_page()
825 fe->pcl->length == map->m_llen) in z_erofs_do_read_page()
826 fe->pcl->partial = false; in z_erofs_do_read_page()
869 struct z_erofs_pcluster *pcl; member
891 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK)) { in z_erofs_do_decompressed_bvec()
894 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_do_decompressed_bvec()
911 unsigned int off0 = be->pcl->pageofs_out; in z_erofs_fill_other_copies()
921 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, in z_erofs_fill_other_copies()
954 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_out_bvecs() local
959 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); in z_erofs_parse_out_bvecs()
960 for (i = 0; i < pcl->vcnt; ++i) { in z_erofs_parse_out_bvecs()
980 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_in_bvecs() local
981 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_parse_in_bvecs()
986 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; in z_erofs_parse_in_bvecs()
996 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_parse_in_bvecs()
1023 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_decompress_pcluster() local
1024 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_decompress_pcluster()
1030 mutex_lock(&pcl->lock); in z_erofs_decompress_pcluster()
1031 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_decompress_pcluster()
1063 if (z_erofs_is_inline_pcluster(pcl)) in z_erofs_decompress_pcluster()
1064 inputsize = pcl->tailpacking_size; in z_erofs_decompress_pcluster()
1072 .pageofs_in = pcl->pageofs_in, in z_erofs_decompress_pcluster()
1073 .pageofs_out = pcl->pageofs_out, in z_erofs_decompress_pcluster()
1075 .outputsize = pcl->length, in z_erofs_decompress_pcluster()
1076 .alg = pcl->algorithmformat, in z_erofs_decompress_pcluster()
1078 .partial_decoding = pcl->partial, in z_erofs_decompress_pcluster()
1079 .fillgaps = pcl->multibases, in z_erofs_decompress_pcluster()
1084 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_decompress_pcluster()
1085 page = pcl->compressed_bvecs[0].page; in z_erofs_decompress_pcluster()
1086 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); in z_erofs_decompress_pcluster()
1090 page = pcl->compressed_bvecs[i].page; in z_erofs_decompress_pcluster()
1097 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in z_erofs_decompress_pcluster()
1123 pcl->length = 0; in z_erofs_decompress_pcluster()
1124 pcl->partial = true; in z_erofs_decompress_pcluster()
1125 pcl->multibases = false; in z_erofs_decompress_pcluster()
1126 pcl->bvset.nextpage = NULL; in z_erofs_decompress_pcluster()
1127 pcl->vcnt = 0; in z_erofs_decompress_pcluster()
1130 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); in z_erofs_decompress_pcluster()
1131 mutex_unlock(&pcl->lock); in z_erofs_decompress_pcluster()
1152 be.pcl = container_of(owned, struct z_erofs_pcluster, next); in z_erofs_decompress_queue()
1153 owned = READ_ONCE(be.pcl->next); in z_erofs_decompress_queue()
1156 erofs_workgroup_put(&be.pcl->obj); in z_erofs_decompress_queue()
1198 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, in pickup_page_for_submission() argument
1203 const pgoff_t index = pcl->obj.index; in pickup_page_for_submission()
1214 page = READ_ONCE(pcl->compressed_bvecs[nr].page); in pickup_page_for_submission()
1230 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1256 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1267 set_page_private(page, (unsigned long)pcl); in pickup_page_for_submission()
1291 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page, in pickup_page_for_submission()
1303 attach_page_private(page, pcl); in pickup_page_for_submission()
1357 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, in move_to_bypass_jobqueue() argument
1368 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); in move_to_bypass_jobqueue()
1371 WRITE_ONCE(*bypass_qtail, &pcl->next); in move_to_bypass_jobqueue()
1373 qtail[JQ_BYPASS] = &pcl->next; in move_to_bypass_jobqueue()
1430 struct z_erofs_pcluster *pcl; in z_erofs_submit_queue() local
1439 pcl = container_of(owned_head, struct z_erofs_pcluster, next); in z_erofs_submit_queue()
1442 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, in z_erofs_submit_queue()
1444 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_submit_queue()
1445 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()
1451 .m_pa = blknr_to_addr(pcl->obj.index), in z_erofs_submit_queue()
1456 end = cur + pcl->pclusterpages; in z_erofs_submit_queue()
1461 page = pickup_page_for_submission(pcl, i++, pagepool, in z_erofs_submit_queue()
1504 qtail[JQ_SUBMIT] = &pcl->next; in z_erofs_submit_queue()
1506 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()