55 	int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
56 	void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
57 	int (*decompress_pages)(struct decompress_io_ctx *dic);
207 static int lzo_decompress_pages(struct decompress_io_ctx *dic)  in lzo_decompress_pages()  argument
211 	ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,  in lzo_decompress_pages()
212 						dic->rbuf, &dic->rlen);  in lzo_decompress_pages()
215 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);  in lzo_decompress_pages()
219 	if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {  in lzo_decompress_pages()
222 					F2FS_I_SB(dic->inode)->sb->s_id,  in lzo_decompress_pages()
223 					dic->rlen,  in lzo_decompress_pages()
224 					PAGE_SIZE << dic->log_cluster_size);  in lzo_decompress_pages()
304 static int lz4_decompress_pages(struct decompress_io_ctx *dic)  in lz4_decompress_pages()  argument
308 	ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,  in lz4_decompress_pages()
309 						dic->clen, dic->rlen);  in lz4_decompress_pages()
312 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);  in lz4_decompress_pages()
316 	if (ret != PAGE_SIZE << dic->log_cluster_size) {  in lz4_decompress_pages()
319 					F2FS_I_SB(dic->inode)->sb->s_id, ret,  in lz4_decompress_pages()
320 					PAGE_SIZE << dic->log_cluster_size);  in lz4_decompress_pages()
424 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)  in zstd_init_decompress_ctx()  argument
430 			MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size);  in zstd_init_decompress_ctx()
434 	workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),  in zstd_init_decompress_ctx()
442 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_init_decompress_ctx()
448 	dic->private = workspace;  in zstd_init_decompress_ctx()
449 	dic->private2 = stream;  in zstd_init_decompress_ctx()
454 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)  in zstd_destroy_decompress_ctx()  argument
456 	kvfree(dic->private);  in zstd_destroy_decompress_ctx()
457 	dic->private = NULL;  in zstd_destroy_decompress_ctx()
458 	dic->private2 = NULL;  in zstd_destroy_decompress_ctx()
461 static int zstd_decompress_pages(struct decompress_io_ctx *dic)  in zstd_decompress_pages()  argument
463 	zstd_dstream *stream = dic->private2;  in zstd_decompress_pages()
469 	inbuf.src = dic->cbuf->cdata;  in zstd_decompress_pages()
470 	inbuf.size = dic->clen;  in zstd_decompress_pages()
473 	outbuf.dst = dic->rbuf;  in zstd_decompress_pages()
474 	outbuf.size = dic->rlen;  in zstd_decompress_pages()
479 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_decompress_pages()
484 	if (dic->rlen != outbuf.pos) {  in zstd_decompress_pages()
487 				F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_decompress_pages()
488 				__func__, dic->rlen,  in zstd_decompress_pages()
489 				PAGE_SIZE << dic->log_cluster_size);  in zstd_decompress_pages()
732 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,
734 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
737 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)  in f2fs_decompress_cluster()  argument
739 	struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);  in f2fs_decompress_cluster()
740 	struct f2fs_inode_info *fi = F2FS_I(dic->inode);  in f2fs_decompress_cluster()
746 	trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,  in f2fs_decompress_cluster()
747 				dic->cluster_size, fi->i_compress_algorithm);  in f2fs_decompress_cluster()
749 	if (dic->failed) {  in f2fs_decompress_cluster()
754 	ret = f2fs_prepare_decomp_mem(dic, false);  in f2fs_decompress_cluster()
760 	dic->clen = le32_to_cpu(dic->cbuf->clen);  in f2fs_decompress_cluster()
761 	dic->rlen = PAGE_SIZE << dic->log_cluster_size;  in f2fs_decompress_cluster()
763 	if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {  in f2fs_decompress_cluster()
769 	ret = cops->decompress_pages(dic);  in f2fs_decompress_cluster()
772 		u32 provided = le32_to_cpu(dic->cbuf->chksum);  in f2fs_decompress_cluster()
773 		u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen);  in f2fs_decompress_cluster()
776 			if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) {  in f2fs_decompress_cluster()
777 				set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT);  in f2fs_decompress_cluster()
780 					KERN_INFO, sbi->sb->s_id, dic->inode->i_ino,  in f2fs_decompress_cluster()
788 	f2fs_release_decomp_mem(dic, bypass_callback, false);  in f2fs_decompress_cluster()
791 	trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,  in f2fs_decompress_cluster()
792 							dic->clen, ret);  in f2fs_decompress_cluster()
793 	f2fs_decompress_end_io(dic, ret, in_task);  in f2fs_decompress_cluster()
805 	struct decompress_io_ctx *dic =  in f2fs_end_read_compressed_page()  local
807 	struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);  in f2fs_end_read_compressed_page()
812 		WRITE_ONCE(dic->failed, true);  in f2fs_end_read_compressed_page()
815 					dic->inode->i_ino, blkaddr);  in f2fs_end_read_compressed_page()
817 	if (atomic_dec_and_test(&dic->remaining_pages))  in f2fs_end_read_compressed_page()
818 		f2fs_decompress_cluster(dic, in_task);  in f2fs_end_read_compressed_page()
1538 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,  in f2fs_prepare_decomp_mem()  argument
1542 		f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];  in f2fs_prepare_decomp_mem()
1545 	if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))  in f2fs_prepare_decomp_mem()
1548 	dic->tpages = page_array_alloc(dic->inode, dic->cluster_size);  in f2fs_prepare_decomp_mem()
1549 	if (!dic->tpages)  in f2fs_prepare_decomp_mem()
1552 	for (i = 0; i < dic->cluster_size; i++) {  in f2fs_prepare_decomp_mem()
1553 		if (dic->rpages[i]) {  in f2fs_prepare_decomp_mem()
1554 			dic->tpages[i] = dic->rpages[i];  in f2fs_prepare_decomp_mem()
1558 		dic->tpages[i] = f2fs_compress_alloc_page();  in f2fs_prepare_decomp_mem()
1559 		if (!dic->tpages[i])  in f2fs_prepare_decomp_mem()
1563 	dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size);  in f2fs_prepare_decomp_mem()
1564 	if (!dic->rbuf)  in f2fs_prepare_decomp_mem()
1567 	dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages);  in f2fs_prepare_decomp_mem()
1568 	if (!dic->cbuf)  in f2fs_prepare_decomp_mem()
1572 		return cops->init_decompress_ctx(dic);  in f2fs_prepare_decomp_mem()
1577 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,  in f2fs_release_decomp_mem()  argument
1581 		f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];  in f2fs_release_decomp_mem()
1583 	if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))  in f2fs_release_decomp_mem()
1587 		cops->destroy_decompress_ctx(dic);  in f2fs_release_decomp_mem()
1589 	if (dic->cbuf)  in f2fs_release_decomp_mem()
1590 		vm_unmap_ram(dic->cbuf, dic->nr_cpages);  in f2fs_release_decomp_mem()
1592 	if (dic->rbuf)  in f2fs_release_decomp_mem()
1593 		vm_unmap_ram(dic->rbuf, dic->cluster_size);  in f2fs_release_decomp_mem()
1596 static void f2fs_free_dic(struct decompress_io_ctx *dic,
1601 	struct decompress_io_ctx *dic;  in f2fs_alloc_dic()  local
1606 	dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi);  in f2fs_alloc_dic()
1607 	if (!dic)  in f2fs_alloc_dic()
1610 	dic->rpages = page_array_alloc(cc->inode, cc->cluster_size);  in f2fs_alloc_dic()
1611 	if (!dic->rpages) {  in f2fs_alloc_dic()
1612 		kmem_cache_free(dic_entry_slab, dic);  in f2fs_alloc_dic()
1616 	dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;  in f2fs_alloc_dic()
1617 	dic->inode = cc->inode;  in f2fs_alloc_dic()
1618 	atomic_set(&dic->remaining_pages, cc->nr_cpages);  in f2fs_alloc_dic()
1619 	dic->cluster_idx = cc->cluster_idx;  in f2fs_alloc_dic()
1620 	dic->cluster_size = cc->cluster_size;  in f2fs_alloc_dic()
1621 	dic->log_cluster_size = cc->log_cluster_size;  in f2fs_alloc_dic()
1622 	dic->nr_cpages = cc->nr_cpages;  in f2fs_alloc_dic()
1623 	refcount_set(&dic->refcnt, 1);  in f2fs_alloc_dic()
1624 	dic->failed = false;  in f2fs_alloc_dic()
1625 	dic->need_verity = f2fs_need_verity(cc->inode, start_idx);  in f2fs_alloc_dic()
1627 	for (i = 0; i < dic->cluster_size; i++)  in f2fs_alloc_dic()
1628 		dic->rpages[i] = cc->rpages[i];  in f2fs_alloc_dic()
1629 	dic->nr_rpages = cc->cluster_size;  in f2fs_alloc_dic()
1631 	dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages);  in f2fs_alloc_dic()
1632 	if (!dic->cpages) {  in f2fs_alloc_dic()
1637 	for (i = 0; i < dic->nr_cpages; i++) {  in f2fs_alloc_dic()
1647 					start_idx + i + 1, dic);  in f2fs_alloc_dic()
1648 		dic->cpages[i] = page;  in f2fs_alloc_dic()
1651 	ret = f2fs_prepare_decomp_mem(dic, true);  in f2fs_alloc_dic()
1655 	return dic;  in f2fs_alloc_dic()
1658 	f2fs_free_dic(dic, true);  in f2fs_alloc_dic()
1662 static void f2fs_free_dic(struct decompress_io_ctx *dic,  in f2fs_free_dic()  argument
1667 	f2fs_release_decomp_mem(dic, bypass_destroy_callback, true);  in f2fs_free_dic()
1669 	if (dic->tpages) {  in f2fs_free_dic()
1670 		for (i = 0; i < dic->cluster_size; i++) {  in f2fs_free_dic()
1671 			if (dic->rpages[i])  in f2fs_free_dic()
1673 			if (!dic->tpages[i])  in f2fs_free_dic()
1675 			f2fs_compress_free_page(dic->tpages[i]);  in f2fs_free_dic()
1677 		page_array_free(dic->inode, dic->tpages, dic->cluster_size);  in f2fs_free_dic()
1680 	if (dic->cpages) {  in f2fs_free_dic()
1681 		for (i = 0; i < dic->nr_cpages; i++) {  in f2fs_free_dic()
1682 			if (!dic->cpages[i])  in f2fs_free_dic()
1684 			f2fs_compress_free_page(dic->cpages[i]);  in f2fs_free_dic()
1686 		page_array_free(dic->inode, dic->cpages, dic->nr_cpages);  in f2fs_free_dic()
1689 	page_array_free(dic->inode, dic->rpages, dic->nr_rpages);  in f2fs_free_dic()
1690 	kmem_cache_free(dic_entry_slab, dic);  in f2fs_free_dic()
1695 	struct decompress_io_ctx *dic =  in f2fs_late_free_dic()  local
1698 	f2fs_free_dic(dic, false);  in f2fs_late_free_dic()
1701 static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task)  in f2fs_put_dic()  argument
1703 	if (refcount_dec_and_test(&dic->refcnt)) {  in f2fs_put_dic()
1705 			f2fs_free_dic(dic, false);  in f2fs_put_dic()
1707 			INIT_WORK(&dic->free_work, f2fs_late_free_dic);  in f2fs_put_dic()
1708 			queue_work(F2FS_I_SB(dic->inode)->post_read_wq,  in f2fs_put_dic()
1709 					&dic->free_work);  in f2fs_put_dic()
1718 static void __f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,  in __f2fs_decompress_end_io()  argument
1723 	for (i = 0; i < dic->cluster_size; i++) {  in __f2fs_decompress_end_io()
1724 		struct page *rpage = dic->rpages[i];  in __f2fs_decompress_end_io()
1740 	f2fs_put_dic(dic, in_task);  in __f2fs_decompress_end_io()
1745 	struct decompress_io_ctx *dic =  in f2fs_verify_cluster()  local
1750 	for (i = 0; i < dic->cluster_size; i++) {  in f2fs_verify_cluster()
1751 		struct page *rpage = dic->rpages[i];  in f2fs_verify_cluster()
1757 	__f2fs_decompress_end_io(dic, false, true);  in f2fs_verify_cluster()
1764 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,  in f2fs_decompress_end_io()  argument
1767 	if (!failed && dic->need_verity) {  in f2fs_decompress_end_io()
1774 		INIT_WORK(&dic->verity_work, f2fs_verify_cluster);  in f2fs_decompress_end_io()
1775 		fsverity_enqueue_verify_work(&dic->verity_work);  in f2fs_decompress_end_io()
1777 		__f2fs_decompress_end_io(dic, failed, in_task);  in f2fs_decompress_end_io()
1788 	struct decompress_io_ctx *dic =  in f2fs_put_page_dic()  local
1791 	f2fs_put_dic(dic, in_task);  in f2fs_put_page_dic()