25 	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,
27 	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);
31 static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,  in z_erofs_lz4_prepare_destpages()  argument
35 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_prepare_destpages()
44 		struct page *const page = rq->out[i];  in z_erofs_lz4_prepare_destpages()
54 			availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];  in z_erofs_lz4_prepare_destpages()
81 		rq->out[i] = victim;  in z_erofs_lz4_prepare_destpages()
86 static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq,  in generic_copy_inplace_data()  argument
93 	struct page **in = rq->in;  in generic_copy_inplace_data()
96 	unsigned int inlen = rq->inputsize - pageofs_in;  in generic_copy_inplace_data()
113 static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)  in z_erofs_lz4_decompress()  argument
120 	if (rq->inputsize > PAGE_SIZE)  in z_erofs_lz4_decompress()
123 	src = kmap_atomic(*rq->in);  in z_erofs_lz4_decompress()
128 	if (EROFS_SB(rq->sb)->feature_incompat &  in z_erofs_lz4_decompress()
136 		if (inputmargin >= rq->inputsize) {  in z_erofs_lz4_decompress()
143 	inlen = rq->inputsize - inputmargin;  in z_erofs_lz4_decompress()
144 	if (rq->inplace_io) {  in z_erofs_lz4_decompress()
145 		const uint oend = (rq->pageofs_out +  in z_erofs_lz4_decompress()
146 				   rq->outputsize) & ~PAGE_MASK;  in z_erofs_lz4_decompress()
147 		const uint nr = PAGE_ALIGN(rq->pageofs_out +  in z_erofs_lz4_decompress()
148 					   rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_decompress()
150 		if (rq->partial_decoding || !support_0padding ||  in z_erofs_lz4_decompress()
151 		    rq->out[nr - 1] != rq->in[0] ||  in z_erofs_lz4_decompress()
152 		    rq->inputsize - oend <  in z_erofs_lz4_decompress()
154 			src = generic_copy_inplace_data(rq, src, inputmargin);  in z_erofs_lz4_decompress()
161 					  inlen, rq->outputsize,  in z_erofs_lz4_decompress()
162 					  rq->outputsize);  in z_erofs_lz4_decompress()
164 		erofs_err(rq->sb, "failed to decompress, in[%u, %u] out[%u]",  in z_erofs_lz4_decompress()
165 			  inlen, inputmargin, rq->outputsize);  in z_erofs_lz4_decompress()
170 			       16, 1, out, rq->outputsize, true);  in z_erofs_lz4_decompress()
219 static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,  in z_erofs_decompress_generic()  argument
223 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_decompress_generic()
224 	const struct z_erofs_decompressor *alg = decompressors + rq->alg;  in z_erofs_decompress_generic()
229 	if (nrpages_out == 1 && !rq->inplace_io) {  in z_erofs_decompress_generic()
230 		DBG_BUGON(!*rq->out);  in z_erofs_decompress_generic()
231 		dst = kmap_atomic(*rq->out);  in z_erofs_decompress_generic()
241 	if (rq->outputsize <= PAGE_SIZE * 7 / 8) {  in z_erofs_decompress_generic()
246 		rq->inplace_io = false;  in z_erofs_decompress_generic()
247 		ret = alg->decompress(rq, dst);  in z_erofs_decompress_generic()
249 			copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,  in z_erofs_decompress_generic()
250 					  rq->outputsize);  in z_erofs_decompress_generic()
256 	ret = alg->prepare_destpages(rq, pagepool);  in z_erofs_decompress_generic()
260 		dst = page_address(*rq->out);  in z_erofs_decompress_generic()
267 		dst = vm_map_ram(rq->out, nrpages_out, -1, PAGE_KERNEL);  in z_erofs_decompress_generic()
281 	ret = alg->decompress(rq, dst + rq->pageofs_out);  in z_erofs_decompress_generic()
290 static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,  in z_erofs_shifted_transform()  argument
294 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_shifted_transform()
295 	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;  in z_erofs_shifted_transform()
303 	if (rq->out[0] == *rq->in) {  in z_erofs_shifted_transform()
308 	src = kmap_atomic(*rq->in);  in z_erofs_shifted_transform()
309 	if (!rq->out[0]) {  in z_erofs_shifted_transform()
312 		dst = kmap_atomic(rq->out[0]);  in z_erofs_shifted_transform()
313 		memcpy(dst + rq->pageofs_out, src, righthalf);  in z_erofs_shifted_transform()
316 	if (rq->out[1] == *rq->in) {  in z_erofs_shifted_transform()
317 		memmove(src, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
321 		DBG_BUGON(!rq->out[1]);  in z_erofs_shifted_transform()
322 		dst = kmap_atomic(rq->out[1]);  in z_erofs_shifted_transform()
323 		memcpy(dst, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
331 int z_erofs_decompress(struct z_erofs_decompress_req *rq,  in z_erofs_decompress()  argument
334 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)  in z_erofs_decompress()
335 		return z_erofs_shifted_transform(rq, pagepool);  in z_erofs_decompress()
336 	return z_erofs_decompress_generic(rq, pagepool);  in z_erofs_decompress()