20 	struct z_erofs_decompress_req *rq;  member
68 	struct z_erofs_decompress_req *rq = ctx->rq;  in z_erofs_lz4_prepare_dstpages()  local
73 				EROFS_SB(rq->sb)->lz4.max_distance_pages;  in z_erofs_lz4_prepare_dstpages()
79 		struct page *const page = rq->out[i];  in z_erofs_lz4_prepare_dstpages()
86 		if (!rq->fillgaps && test_bit(j, bounced)) {  in z_erofs_lz4_prepare_dstpages()
89 			availables[top++] = rq->out[i - lz4_max_distance_pages];  in z_erofs_lz4_prepare_dstpages()
119 		rq->out[i] = victim;  in z_erofs_lz4_prepare_dstpages()
128 	struct z_erofs_decompress_req *rq = ctx->rq;  in z_erofs_lz4_handle_overlap()  local
133 	if (rq->inplace_io) {  in z_erofs_lz4_handle_overlap()
135 		if (rq->partial_decoding || !may_inplace ||  in z_erofs_lz4_handle_overlap()
136 		    omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))  in z_erofs_lz4_handle_overlap()
140 			DBG_BUGON(rq->in[i] == NULL);  in z_erofs_lz4_handle_overlap()
142 				if (rq->out[j] == rq->in[i])  in z_erofs_lz4_handle_overlap()
153 	src = erofs_vm_map_ram(rq->in, ctx->inpages);  in z_erofs_lz4_handle_overlap()
161 	in = rq->in;  in z_erofs_lz4_handle_overlap()
170 	total = rq->inputsize;  in z_erofs_lz4_handle_overlap()
194 int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,  in z_erofs_fixup_insize()  argument
202 	rq->inputsize -= padend - padbuf;  in z_erofs_fixup_insize()
203 	rq->pageofs_in += padend - padbuf;  in z_erofs_fixup_insize()
210 	struct z_erofs_decompress_req *rq = ctx->rq;  in z_erofs_lz4_decompress_mem()  local
216 	DBG_BUGON(*rq->in == NULL);  in z_erofs_lz4_decompress_mem()
217 	headpage = kmap_local_page(*rq->in);  in z_erofs_lz4_decompress_mem()
220 	if (erofs_sb_has_zero_padding(EROFS_SB(rq->sb))) {  in z_erofs_lz4_decompress_mem()
222 		ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,  in z_erofs_lz4_decompress_mem()
223 				min_t(unsigned int, rq->inputsize,  in z_erofs_lz4_decompress_mem()
224 				      rq->sb->s_blocksize - rq->pageofs_in));  in z_erofs_lz4_decompress_mem()
229 		may_inplace = !((rq->pageofs_in + rq->inputsize) &  in z_erofs_lz4_decompress_mem()
230 				(rq->sb->s_blocksize - 1));  in z_erofs_lz4_decompress_mem()
233 	inputmargin = rq->pageofs_in;  in z_erofs_lz4_decompress_mem()
240 	if (rq->partial_decoding || !support_0padding)  in z_erofs_lz4_decompress_mem()
242 				rq->inputsize, rq->outputsize, rq->outputsize);  in z_erofs_lz4_decompress_mem()
245 					  rq->inputsize, rq->outputsize);  in z_erofs_lz4_decompress_mem()
247 	if (ret != rq->outputsize) {  in z_erofs_lz4_decompress_mem()
248 		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",  in z_erofs_lz4_decompress_mem()
249 			  ret, rq->inputsize, inputmargin, rq->outputsize);  in z_erofs_lz4_decompress_mem()
252 			       16, 1, src + inputmargin, rq->inputsize, true);  in z_erofs_lz4_decompress_mem()
254 			       16, 1, out, rq->outputsize, true);  in z_erofs_lz4_decompress_mem()
257 			memset(out + ret, 0, rq->outputsize - ret);  in z_erofs_lz4_decompress_mem()
276 static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,  in z_erofs_lz4_decompress()  argument
284 	ctx.rq = rq;  in z_erofs_lz4_decompress()
285 	ctx.oend = rq->pageofs_out + rq->outputsize;  in z_erofs_lz4_decompress()
287 	ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;  in z_erofs_lz4_decompress()
290 	if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) {  in z_erofs_lz4_decompress()
291 		DBG_BUGON(!*rq->out);  in z_erofs_lz4_decompress()
292 		dst = kmap_local_page(*rq->out);  in z_erofs_lz4_decompress()
302 		dst = page_address(*rq->out);  in z_erofs_lz4_decompress()
305 		dst = erofs_vm_map_ram(rq->out, ctx.outpages);  in z_erofs_lz4_decompress()
312 	ret = z_erofs_lz4_decompress_mem(&ctx, dst + rq->pageofs_out);  in z_erofs_lz4_decompress()
320 static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,  in z_erofs_transform_plain()  argument
323 	const unsigned int inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;  in z_erofs_transform_plain()
325 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_transform_plain()
326 	const unsigned int righthalf = min_t(unsigned int, rq->outputsize,  in z_erofs_transform_plain()
327 					     PAGE_SIZE - rq->pageofs_out);  in z_erofs_transform_plain()
328 	const unsigned int lefthalf = rq->outputsize - righthalf;  in z_erofs_transform_plain()
330 		rq->alg == Z_EROFS_COMPRESSION_SHIFTED ? 0 : rq->pageofs_out;  in z_erofs_transform_plain()
333 	if (outpages > 2 && rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {  in z_erofs_transform_plain()
338 	if (rq->out[0] == *rq->in) {  in z_erofs_transform_plain()
339 		DBG_BUGON(rq->pageofs_out);  in z_erofs_transform_plain()
343 	src = kmap_local_page(rq->in[inpages - 1]) + rq->pageofs_in;  in z_erofs_transform_plain()
344 	if (rq->out[0])  in z_erofs_transform_plain()
345 		memcpy_to_page(rq->out[0], rq->pageofs_out,  in z_erofs_transform_plain()
349 		DBG_BUGON(!rq->out[outpages - 1]);  in z_erofs_transform_plain()
350 		if (rq->out[outpages - 1] != rq->in[inpages - 1]) {  in z_erofs_transform_plain()
351 			memcpy_to_page(rq->out[outpages - 1], 0, src +  in z_erofs_transform_plain()
356 			flush_dcache_page(rq->in[inpages - 1]);  in z_erofs_transform_plain()