24 	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,
26 	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);
66 static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,  in z_erofs_lz4_prepare_destpages()  argument
70 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_prepare_destpages()
75 				EROFS_SB(rq->sb)->lz4.max_distance_pages;  in z_erofs_lz4_prepare_destpages()
81 		struct page *const page = rq->out[i];  in z_erofs_lz4_prepare_destpages()
91 			availables[top++] = rq->out[i - lz4_max_distance_pages];  in z_erofs_lz4_prepare_destpages()
117 		rq->out[i] = victim;  in z_erofs_lz4_prepare_destpages()
122 static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq,  in z_erofs_handle_inplace_io()  argument
131 	inputsize = rq->inputsize;  in z_erofs_handle_inplace_io()
133 	oend = rq->pageofs_out + rq->outputsize;  in z_erofs_handle_inplace_io()
137 	if (rq->inplace_io) {  in z_erofs_handle_inplace_io()
138 		if (rq->partial_decoding || !support_0padding ||  in z_erofs_handle_inplace_io()
143 			DBG_BUGON(rq->in[i] == NULL);  in z_erofs_handle_inplace_io()
145 				if (rq->out[j] == rq->in[i])  in z_erofs_handle_inplace_io()
156 	src = erofs_vm_map_ram(rq->in, nrpages_in);  in z_erofs_handle_inplace_io()
164 	in = rq->in;  in z_erofs_handle_inplace_io()
173 	total = rq->inputsize;  in z_erofs_handle_inplace_io()
192 static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)  in z_erofs_lz4_decompress()  argument
199 	DBG_BUGON(*rq->in == NULL);  in z_erofs_lz4_decompress()
200 	headpage = kmap_atomic(*rq->in);  in z_erofs_lz4_decompress()
205 	if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) {  in z_erofs_lz4_decompress()
212 		if (inputmargin >= rq->inputsize) {  in z_erofs_lz4_decompress()
218 	rq->inputsize -= inputmargin;  in z_erofs_lz4_decompress()
219 	src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype,  in z_erofs_lz4_decompress()
225 	if (rq->partial_decoding || !support_0padding)  in z_erofs_lz4_decompress()
227 				rq->inputsize, rq->outputsize, rq->outputsize);  in z_erofs_lz4_decompress()
230 					  rq->inputsize, rq->outputsize);  in z_erofs_lz4_decompress()
232 	if (ret != rq->outputsize) {  in z_erofs_lz4_decompress()
233 		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",  in z_erofs_lz4_decompress()
234 			  ret, rq->inputsize, inputmargin, rq->outputsize);  in z_erofs_lz4_decompress()
238 			       16, 1, src + inputmargin, rq->inputsize, true);  in z_erofs_lz4_decompress()
240 			       16, 1, out, rq->outputsize, true);  in z_erofs_lz4_decompress()
243 			memset(out + ret, 0, rq->outputsize - ret);  in z_erofs_lz4_decompress()
250 		vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT);  in z_erofs_lz4_decompress()
298 static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,  in z_erofs_decompress_generic()  argument
302 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_decompress_generic()
303 	const struct z_erofs_decompressor *alg = decompressors + rq->alg;  in z_erofs_decompress_generic()
309 	if (rq->inputsize <= PAGE_SIZE) {  in z_erofs_decompress_generic()
310 		if (nrpages_out == 1 && !rq->inplace_io) {  in z_erofs_decompress_generic()
311 			DBG_BUGON(!*rq->out);  in z_erofs_decompress_generic()
312 			dst = kmap_atomic(*rq->out);  in z_erofs_decompress_generic()
322 		if (rq->outputsize <= PAGE_SIZE * 7 / 8) {  in z_erofs_decompress_generic()
327 			rq->inplace_io = false;  in z_erofs_decompress_generic()
328 			ret = alg->decompress(rq, dst);  in z_erofs_decompress_generic()
330 				copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,  in z_erofs_decompress_generic()
331 						  rq->outputsize);  in z_erofs_decompress_generic()
339 	ret = alg->prepare_destpages(rq, pagepool);  in z_erofs_decompress_generic()
343 		dst = page_address(*rq->out);  in z_erofs_decompress_generic()
348 	dst = erofs_vm_map_ram(rq->out, nrpages_out);  in z_erofs_decompress_generic()
354 	ret = alg->decompress(rq, dst + rq->pageofs_out);  in z_erofs_decompress_generic()
363 static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,  in z_erofs_shifted_transform()  argument
367 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_shifted_transform()
368 	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;  in z_erofs_shifted_transform()
376 	if (rq->out[0] == *rq->in) {  in z_erofs_shifted_transform()
381 	src = kmap_atomic(*rq->in);  in z_erofs_shifted_transform()
382 	if (rq->out[0]) {  in z_erofs_shifted_transform()
383 		dst = kmap_atomic(rq->out[0]);  in z_erofs_shifted_transform()
384 		memcpy(dst + rq->pageofs_out, src, righthalf);  in z_erofs_shifted_transform()
389 		DBG_BUGON(!rq->out[1]);  in z_erofs_shifted_transform()
390 		if (rq->out[1] == *rq->in) {  in z_erofs_shifted_transform()
391 			memmove(src, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
393 			dst = kmap_atomic(rq->out[1]);  in z_erofs_shifted_transform()
394 			memcpy(dst, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
402 int z_erofs_decompress(struct z_erofs_decompress_req *rq,  in z_erofs_decompress()  argument
405 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)  in z_erofs_decompress()
406 		return z_erofs_shifted_transform(rq, pagepool);  in z_erofs_decompress()
407 	return z_erofs_decompress_generic(rq, pagepool);  in z_erofs_decompress()