48 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_rs8()  argument
57 	return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,  in fec_decode_rs8()
58 			  fio->erasures, 0, NULL);  in fec_decode_rs8()
109 				      struct dm_verity_fec_io *fio,  in fec_buffer_rs_block()  argument
112 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
128 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_bufs()  argument
145 	fec_for_each_buffer_rs_block(fio, n, i) {  in fec_decode_bufs()
146 		block = fec_buffer_rs_block(v, fio, n, i);  in fec_decode_bufs()
147 		res = fec_decode_rs8(v, fio, block, &par[offset], neras);  in fec_decode_bufs()
154 		fio->output[block_offset] = block[byte_index];  in fec_decode_bufs()
212 	struct dm_verity_fec_io *fio = fec_io(io);  in fec_read_bufs()  local
261 				fio->erasures[(*neras)++] = i;  in fec_read_bufs()
280 				fio->erasures[(*neras)++] = i;  in fec_read_bufs()
287 		fec_for_each_buffer_rs_block(fio, n, j) {  in fec_read_bufs()
293 			rs_block = fec_buffer_rs_block(v, fio, n, j);  in fec_read_bufs()
307 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_alloc_bufs()  argument
311 	if (!fio->rs)  in fec_alloc_bufs()
312 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
315 		if (fio->bufs[n])  in fec_alloc_bufs()
318 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
319 		if (unlikely(!fio->bufs[n])) {  in fec_alloc_bufs()
326 	fec_for_each_extra_buffer(fio, n) {  in fec_alloc_bufs()
327 		if (fio->bufs[n])  in fec_alloc_bufs()
330 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
332 		if (unlikely(!fio->bufs[n]))  in fec_alloc_bufs()
335 	fio->nbufs = n;  in fec_alloc_bufs()
337 	if (!fio->output)  in fec_alloc_bufs()
338 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
347 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_init_bufs()  argument
351 	fec_for_each_buffer(fio, n)  in fec_init_bufs()
352 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
354 	memset(fio->erasures, 0, sizeof(fio->erasures));  in fec_init_bufs()
363 			  struct dm_verity_fec_io *fio, u64 rsb, u64 offset,  in fec_decode_rsb()  argument
369 	r = fec_alloc_bufs(v, fio);  in fec_decode_rsb()
374 		fec_init_bufs(v, fio);  in fec_decode_rsb()
381 		r = fec_decode_bufs(v, fio, rsb, r, pos, neras);  in fec_decode_rsb()
385 		pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;  in fec_decode_rsb()
389 	r = verity_hash(v, verity_io_hash_req(v, io), fio->output,  in fec_decode_rsb()
408 	struct dm_verity_fec_io *fio = fec_io(io);  in fec_bv_copy()  local
410 	memcpy(data, &fio->output[fio->output_pos], len);  in fec_bv_copy()
411 	fio->output_pos += len;  in fec_bv_copy()
425 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_decode()  local
431 	if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) {  in verity_fec_decode()
436 	fio->level++;  in verity_fec_decode()
465 	r = fec_decode_rsb(v, io, fio, rsb, offset, false);  in verity_fec_decode()
467 		r = fec_decode_rsb(v, io, fio, rsb, offset, true);  in verity_fec_decode()
473 		memcpy(dest, fio->output, 1 << v->data_dev_block_bits);  in verity_fec_decode()
475 		fio->output_pos = 0;  in verity_fec_decode()
480 	fio->level--;  in verity_fec_decode()
491 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_finish_io()  local
496 	mempool_free(fio->rs, &f->rs_pool);  in verity_fec_finish_io()
499 		mempool_free(fio->bufs[n], &f->prealloc_pool);  in verity_fec_finish_io()
501 	fec_for_each_extra_buffer(fio, n)  in verity_fec_finish_io()
502 		mempool_free(fio->bufs[n], &f->extra_pool);  in verity_fec_finish_io()
504 	mempool_free(fio->output, &f->output_pool);  in verity_fec_finish_io()
512 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_init_io()  local
517 	fio->rs = NULL;  in verity_fec_init_io()
518 	memset(fio->bufs, 0, sizeof(fio->bufs));  in verity_fec_init_io()
519 	fio->nbufs = 0;  in verity_fec_init_io()
520 	fio->output = NULL;  in verity_fec_init_io()
521 	fio->level = 0;  in verity_fec_init_io()