8 #include "dm-verity-fec.h"
11 #define DM_MSG_PREFIX	"verity-fec"
18 	return v->fec && v->fec->dev;  in verity_fec_is_enabled()
37 	mod = do_div(offset, v->fec->rsn);  in fec_interleave()
38 	return offset + mod * (v->fec->rounds << v->data_dev_block_bits);  in fec_interleave()
45 			  u8 *data, u8 *fec, int neras)  in fec_decode_rs8()  argument
50 	for (i = 0; i < v->fec->roots; i++)  in fec_decode_rs8()
51 		par[i] = fec[i];  in fec_decode_rs8()
53 	return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,  in fec_decode_rs8()
67 	position = (index + rsb) * v->fec->roots;  in fec_read_parity()
68 	block = div64_u64_rem(position, v->fec->io_size, &rem);  in fec_read_parity()
71 	res = dm_bufio_read(v->fec->bufio, block, buf);  in fec_read_parity()
73 		DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",  in fec_read_parity()
107 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
138 	 * one corrected target byte and consumes fec->roots parity bytes.  in fec_decode_bufs()
156 		offset += v->fec->roots;  in fec_decode_bufs()
157 		if (offset >= v->fec->io_size) {  in fec_decode_bufs()
171 		DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",  in fec_decode_bufs()
174 		DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",  in fec_decode_bufs()
223 	for (i = 0; i < v->fec->rsn; i++) {  in fec_read_bufs()
224 		ileaved = fec_interleave(v, rsb * v->fec->rsn + i);  in fec_read_bufs()
234 		bufio = v->fec->data_bufio;  in fec_read_bufs()
243 			if (unlikely(block >= v->fec->hash_blocks))  in fec_read_bufs()
252 			DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",  in fec_read_bufs()
258 			if (neras && *neras <= v->fec->roots)  in fec_read_bufs()
265 		if (bufio == v->fec->data_bufio &&  in fec_read_bufs()
274 			 * maximum number (i.e. fec->roots) of erasures  in fec_read_bufs()
276 			if (neras && *neras <= v->fec->roots &&  in fec_read_bufs()
302  * Allocate RS control structure and FEC buffers from preallocated mempools,
310 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
316 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
318 			DMERR("failed to allocate FEC buffer");  in fec_alloc_bufs()
328 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
336 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
350 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
395 		DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",  in fec_decode_rsb()
430 		DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);  in verity_fec_decode()
440 	 * For RS(M, N), the continuous FEC data is divided into blocks of N  in verity_fec_decode()
450 	res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
456 	rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
488 	struct dm_verity_fec *f = io->v->fec;  in verity_fec_finish_io()
535 	       v->fec->dev->name,  in verity_fec_status_table()
536 	       (unsigned long long)v->fec->blocks,  in verity_fec_status_table()
537 	       (unsigned long long)v->fec->start,  in verity_fec_status_table()
538 	       v->fec->roots);  in verity_fec_status_table()
545 	struct dm_verity_fec *f = v->fec;  in verity_fec_dtr()
565 	v->fec = NULL;  in verity_fec_dtr()
572 	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);  in fec_rs_alloc()
602 		ti->error = "FEC feature arguments require a value";  in verity_fec_parse_opt_args()
610 		r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev);  in verity_fec_parse_opt_args()
612 			ti->error = "FEC device lookup failed";  in verity_fec_parse_opt_args()
623 		v->fec->blocks = num_ll;  in verity_fec_parse_opt_args()
632 		v->fec->start = num_ll;  in verity_fec_parse_opt_args()
641 		v->fec->roots = num_c;  in verity_fec_parse_opt_args()
644 		ti->error = "Unrecognized verity FEC feature request";  in verity_fec_parse_opt_args()
652  * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
660 		v->ti->error = "Cannot allocate FEC structure";  in verity_fec_ctr_alloc()
663 	v->fec = f;  in verity_fec_ctr_alloc()
674 	struct dm_verity_fec *f = v->fec;  in verity_fec_ctr()
685 	 * FEC is computed over data blocks, possible metadata, and  in verity_fec_ctr()
686 	 * hash blocks. In other words, FEC covers total of fec_blocks  in verity_fec_ctr()
692 	 * where all data is stored on the same device and FEC covers  in verity_fec_ctr()
706 		ti->error = "Block sizes must match to use FEC";  in verity_fec_ctr()
748 		f->io_size = v->fec->roots << SECTOR_SHIFT;  in verity_fec_ctr()
754 		ti->error = "Cannot initialize FEC bufio client";  in verity_fec_ctr()
760 	fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);  in verity_fec_ctr()
762 		ti->error = "FEC device is too small";  in verity_fec_ctr()
770 		ti->error = "Cannot initialize FEC data bufio client";  in verity_fec_ctr()
791 		ti->error = "Cannot create FEC buffer cache";  in verity_fec_ctr()
800 		ti->error = "Cannot allocate FEC buffer prealloc pool";  in verity_fec_ctr()
806 		ti->error = "Cannot allocate FEC buffer extra pool";  in verity_fec_ctr()
814 		ti->error = "Cannot allocate FEC output pool";  in verity_fec_ctr()