20 bool verity_fec_is_enabled(struct dm_verity *v)  in verity_fec_is_enabled()  argument
22 	return v->fec && v->fec->dev;  in verity_fec_is_enabled()
31 	return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io);  in fec_io()
37 static inline u64 fec_interleave(struct dm_verity *v, u64 offset)  in fec_interleave()  argument
41 	mod = do_div(offset, v->fec->rsn);  in fec_interleave()
42 	return offset + mod * (v->fec->rounds << v->data_dev_block_bits);  in fec_interleave()
48 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_rs8()  argument
54 	for (i = 0; i < v->fec->roots; i++)  in fec_decode_rs8()
57 	return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,  in fec_decode_rs8()
65 static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,  in fec_read_parity()  argument
71 	position = (index + rsb) * v->fec->roots;  in fec_read_parity()
72 	block = position >> v->data_dev_block_bits;  in fec_read_parity()
73 	*offset = (unsigned)(position - (block << v->data_dev_block_bits));  in fec_read_parity()
75 	res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf);  in fec_read_parity()
78 		      v->data_dev->name, (unsigned long long)rsb,  in fec_read_parity()
79 		      (unsigned long long)(v->fec->start + block),  in fec_read_parity()
108 static inline u8 *fec_buffer_rs_block(struct dm_verity *v,  in fec_buffer_rs_block()  argument
112 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
128 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_bufs()  argument
137 	par = fec_read_parity(v, rsb, block_offset, &offset, &buf);  in fec_decode_bufs()
146 		block = fec_buffer_rs_block(v, fio, n, i);  in fec_decode_bufs()
147 		res = fec_decode_rs8(v, fio, block, &par[offset], neras);  in fec_decode_bufs()
157 		if (block_offset >= 1 << v->data_dev_block_bits)  in fec_decode_bufs()
161 		offset += v->fec->roots;  in fec_decode_bufs()
162 		if (offset >= 1 << v->data_dev_block_bits) {  in fec_decode_bufs()
165 			par = fec_read_parity(v, rsb, block_offset, &offset, &buf);  in fec_decode_bufs()
177 			    v->data_dev->name, (unsigned long long)rsb, r);  in fec_decode_bufs()
180 			     v->data_dev->name, (unsigned long long)rsb, r);  in fec_decode_bufs()
188 static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,  in fec_is_erasure()  argument
191 	if (unlikely(verity_hash(v, verity_io_hash_req(v, io),  in fec_is_erasure()
192 				 data, 1 << v->data_dev_block_bits,  in fec_is_erasure()
193 				 verity_io_real_digest(v, io))))  in fec_is_erasure()
196 	return memcmp(verity_io_real_digest(v, io), want_digest,  in fec_is_erasure()
197 		      v->digest_size) != 0;  in fec_is_erasure()
204 static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,  in fec_read_bufs()  argument
215 	u8 want_digest[v->digest_size];  in fec_read_bufs()
225 	for (i = 0; i < v->fec->rsn; i++) {  in fec_read_bufs()
226 		ileaved = fec_interleave(v, rsb * v->fec->rsn + i);  in fec_read_bufs()
235 		block = ileaved >> v->data_dev_block_bits;  in fec_read_bufs()
236 		bufio = v->fec->data_bufio;  in fec_read_bufs()
238 		if (block >= v->data_blocks) {  in fec_read_bufs()
239 			block -= v->data_blocks;  in fec_read_bufs()
245 			if (unlikely(block >= v->fec->hash_blocks))  in fec_read_bufs()
248 			block += v->hash_start;  in fec_read_bufs()
249 			bufio = v->bufio;  in fec_read_bufs()
255 				     v->data_dev->name,  in fec_read_bufs()
260 			if (neras && *neras <= v->fec->roots)  in fec_read_bufs()
267 		if (bufio == v->fec->data_bufio &&  in fec_read_bufs()
268 		    verity_hash_for_block(v, io, block, want_digest,  in fec_read_bufs()
278 			if (neras && *neras <= v->fec->roots &&  in fec_read_bufs()
279 			    fec_is_erasure(v, io, want_digest, bbuf))  in fec_read_bufs()
290 			if (k >= 1 << v->data_dev_block_bits)  in fec_read_bufs()
293 			rs_block = fec_buffer_rs_block(v, fio, n, j);  in fec_read_bufs()
307 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_alloc_bufs()  argument
312 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
318 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
330 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
338 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
347 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_init_bufs()  argument
352 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
362 static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,  in fec_decode_rsb()  argument
369 	r = fec_alloc_bufs(v, fio);  in fec_decode_rsb()
373 	for (pos = 0; pos < 1 << v->data_dev_block_bits; ) {  in fec_decode_rsb()
374 		fec_init_bufs(v, fio);  in fec_decode_rsb()
376 		r = fec_read_bufs(v, io, rsb, offset, pos,  in fec_decode_rsb()
381 		r = fec_decode_bufs(v, fio, rsb, r, pos, neras);  in fec_decode_rsb()
389 	r = verity_hash(v, verity_io_hash_req(v, io), fio->output,  in fec_decode_rsb()
390 			1 << v->data_dev_block_bits,  in fec_decode_rsb()
391 			verity_io_real_digest(v, io));  in fec_decode_rsb()
395 	if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io),  in fec_decode_rsb()
396 		   v->digest_size)) {  in fec_decode_rsb()
398 			    v->data_dev->name, (unsigned long long)rsb, neras);  in fec_decode_rsb()
405 static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data,  in fec_bv_copy()  argument
420 int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,  in verity_fec_decode()  argument
428 	if (!verity_fec_is_enabled(v))  in verity_fec_decode()
432 		DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);  in verity_fec_decode()
439 		block += v->data_blocks;  in verity_fec_decode()
451 	offset = block << v->data_dev_block_bits;  in verity_fec_decode()
452 	res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
458 	rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
465 	r = fec_decode_rsb(v, io, fio, rsb, offset, false);  in verity_fec_decode()
467 		r = fec_decode_rsb(v, io, fio, rsb, offset, true);  in verity_fec_decode()
473 		memcpy(dest, fio->output, 1 << v->data_dev_block_bits);  in verity_fec_decode()
476 		r = verity_for_bv_block(v, io, iter, fec_bv_copy);  in verity_fec_decode()
490 	struct dm_verity_fec *f = io->v->fec;  in verity_fec_finish_io()
493 	if (!verity_fec_is_enabled(io->v))  in verity_fec_finish_io()
514 	if (!verity_fec_is_enabled(io->v))  in verity_fec_init_io()
527 unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz,  in verity_fec_status_table()  argument
530 	if (!verity_fec_is_enabled(v))  in verity_fec_status_table()
537 	       v->fec->dev->name,  in verity_fec_status_table()
538 	       (unsigned long long)v->fec->blocks,  in verity_fec_status_table()
539 	       (unsigned long long)v->fec->start,  in verity_fec_status_table()
540 	       v->fec->roots);  in verity_fec_status_table()
545 void verity_fec_dtr(struct dm_verity *v)  in verity_fec_dtr()  argument
547 	struct dm_verity_fec *f = v->fec;  in verity_fec_dtr()
549 	if (!verity_fec_is_enabled(v))  in verity_fec_dtr()
563 		dm_put_device(v->ti, f->dev);  in verity_fec_dtr()
566 	v->fec = NULL;  in verity_fec_dtr()
571 	struct dm_verity *v = (struct dm_verity *)pool_data;  in fec_rs_alloc()  local
573 	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);  in fec_rs_alloc()
592 int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,  in verity_fec_parse_opt_args()  argument
596 	struct dm_target *ti = v->ti;  in verity_fec_parse_opt_args()
611 		r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev);  in verity_fec_parse_opt_args()
619 		    ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))  in verity_fec_parse_opt_args()
620 		     >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {  in verity_fec_parse_opt_args()
624 		v->fec->blocks = num_ll;  in verity_fec_parse_opt_args()
628 		    ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >>  in verity_fec_parse_opt_args()
629 		     (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {  in verity_fec_parse_opt_args()
633 		v->fec->start = num_ll;  in verity_fec_parse_opt_args()
642 		v->fec->roots = num_c;  in verity_fec_parse_opt_args()
655 int verity_fec_ctr_alloc(struct dm_verity *v)  in verity_fec_ctr_alloc()  argument
661 		v->ti->error = "Cannot allocate FEC structure";  in verity_fec_ctr_alloc()
664 	v->fec = f;  in verity_fec_ctr_alloc()
673 int verity_fec_ctr(struct dm_verity *v)  in verity_fec_ctr()  argument
675 	struct dm_verity_fec *f = v->fec;  in verity_fec_ctr()
676 	struct dm_target *ti = v->ti;  in verity_fec_ctr()
680 	if (!verity_fec_is_enabled(v)) {  in verity_fec_ctr()
681 		verity_fec_dtr(v);  in verity_fec_ctr()
700 	hash_blocks = v->hash_blocks - v->hash_start;  in verity_fec_ctr()
706 	if (v->data_dev_block_bits != v->hash_dev_block_bits) {  in verity_fec_ctr()
730 	if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) {  in verity_fec_ctr()
739 	f->hash_blocks = f->blocks - v->data_blocks;  in verity_fec_ctr()
740 	if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) {  in verity_fec_ctr()
747 					  1 << v->data_dev_block_bits,  in verity_fec_ctr()
755 	    ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) {  in verity_fec_ctr()
760 	f->data_bufio = dm_bufio_client_create(v->data_dev->bdev,  in verity_fec_ctr()
761 					       1 << v->data_dev_block_bits,  in verity_fec_ctr()
768 	if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) {  in verity_fec_ctr()
775 			   fec_rs_free, (void *) v);  in verity_fec_ctr()
806 					1 << v->data_dev_block_bits);  in verity_fec_ctr()