46 	struct dm_verity *v;  member
80 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)  in verity_map_sector()  argument
82 	return v->data_start + dm_target_offset(v->ti, bi_sector);  in verity_map_sector()
91 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,  in verity_position_at_level()  argument
94 	return block >> (level * v->hash_per_block_bits);  in verity_position_at_level()
97 static int verity_hash_update(struct dm_verity *v, struct ahash_request *req,  in verity_hash_update()  argument
128 static int verity_hash_init(struct dm_verity *v, struct ahash_request *req,  in verity_hash_init()  argument
133 	ahash_request_set_tfm(req, v->tfm);  in verity_hash_init()
146 	if (likely(v->salt_size && (v->version >= 1)))  in verity_hash_init()
147 		r = verity_hash_update(v, req, v->salt, v->salt_size, wait);  in verity_hash_init()
152 static int verity_hash_final(struct dm_verity *v, struct ahash_request *req,  in verity_hash_final()  argument
157 	if (unlikely(v->salt_size && (!v->version))) {  in verity_hash_final()
158 		r = verity_hash_update(v, req, v->salt, v->salt_size, wait);  in verity_hash_final()
172 int verity_hash(struct dm_verity *v, struct ahash_request *req,  in verity_hash()  argument
178 	r = verity_hash_init(v, req, &wait);  in verity_hash()
182 	r = verity_hash_update(v, req, data, len, &wait);  in verity_hash()
186 	r = verity_hash_final(v, req, digest, &wait);  in verity_hash()
192 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,  in verity_hash_at_level()  argument
195 	sector_t position = verity_position_at_level(v, block, level);  in verity_hash_at_level()
198 	*hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);  in verity_hash_at_level()
203 	idx = position & ((1 << v->hash_per_block_bits) - 1);  in verity_hash_at_level()
204 	if (!v->version)  in verity_hash_at_level()
205 		*offset = idx * v->digest_size;  in verity_hash_at_level()
207 		*offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);  in verity_hash_at_level()
213 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,  in verity_handle_err()  argument
219 	struct mapped_device *md = dm_table_get_md(v->ti->table);  in verity_handle_err()
222 	v->hash_failed = 1;  in verity_handle_err()
224 	if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)  in verity_handle_err()
227 	v->corrupted_errs++;  in verity_handle_err()
240 	DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,  in verity_handle_err()
243 	if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS)  in verity_handle_err()
244 		DMERR("%s: reached maximum errors", v->data_dev->name);  in verity_handle_err()
252 	if (v->mode == DM_VERITY_MODE_LOGGING)  in verity_handle_err()
255 	if (v->mode == DM_VERITY_MODE_RESTART)  in verity_handle_err()
258 	if (v->mode == DM_VERITY_MODE_PANIC)  in verity_handle_err()
268  * On successful return, verity_io_want_digest(v, io) contains the hash value
273  * against current value of verity_io_want_digest(v, io).
275 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,  in verity_verify_level()  argument
286 	verity_hash_at_level(v, block, level, &hash_block, &offset);  in verity_verify_level()
288 	data = dm_bufio_read(v->bufio, hash_block, &buf);  in verity_verify_level()
300 		r = verity_hash(v, verity_io_hash_req(v, io),  in verity_verify_level()
301 				data, 1 << v->hash_dev_block_bits,  in verity_verify_level()
302 				verity_io_real_digest(v, io));  in verity_verify_level()
306 		if (likely(memcmp(verity_io_real_digest(v, io), want_digest,  in verity_verify_level()
307 				  v->digest_size) == 0))  in verity_verify_level()
309 		else if (verity_fec_decode(v, io,  in verity_verify_level()
313 		else if (verity_handle_err(v,  in verity_verify_level()
322 	memcpy(want_digest, data, v->digest_size);  in verity_verify_level()
334 int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,  in verity_hash_for_block()  argument
339 	if (likely(v->levels)) {  in verity_hash_for_block()
347 		r = verity_verify_level(v, io, block, 0, true, digest);  in verity_hash_for_block()
352 	memcpy(digest, v->root_digest, v->digest_size);  in verity_hash_for_block()
354 	for (i = v->levels - 1; i >= 0; i--) {  in verity_hash_for_block()
355 		r = verity_verify_level(v, io, block, i, false, digest);  in verity_hash_for_block()
360 	if (!r && v->zero_digest)  in verity_hash_for_block()
361 		*is_zero = !memcmp(v->zero_digest, digest, v->digest_size);  in verity_hash_for_block()
371 static int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io,  in verity_for_io_block()  argument
374 	unsigned int todo = 1 << v->data_dev_block_bits;  in verity_for_io_block()
375 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);  in verity_for_io_block()
377 	struct ahash_request *req = verity_io_hash_req(v, io);  in verity_for_io_block()
412  * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec
415 int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io,  in verity_for_bv_block()  argument
417 			int (*process)(struct dm_verity *v,  in verity_for_bv_block()  argument
421 	unsigned todo = 1 << v->data_dev_block_bits;  in verity_for_bv_block()
422 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);  in verity_for_bv_block()
436 		r = process(v, io, page + bv.bv_offset, len);  in verity_for_bv_block()
449 static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io,  in verity_bv_zero()  argument
459 static inline void verity_bv_skip_block(struct dm_verity *v,  in verity_bv_skip_block()  argument
463 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);  in verity_bv_skip_block()
465 	bio_advance_iter(bio, iter, 1 << v->data_dev_block_bits);  in verity_bv_skip_block()
474 	struct dm_verity *v = io->v;  in verity_verify_io()  local
482 		struct ahash_request *req = verity_io_hash_req(v, io);  in verity_verify_io()
484 		if (v->validated_blocks &&  in verity_verify_io()
485 		    likely(test_bit(cur_block, v->validated_blocks))) {  in verity_verify_io()
486 			verity_bv_skip_block(v, io, &io->iter);  in verity_verify_io()
490 		r = verity_hash_for_block(v, io, cur_block,  in verity_verify_io()
491 					  verity_io_want_digest(v, io),  in verity_verify_io()
501 			r = verity_for_bv_block(v, io, &io->iter,  in verity_verify_io()
509 		r = verity_hash_init(v, req, &wait);  in verity_verify_io()
514 		r = verity_for_io_block(v, io, &io->iter, &wait);  in verity_verify_io()
518 		r = verity_hash_final(v, req, verity_io_real_digest(v, io),  in verity_verify_io()
523 		if (likely(memcmp(verity_io_real_digest(v, io),  in verity_verify_io()
524 				  verity_io_want_digest(v, io), v->digest_size) == 0)) {  in verity_verify_io()
525 			if (v->validated_blocks)  in verity_verify_io()
526 				set_bit(cur_block, v->validated_blocks);  in verity_verify_io()
529 		else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,  in verity_verify_io()
532 		else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,  in verity_verify_io()
545 	struct dm_verity *v = io->v;  in verity_finish_io()  local
546 	struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);  in verity_finish_io()
567 	if (bio->bi_status && !verity_fec_is_enabled(io->v)) {  in verity_end_io()
573 	queue_work(io->v->verify_wq, &io->work);  in verity_end_io()
585 	struct dm_verity *v = pw->v;  in verity_prefetch_io()  local
588 	for (i = v->levels - 2; i >= 0; i--) {  in verity_prefetch_io()
591 		verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);  in verity_prefetch_io()
592 		verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);  in verity_prefetch_io()
596 			cluster >>= v->data_dev_block_bits;  in verity_prefetch_io()
605 			if (unlikely(hash_block_end >= v->hash_blocks))  in verity_prefetch_io()
606 				hash_block_end = v->hash_blocks - 1;  in verity_prefetch_io()
609 		dm_bufio_prefetch(v->bufio, hash_block_start,  in verity_prefetch_io()
616 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)  in verity_submit_prefetch()  argument
622 	if (v->validated_blocks) {  in verity_submit_prefetch()
623 		while (n_blocks && test_bit(block, v->validated_blocks)) {  in verity_submit_prefetch()
628 					    v->validated_blocks))  in verity_submit_prefetch()
641 	pw->v = v;  in verity_submit_prefetch()
644 	queue_work(v->verify_wq, &pw->work);  in verity_submit_prefetch()
653 	struct dm_verity *v = ti->private;  in verity_map()  local
656 	bio_set_dev(bio, v->data_dev->bdev);  in verity_map()
657 	bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);  in verity_map()
660 	    ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {  in verity_map()
666 	    (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {  in verity_map()
675 	io->v = v;  in verity_map()
677 	io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);  in verity_map()
678 	io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;  in verity_map()
686 	verity_submit_prefetch(v, io);  in verity_map()
694  * Status: V (valid) or C (corruption found)
699 	struct dm_verity *v = ti->private;  in verity_status()  local
706 		DMEMIT("%c", v->hash_failed ? 'C' : 'V');  in verity_status()
710 			v->version,  in verity_status()
711 			v->data_dev->name,  in verity_status()
712 			v->hash_dev->name,  in verity_status()
713 			1 << v->data_dev_block_bits,  in verity_status()
714 			1 << v->hash_dev_block_bits,  in verity_status()
715 			(unsigned long long)v->data_blocks,  in verity_status()
716 			(unsigned long long)v->hash_start,  in verity_status()
717 			v->alg_name  in verity_status()
719 		for (x = 0; x < v->digest_size; x++)  in verity_status()
720 			DMEMIT("%02x", v->root_digest[x]);  in verity_status()
722 		if (!v->salt_size)  in verity_status()
725 			for (x = 0; x < v->salt_size; x++)  in verity_status()
726 				DMEMIT("%02x", v->salt[x]);  in verity_status()
727 		if (v->mode != DM_VERITY_MODE_EIO)  in verity_status()
729 		if (verity_fec_is_enabled(v))  in verity_status()
731 		if (v->zero_digest)  in verity_status()
733 		if (v->validated_blocks)  in verity_status()
735 		if (v->signature_key_desc)  in verity_status()
740 		if (v->mode != DM_VERITY_MODE_EIO) {  in verity_status()
742 			switch (v->mode) {  in verity_status()
756 		if (v->zero_digest)  in verity_status()
758 		if (v->validated_blocks)  in verity_status()
760 		sz = verity_fec_status_table(v, sz, result, maxlen);  in verity_status()
761 		if (v->signature_key_desc)  in verity_status()
763 				" %s", v->signature_key_desc);  in verity_status()
770 	struct dm_verity *v = ti->private;  in verity_prepare_ioctl()  local
772 	*bdev = v->data_dev->bdev;  in verity_prepare_ioctl()
774 	if (v->data_start ||  in verity_prepare_ioctl()
775 	    ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)  in verity_prepare_ioctl()
783 	struct dm_verity *v = ti->private;  in verity_iterate_devices()  local
785 	return fn(ti, v->data_dev, v->data_start, ti->len, data);  in verity_iterate_devices()
790 	struct dm_verity *v = ti->private;  in verity_io_hints()  local
792 	if (limits->logical_block_size < 1 << v->data_dev_block_bits)  in verity_io_hints()
793 		limits->logical_block_size = 1 << v->data_dev_block_bits;  in verity_io_hints()
795 	if (limits->physical_block_size < 1 << v->data_dev_block_bits)  in verity_io_hints()
796 		limits->physical_block_size = 1 << v->data_dev_block_bits;  in verity_io_hints()
803 	struct dm_verity *v = ti->private;  in verity_dtr()  local
805 	if (v->verify_wq)  in verity_dtr()
806 		destroy_workqueue(v->verify_wq);  in verity_dtr()
808 	if (v->bufio)  in verity_dtr()
809 		dm_bufio_client_destroy(v->bufio);  in verity_dtr()
811 	kvfree(v->validated_blocks);  in verity_dtr()
812 	kfree(v->salt);  in verity_dtr()
813 	kfree(v->root_digest);  in verity_dtr()
814 	kfree(v->zero_digest);  in verity_dtr()
816 	if (v->tfm)  in verity_dtr()
817 		crypto_free_ahash(v->tfm);  in verity_dtr()
819 	kfree(v->alg_name);  in verity_dtr()
821 	if (v->hash_dev)  in verity_dtr()
822 		dm_put_device(ti, v->hash_dev);  in verity_dtr()
824 	if (v->data_dev)  in verity_dtr()
825 		dm_put_device(ti, v->data_dev);  in verity_dtr()
827 	verity_fec_dtr(v);  in verity_dtr()
829 	kfree(v->signature_key_desc);  in verity_dtr()
831 	kfree(v);  in verity_dtr()
834 static int verity_alloc_most_once(struct dm_verity *v)  in verity_alloc_most_once()  argument
836 	struct dm_target *ti = v->ti;  in verity_alloc_most_once()
839 	if (v->data_blocks > INT_MAX) {  in verity_alloc_most_once()
844 	v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),  in verity_alloc_most_once()
847 	if (!v->validated_blocks) {  in verity_alloc_most_once()
855 static int verity_alloc_zero_digest(struct dm_verity *v)  in verity_alloc_zero_digest()  argument
861 	v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);  in verity_alloc_zero_digest()
863 	if (!v->zero_digest)  in verity_alloc_zero_digest()
866 	req = kmalloc(v->ahash_reqsize, GFP_KERNEL);  in verity_alloc_zero_digest()
871 	zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);  in verity_alloc_zero_digest()
876 	r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits,  in verity_alloc_zero_digest()
877 			v->zero_digest);  in verity_alloc_zero_digest()
886 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,  in verity_parse_opt_args()  argument
891 	struct dm_target *ti = v->ti;  in verity_parse_opt_args()
910 			v->mode = DM_VERITY_MODE_LOGGING;  in verity_parse_opt_args()
914 			v->mode = DM_VERITY_MODE_RESTART;  in verity_parse_opt_args()
918 			v->mode = DM_VERITY_MODE_PANIC;  in verity_parse_opt_args()
922 			r = verity_alloc_zero_digest(v);  in verity_parse_opt_args()
930 			r = verity_alloc_most_once(v);  in verity_parse_opt_args()
936 			r = verity_fec_parse_opt_args(as, v, &argc, arg_name);  in verity_parse_opt_args()
941 			r = verity_verify_sig_parse_opt_args(as, v,  in verity_parse_opt_args()
973 	struct dm_verity *v;  in verity_ctr()  local
984 	v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);  in verity_ctr()
985 	if (!v) {  in verity_ctr()
989 	ti->private = v;  in verity_ctr()
990 	v->ti = ti;  in verity_ctr()
992 	r = verity_fec_ctr_alloc(v);  in verity_ctr()
1014 	v->version = num;  in verity_ctr()
1016 	r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);  in verity_ctr()
1022 	r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);  in verity_ctr()
1030 	    num < bdev_logical_block_size(v->data_dev->bdev) ||  in verity_ctr()
1036 	v->data_dev_block_bits = __ffs(num);  in verity_ctr()
1040 	    num < bdev_logical_block_size(v->hash_dev->bdev) ||  in verity_ctr()
1046 	v->hash_dev_block_bits = __ffs(num);  in verity_ctr()
1049 	    (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))  in verity_ctr()
1050 	    >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {  in verity_ctr()
1055 	v->data_blocks = num_ll;  in verity_ctr()
1057 	if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {  in verity_ctr()
1064 	    (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))  in verity_ctr()
1065 	    >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {  in verity_ctr()
1070 	v->hash_start = num_ll;  in verity_ctr()
1072 	v->alg_name = kstrdup(argv[7], GFP_KERNEL);  in verity_ctr()
1073 	if (!v->alg_name) {  in verity_ctr()
1079 	v->tfm = crypto_alloc_ahash(v->alg_name, 0, 0);  in verity_ctr()
1080 	if (IS_ERR(v->tfm)) {  in verity_ctr()
1082 		r = PTR_ERR(v->tfm);  in verity_ctr()
1083 		v->tfm = NULL;  in verity_ctr()
1092 	DMINFO("%s using implementation \"%s\"", v->alg_name,  in verity_ctr()
1093 	       crypto_hash_alg_common(v->tfm)->base.cra_driver_name);  in verity_ctr()
1095 	v->digest_size = crypto_ahash_digestsize(v->tfm);  in verity_ctr()
1096 	if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {  in verity_ctr()
1101 	v->ahash_reqsize = sizeof(struct ahash_request) +  in verity_ctr()
1102 		crypto_ahash_reqsize(v->tfm);  in verity_ctr()
1104 	v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);  in verity_ctr()
1105 	if (!v->root_digest) {  in verity_ctr()
1110 	if (strlen(argv[8]) != v->digest_size * 2 ||  in verity_ctr()
1111 	    hex2bin(v->root_digest, argv[8], v->digest_size)) {  in verity_ctr()
1119 		v->salt_size = strlen(argv[9]) / 2;  in verity_ctr()
1120 		v->salt = kmalloc(v->salt_size, GFP_KERNEL);  in verity_ctr()
1121 		if (!v->salt) {  in verity_ctr()
1126 		if (strlen(argv[9]) != v->salt_size * 2 ||  in verity_ctr()
1127 		    hex2bin(v->salt, argv[9], v->salt_size)) {  in verity_ctr()
1142 		r = verity_parse_opt_args(&as, v, &verify_args);  in verity_ctr()
1156 	v->hash_per_block_bits =  in verity_ctr()
1157 		__fls((1 << v->hash_dev_block_bits) / v->digest_size);  in verity_ctr()
1159 	v->levels = 0;  in verity_ctr()
1160 	if (v->data_blocks)  in verity_ctr()
1161 		while (v->hash_per_block_bits * v->levels < 64 &&  in verity_ctr()
1162 		       (unsigned long long)(v->data_blocks - 1) >>  in verity_ctr()
1163 		       (v->hash_per_block_bits * v->levels))  in verity_ctr()
1164 			v->levels++;  in verity_ctr()
1166 	if (v->levels > DM_VERITY_MAX_LEVELS) {  in verity_ctr()
1172 	hash_position = v->hash_start;  in verity_ctr()
1173 	for (i = v->levels - 1; i >= 0; i--) {  in verity_ctr()
1175 		v->hash_level_block[i] = hash_position;  in verity_ctr()
1176 		s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)  in verity_ctr()
1177 					>> ((i + 1) * v->hash_per_block_bits);  in verity_ctr()
1185 	v->hash_blocks = hash_position;  in verity_ctr()
1187 	v->bufio = dm_bufio_client_create(v->hash_dev->bdev,  in verity_ctr()
1188 		1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),  in verity_ctr()
1190 	if (IS_ERR(v->bufio)) {  in verity_ctr()
1192 		r = PTR_ERR(v->bufio);  in verity_ctr()
1193 		v->bufio = NULL;  in verity_ctr()
1197 	if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {  in verity_ctr()
1204 …v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_onl…  in verity_ctr()
1205 	if (!v->verify_wq) {  in verity_ctr()
1212 				v->ahash_reqsize + v->digest_size * 2;  in verity_ctr()
1214 	r = verity_fec_ctr(v);  in verity_ctr()