37 static int crypto_check_alg(struct crypto_alg *alg)  in crypto_check_alg()  argument
39 	crypto_check_module_sig(alg->cra_module);  in crypto_check_alg()
41 	if (!alg->cra_name[0] || !alg->cra_driver_name[0])  in crypto_check_alg()
44 	if (alg->cra_alignmask & (alg->cra_alignmask + 1))  in crypto_check_alg()
48 	if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)  in crypto_check_alg()
51 	if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)  in crypto_check_alg()
55 	if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==  in crypto_check_alg()
57 		if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)  in crypto_check_alg()
60 		if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)  in crypto_check_alg()
64 	if (alg->cra_priority < 0)  in crypto_check_alg()
67 	refcount_set(&alg->cra_refcnt, 1);  in crypto_check_alg()
74 	inst->alg.cra_type->free(inst);  in crypto_free_instance()
77 static void crypto_destroy_instance(struct crypto_alg *alg)  in crypto_destroy_instance()  argument
79 	struct crypto_instance *inst = (void *)alg;  in crypto_destroy_instance()
95 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,  in crypto_more_spawns()  argument
116 	return &n->inst->alg.cra_users;  in crypto_more_spawns()
124 	if (crypto_is_dead(&inst->alg))  in crypto_remove_instance()
127 	inst->alg.cra_flags |= CRYPTO_ALG_DEAD;  in crypto_remove_instance()
132 	list_move(&inst->alg.cra_list, list);  in crypto_remove_instance()
134 	inst->alg.cra_destroy = crypto_destroy_instance;  in crypto_remove_instance()
136 	BUG_ON(!list_empty(&inst->alg.cra_users));  in crypto_remove_instance()
145 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,  in crypto_remove_spawns()  argument
148 	u32 new_type = (nalg ?: alg)->cra_flags;  in crypto_remove_spawns()
155 	spawns = &alg->cra_users;  in crypto_remove_spawns()
157 		if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)  in crypto_remove_spawns()
178 			spawn->dead = !spawn->registered || &inst->alg != nalg;  in crypto_remove_spawns()
183 			BUG_ON(&inst->alg == alg);  in crypto_remove_spawns()
185 			if (&inst->alg == nalg)  in crypto_remove_spawns()
188 			spawns = &inst->alg.cra_users;  in crypto_remove_spawns()
208 	} while ((spawns = crypto_more_spawns(alg, &stack, &top,  in crypto_remove_spawns()
218 			list_move(&spawn->list, &spawn->alg->cra_users);  in crypto_remove_spawns()
225 static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)  in crypto_alloc_test_larval()  argument
232 	larval = crypto_larval_alloc(alg->cra_name,  in crypto_alloc_test_larval()
233 				     alg->cra_flags | CRYPTO_ALG_TESTED, 0);  in crypto_alloc_test_larval()
237 	larval->adult = crypto_mod_get(alg);  in crypto_alloc_test_larval()
243 	refcount_set(&larval->alg.cra_refcnt, 1);  in crypto_alloc_test_larval()
244 	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,  in crypto_alloc_test_larval()
246 	larval->alg.cra_priority = alg->cra_priority;  in crypto_alloc_test_larval()
251 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)  in __crypto_register_alg()  argument
257 	if (crypto_is_dead(alg))  in __crypto_register_alg()
260 	INIT_LIST_HEAD(&alg->cra_users);  in __crypto_register_alg()
263 	alg->cra_flags &= ~CRYPTO_ALG_TESTED;  in __crypto_register_alg()
268 		if (q == alg)  in __crypto_register_alg()
275 			if (!strcmp(alg->cra_driver_name, q->cra_driver_name))  in __crypto_register_alg()
280 		if (!strcmp(q->cra_driver_name, alg->cra_name) ||  in __crypto_register_alg()
281 		    !strcmp(q->cra_name, alg->cra_driver_name))  in __crypto_register_alg()
285 	larval = crypto_alloc_test_larval(alg);  in __crypto_register_alg()
289 	list_add(&alg->cra_list, &crypto_alg_list);  in __crypto_register_alg()
292 		list_add(&larval->alg.cra_list, &crypto_alg_list);  in __crypto_register_alg()
294 		alg->cra_flags |= CRYPTO_ALG_TESTED;  in __crypto_register_alg()
296 	crypto_stats_init(alg);  in __crypto_register_alg()
309 	struct crypto_alg *alg;  in crypto_alg_tested()  local
330 	alg = test->adult;  in crypto_alg_tested()
332 	if (list_empty(&alg->cra_list))  in crypto_alg_tested()
336 		alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;  in crypto_alg_tested()
340 		alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;  in crypto_alg_tested()
342 	alg->cra_flags |= CRYPTO_ALG_TESTED;  in crypto_alg_tested()
350 		if (strcmp(alg->cra_name, q->cra_name))  in crypto_alg_tested()
353 		if (q->cra_priority > alg->cra_priority) {  in crypto_alg_tested()
360 		if (q == alg)  in crypto_alg_tested()
374 			if (strcmp(alg->cra_name, q->cra_name) &&  in crypto_alg_tested()
375 			    strcmp(alg->cra_driver_name, q->cra_name))  in crypto_alg_tested()
380 			if ((q->cra_flags ^ alg->cra_flags) & larval->mask)  in crypto_alg_tested()
383 			if (best && crypto_mod_get(alg))  in crypto_alg_tested()
384 				larval->adult = alg;  in crypto_alg_tested()
391 		if (strcmp(alg->cra_name, q->cra_name))  in crypto_alg_tested()
394 		if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&  in crypto_alg_tested()
395 		    q->cra_priority > alg->cra_priority)  in crypto_alg_tested()
398 		crypto_remove_spawns(q, &list, alg);  in crypto_alg_tested()
413 	struct crypto_alg *alg;  in crypto_remove_final()  local
416 	list_for_each_entry_safe(alg, n, list, cra_list) {  in crypto_remove_final()
417 		list_del_init(&alg->cra_list);  in crypto_remove_final()
418 		crypto_alg_put(alg);  in crypto_remove_final()
423 int crypto_register_alg(struct crypto_alg *alg)  in crypto_register_alg()  argument
429 	alg->cra_flags &= ~CRYPTO_ALG_DEAD;  in crypto_register_alg()
430 	err = crypto_check_alg(alg);  in crypto_register_alg()
435 	larval = __crypto_register_alg(alg);  in crypto_register_alg()
450 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)  in crypto_remove_alg()  argument
452 	if (unlikely(list_empty(&alg->cra_list)))  in crypto_remove_alg()
455 	alg->cra_flags |= CRYPTO_ALG_DEAD;  in crypto_remove_alg()
457 	list_del_init(&alg->cra_list);  in crypto_remove_alg()
458 	crypto_remove_spawns(alg, list, NULL);  in crypto_remove_alg()
463 void crypto_unregister_alg(struct crypto_alg *alg)  in crypto_unregister_alg()  argument
469 	ret = crypto_remove_alg(alg, &list);  in crypto_unregister_alg()
472 	if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))  in crypto_unregister_alg()
475 	BUG_ON(refcount_read(&alg->cra_refcnt) != 1);  in crypto_unregister_alg()
476 	if (alg->cra_destroy)  in crypto_unregister_alg()
477 		alg->cra_destroy(alg);  in crypto_unregister_alg()
566 		int err = crypto_remove_alg(&inst->alg, &users);  in crypto_unregister_template()
574 		BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);  in crypto_unregister_template()
624 	err = crypto_check_alg(&inst->alg);  in crypto_register_instance()
628 	inst->alg.cra_module = tmpl->module;  in crypto_register_instance()
629 	inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;  in crypto_register_instance()
644 		fips_internal |= spawn->alg->cra_flags;  in crypto_register_instance()
646 		crypto_mod_put(spawn->alg);  in crypto_register_instance()
651 	inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);  in crypto_register_instance()
653 	larval = __crypto_register_alg(&inst->alg);  in crypto_register_instance()
683 	crypto_remove_spawns(&inst->alg, &list, NULL);  in crypto_unregister_instance()
695 	struct crypto_alg *alg;  in crypto_grab_spawn()  local
705 	alg = crypto_find_alg(name, spawn->frontend,  in crypto_grab_spawn()
707 	if (IS_ERR(alg))  in crypto_grab_spawn()
708 		return PTR_ERR(alg);  in crypto_grab_spawn()
711 	if (!crypto_is_moribund(alg)) {  in crypto_grab_spawn()
712 		list_add(&spawn->list, &alg->cra_users);  in crypto_grab_spawn()
713 		spawn->alg = alg;  in crypto_grab_spawn()
717 		inst->alg.cra_flags |=  in crypto_grab_spawn()
718 			(alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);  in crypto_grab_spawn()
723 		crypto_mod_put(alg);  in crypto_grab_spawn()
730 	if (!spawn->alg) /* not yet initialized? */  in crypto_drop_spawn()
739 		crypto_mod_put(spawn->alg);  in crypto_drop_spawn()
745 	struct crypto_alg *alg = ERR_PTR(-EAGAIN);  in crypto_spawn_alg()  local
751 		alg = spawn->alg;  in crypto_spawn_alg()
752 		if (!crypto_mod_get(alg)) {  in crypto_spawn_alg()
753 			target = crypto_alg_get(alg);  in crypto_spawn_alg()
755 			alg = ERR_PTR(-EAGAIN);  in crypto_spawn_alg()
765 	return alg;  in crypto_spawn_alg()
771 	struct crypto_alg *alg;  in crypto_spawn_tfm()  local
774 	alg = crypto_spawn_alg(spawn);  in crypto_spawn_tfm()
775 	if (IS_ERR(alg))  in crypto_spawn_tfm()
776 		return ERR_CAST(alg);  in crypto_spawn_tfm()
779 	if (unlikely((alg->cra_flags ^ type) & mask))  in crypto_spawn_tfm()
782 	tfm = __crypto_alloc_tfm(alg, type, mask);  in crypto_spawn_tfm()
789 	crypto_mod_put(alg);  in crypto_spawn_tfm()
796 	struct crypto_alg *alg;  in crypto_spawn_tfm2()  local
799 	alg = crypto_spawn_alg(spawn);  in crypto_spawn_tfm2()
800 	if (IS_ERR(alg))  in crypto_spawn_tfm2()
801 		return ERR_CAST(alg);  in crypto_spawn_tfm2()
803 	tfm = crypto_create_tfm(alg, spawn->frontend);  in crypto_spawn_tfm2()
810 	crypto_mod_put(alg);  in crypto_spawn_tfm2()
896 			struct crypto_alg *alg)  in crypto_inst_setname()  argument
898 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,  in crypto_inst_setname()
899 		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)  in crypto_inst_setname()
902 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",  in crypto_inst_setname()
903 		     name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)  in crypto_inst_setname()
1000 unsigned int crypto_alg_extsize(struct crypto_alg *alg)  in crypto_alg_extsize()  argument
1002 	return alg->cra_ctxsize +  in crypto_alg_extsize()
1003 	       (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));  in crypto_alg_extsize()
1011 	struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);  in crypto_type_has_alg()  local
1013 	if (!IS_ERR(alg)) {  in crypto_type_has_alg()
1014 		crypto_mod_put(alg);  in crypto_type_has_alg()
1023 void crypto_stats_init(struct crypto_alg *alg)  in crypto_stats_init()  argument
1025 	memset(&alg->stats, 0, sizeof(alg->stats));  in crypto_stats_init()
1029 void crypto_stats_get(struct crypto_alg *alg)  in crypto_stats_get()  argument
1031 	crypto_alg_get(alg);  in crypto_stats_get()
1035 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,  in crypto_stats_aead_encrypt()  argument
1039 		atomic64_inc(&alg->stats.aead.err_cnt);  in crypto_stats_aead_encrypt()
1041 		atomic64_inc(&alg->stats.aead.encrypt_cnt);  in crypto_stats_aead_encrypt()
1042 		atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);  in crypto_stats_aead_encrypt()
1044 	crypto_alg_put(alg);  in crypto_stats_aead_encrypt()
1048 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,  in crypto_stats_aead_decrypt()  argument
1052 		atomic64_inc(&alg->stats.aead.err_cnt);  in crypto_stats_aead_decrypt()
1054 		atomic64_inc(&alg->stats.aead.decrypt_cnt);  in crypto_stats_aead_decrypt()
1055 		atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);  in crypto_stats_aead_decrypt()
1057 	crypto_alg_put(alg);  in crypto_stats_aead_decrypt()
1062 				   struct crypto_alg *alg)  in crypto_stats_akcipher_encrypt()  argument
1065 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_encrypt()
1067 		atomic64_inc(&alg->stats.akcipher.encrypt_cnt);  in crypto_stats_akcipher_encrypt()
1068 		atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);  in crypto_stats_akcipher_encrypt()
1070 	crypto_alg_put(alg);  in crypto_stats_akcipher_encrypt()
1075 				   struct crypto_alg *alg)  in crypto_stats_akcipher_decrypt()  argument
1078 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_decrypt()
1080 		atomic64_inc(&alg->stats.akcipher.decrypt_cnt);  in crypto_stats_akcipher_decrypt()
1081 		atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);  in crypto_stats_akcipher_decrypt()
1083 	crypto_alg_put(alg);  in crypto_stats_akcipher_decrypt()
1087 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)  in crypto_stats_akcipher_sign()  argument
1090 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_sign()
1092 		atomic64_inc(&alg->stats.akcipher.sign_cnt);  in crypto_stats_akcipher_sign()
1093 	crypto_alg_put(alg);  in crypto_stats_akcipher_sign()
1097 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)  in crypto_stats_akcipher_verify()  argument
1100 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_verify()
1102 		atomic64_inc(&alg->stats.akcipher.verify_cnt);  in crypto_stats_akcipher_verify()
1103 	crypto_alg_put(alg);  in crypto_stats_akcipher_verify()
1107 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)  in crypto_stats_compress()  argument
1110 		atomic64_inc(&alg->stats.compress.err_cnt);  in crypto_stats_compress()
1112 		atomic64_inc(&alg->stats.compress.compress_cnt);  in crypto_stats_compress()
1113 		atomic64_add(slen, &alg->stats.compress.compress_tlen);  in crypto_stats_compress()
1115 	crypto_alg_put(alg);  in crypto_stats_compress()
1119 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)  in crypto_stats_decompress()  argument
1122 		atomic64_inc(&alg->stats.compress.err_cnt);  in crypto_stats_decompress()
1124 		atomic64_inc(&alg->stats.compress.decompress_cnt);  in crypto_stats_decompress()
1125 		atomic64_add(slen, &alg->stats.compress.decompress_tlen);  in crypto_stats_decompress()
1127 	crypto_alg_put(alg);  in crypto_stats_decompress()
1132 			       struct crypto_alg *alg)  in crypto_stats_ahash_update()  argument
1135 		atomic64_inc(&alg->stats.hash.err_cnt);  in crypto_stats_ahash_update()
1137 		atomic64_add(nbytes, &alg->stats.hash.hash_tlen);  in crypto_stats_ahash_update()
1138 	crypto_alg_put(alg);  in crypto_stats_ahash_update()
1143 			      struct crypto_alg *alg)  in crypto_stats_ahash_final()  argument
1146 		atomic64_inc(&alg->stats.hash.err_cnt);  in crypto_stats_ahash_final()
1148 		atomic64_inc(&alg->stats.hash.hash_cnt);  in crypto_stats_ahash_final()
1149 		atomic64_add(nbytes, &alg->stats.hash.hash_tlen);  in crypto_stats_ahash_final()
1151 	crypto_alg_put(alg);  in crypto_stats_ahash_final()
1155 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_set_secret()  argument
1158 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_set_secret()
1160 		atomic64_inc(&alg->stats.kpp.setsecret_cnt);  in crypto_stats_kpp_set_secret()
1161 	crypto_alg_put(alg);  in crypto_stats_kpp_set_secret()
1165 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_generate_public_key()  argument
1168 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_generate_public_key()
1170 		atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);  in crypto_stats_kpp_generate_public_key()
1171 	crypto_alg_put(alg);  in crypto_stats_kpp_generate_public_key()
1175 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_compute_shared_secret()  argument
1178 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_compute_shared_secret()
1180 		atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);  in crypto_stats_kpp_compute_shared_secret()
1181 	crypto_alg_put(alg);  in crypto_stats_kpp_compute_shared_secret()
1185 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)  in crypto_stats_rng_seed()  argument
1188 		atomic64_inc(&alg->stats.rng.err_cnt);  in crypto_stats_rng_seed()
1190 		atomic64_inc(&alg->stats.rng.seed_cnt);  in crypto_stats_rng_seed()
1191 	crypto_alg_put(alg);  in crypto_stats_rng_seed()
1195 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,  in crypto_stats_rng_generate()  argument
1199 		atomic64_inc(&alg->stats.rng.err_cnt);  in crypto_stats_rng_generate()
1201 		atomic64_inc(&alg->stats.rng.generate_cnt);  in crypto_stats_rng_generate()
1202 		atomic64_add(dlen, &alg->stats.rng.generate_tlen);  in crypto_stats_rng_generate()
1204 	crypto_alg_put(alg);  in crypto_stats_rng_generate()
1209 				   struct crypto_alg *alg)  in crypto_stats_skcipher_encrypt()  argument
1212 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_skcipher_encrypt()
1214 		atomic64_inc(&alg->stats.cipher.encrypt_cnt);  in crypto_stats_skcipher_encrypt()
1215 		atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);  in crypto_stats_skcipher_encrypt()
1217 	crypto_alg_put(alg);  in crypto_stats_skcipher_encrypt()
1222 				   struct crypto_alg *alg)  in crypto_stats_skcipher_decrypt()  argument
1225 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_skcipher_decrypt()
1227 		atomic64_inc(&alg->stats.cipher.decrypt_cnt);  in crypto_stats_skcipher_decrypt()
1228 		atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);  in crypto_stats_skcipher_decrypt()
1230 	crypto_alg_put(alg);  in crypto_stats_skcipher_decrypt()