341 static void finish_scattered_hmac(struct crypt_ctl *crypt)  in finish_scattered_hmac()  argument
343 	struct aead_request *req = crypt->data.aead_req;  in finish_scattered_hmac()
353 	dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);  in finish_scattered_hmac()
359 	struct crypt_ctl *crypt;  in one_packet()  local
365 	crypt = crypt_phys2virt(phys);  in one_packet()
367 	switch (crypt->ctl_flags & CTL_FLAG_MASK) {  in one_packet()
369 		struct aead_request *req = crypt->data.aead_req;  in one_packet()
372 		free_buf_chain(dev, req_ctx->src, crypt->src_buf);  in one_packet()
373 		free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);  in one_packet()
375 			finish_scattered_hmac(crypt);  in one_packet()
381 		struct ablkcipher_request *req = crypt->data.ablk_req;  in one_packet()
385 			free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);  in one_packet()
387 		free_buf_chain(dev, req_ctx->src, crypt->src_buf);  in one_packet()
392 		ctx = crypto_tfm_ctx(crypt->data.tfm);  in one_packet()
393 		dma_pool_free(ctx_pool, crypt->regist_ptr,  in one_packet()
394 				crypt->regist_buf->phys_addr);  in one_packet()
395 		dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);  in one_packet()
400 		ctx = crypto_tfm_ctx(crypt->data.tfm);  in one_packet()
408 	crypt->ctl_flags = CTL_FLAG_UNUSED;  in one_packet()
602 	struct crypt_ctl *crypt;  in register_chain_var()  local
617 	crypt = get_crypt_desc_emerg();  in register_chain_var()
618 	if (!crypt) {  in register_chain_var()
630 	crypt->data.tfm = tfm;  in register_chain_var()
631 	crypt->regist_ptr = pad;  in register_chain_var()
632 	crypt->regist_buf = buf;  in register_chain_var()
634 	crypt->auth_offs = 0;  in register_chain_var()
635 	crypt->auth_len = HMAC_PAD_BLOCKLEN;  in register_chain_var()
636 	crypt->crypto_ctx = ctx_addr;  in register_chain_var()
637 	crypt->src_buf = buf_phys;  in register_chain_var()
638 	crypt->icv_rev_aes = target;  in register_chain_var()
639 	crypt->mode = NPE_OP_HASH_GEN_ICV;  in register_chain_var()
640 	crypt->init_len = init_len;  in register_chain_var()
641 	crypt->ctl_flags |= CTL_FLAG_GEN_ICV;  in register_chain_var()
649 	qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));  in register_chain_var()
703 	struct crypt_ctl *crypt;  in gen_rev_aes_key()  local
707 	crypt = get_crypt_desc_emerg();  in gen_rev_aes_key()
708 	if (!crypt) {  in gen_rev_aes_key()
713 	crypt->data.tfm = tfm;  in gen_rev_aes_key()
714 	crypt->crypt_offs = 0;  in gen_rev_aes_key()
715 	crypt->crypt_len = AES_BLOCK128;  in gen_rev_aes_key()
716 	crypt->src_buf = 0;  in gen_rev_aes_key()
717 	crypt->crypto_ctx = dir->npe_ctx_phys;  in gen_rev_aes_key()
718 	crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32);  in gen_rev_aes_key()
719 	crypt->mode = NPE_OP_ENC_GEN_KEY;  in gen_rev_aes_key()
720 	crypt->init_len = dir->npe_ctx_idx;  in gen_rev_aes_key()
721 	crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;  in gen_rev_aes_key()
724 	qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));  in gen_rev_aes_key()
877 	struct crypt_ctl *crypt;  in ablk_perform()  local
893 	crypt = get_crypt_desc();  in ablk_perform()
894 	if (!crypt)  in ablk_perform()
897 	crypt->data.ablk_req = req;  in ablk_perform()
898 	crypt->crypto_ctx = dir->npe_ctx_phys;  in ablk_perform()
899 	crypt->mode = dir->npe_mode;  in ablk_perform()
900 	crypt->init_len = dir->npe_ctx_idx;  in ablk_perform()
902 	crypt->crypt_offs = 0;  in ablk_perform()
903 	crypt->crypt_len = nbytes;  in ablk_perform()
906 	memcpy(crypt->iv, req->info, ivsize);  in ablk_perform()
909 		crypt->mode |= NPE_OP_NOT_IN_PLACE;  in ablk_perform()
918 		crypt->dst_buf = dst_hook.phys_next;  in ablk_perform()
928 	crypt->src_buf = src_hook.phys_next;  in ablk_perform()
929 	crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;  in ablk_perform()
930 	qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));  in ablk_perform()
935 	free_buf_chain(dev, req_ctx->src, crypt->src_buf);  in ablk_perform()
938 		free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);  in ablk_perform()
940 	crypt->ctl_flags = CTL_FLAG_UNUSED;  in ablk_perform()
984 	struct crypt_ctl *crypt;  in aead_perform()  local
1008 	crypt = get_crypt_desc();  in aead_perform()
1009 	if (!crypt)  in aead_perform()
1012 	crypt->data.aead_req = req;  in aead_perform()
1013 	crypt->crypto_ctx = dir->npe_ctx_phys;  in aead_perform()
1014 	crypt->mode = dir->npe_mode;  in aead_perform()
1015 	crypt->init_len = dir->npe_ctx_idx;  in aead_perform()
1017 	crypt->crypt_offs = cryptoffset;  in aead_perform()
1018 	crypt->crypt_len = eff_cryptlen;  in aead_perform()
1020 	crypt->auth_offs = 0;  in aead_perform()
1021 	crypt->auth_len = req->assoclen + cryptlen;  in aead_perform()
1023 	memcpy(crypt->iv, req->iv, ivsize);  in aead_perform()
1025 	buf = chainup_buffers(dev, req->src, crypt->auth_len,  in aead_perform()
1028 	crypt->src_buf = src_hook.phys_next;  in aead_perform()
1034 		crypt->icv_rev_aes = buf->phys_addr +  in aead_perform()
1042 		crypt->mode |= NPE_OP_NOT_IN_PLACE;  in aead_perform()
1045 		buf = chainup_buffers(dev, req->dst, crypt->auth_len,  in aead_perform()
1048 		crypt->dst_buf = dst_hook.phys_next;  in aead_perform()
1056 				crypt->icv_rev_aes = buf->phys_addr +  in aead_perform()
1065 				&crypt->icv_rev_aes);  in aead_perform()
1077 	crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;  in aead_perform()
1078 	qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));  in aead_perform()
1083 	free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);  in aead_perform()
1085 	free_buf_chain(dev, req_ctx->src, crypt->src_buf);  in aead_perform()
1086 	crypt->ctl_flags = CTL_FLAG_UNUSED;  in aead_perform()