464 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmac_cmd()  local
471 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmac_cmd()
472 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmac_cmd()
473 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmac_cmd()
476 	if (aes->src_len & (AES_BLOCK_SIZE - 1))  in ccp_run_aes_cmac_cmd()
479 	if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
482 	if (!aes->key || !aes->iv || !aes->src)  in ccp_run_aes_cmac_cmd()
485 	if (aes->cmac_final) {  in ccp_run_aes_cmac_cmd()
486 		if (aes->cmac_key_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
489 		if (!aes->cmac_key)  in ccp_run_aes_cmac_cmd()
503 	op.u.aes.type = aes->type;  in ccp_run_aes_cmac_cmd()
504 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmac_cmd()
505 	op.u.aes.action = aes->action;  in ccp_run_aes_cmac_cmd()
518 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmac_cmd()
519 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmac_cmd()
540 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
551 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
558 		if (aes->cmac_final && !src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
570 			ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,  in ccp_run_aes_cmac_cmd()
571 					      aes->cmac_key_len);  in ccp_run_aes_cmac_cmd()
582 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmac_cmd()
603 	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
620 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_gcm_cmd()  local
637 	if (!aes->iv)  in ccp_run_aes_gcm_cmd()
640 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_gcm_cmd()
641 		(aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_gcm_cmd()
642 		(aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_gcm_cmd()
645 	if (!aes->key) /* Gotta have a key SGL */  in ccp_run_aes_gcm_cmd()
654 	p_aad = aes->src;  in ccp_run_aes_gcm_cmd()
655 	p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);  in ccp_run_aes_gcm_cmd()
656 	p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);  in ccp_run_aes_gcm_cmd()
657 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
658 		ilen = aes->src_len;  in ccp_run_aes_gcm_cmd()
662 		ilen = aes->src_len - AES_BLOCK_SIZE;  in ccp_run_aes_gcm_cmd()
672 	op.u.aes.type = aes->type;  in ccp_run_aes_gcm_cmd()
681 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_gcm_cmd()
682 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_gcm_cmd()
702 	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;  in ccp_run_aes_gcm_cmd()
703 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
715 	if (aes->aad_len > 0) {  in ccp_run_aes_gcm_cmd()
717 		ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,  in ccp_run_aes_gcm_cmd()
723 		op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
724 		op.u.aes.action = CCP_AES_GHASHAAD;  in ccp_run_aes_gcm_cmd()
729 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
740 	op.u.aes.mode = CCP_AES_MODE_GCTR;  in ccp_run_aes_gcm_cmd()
741 	op.u.aes.action = aes->action;  in ccp_run_aes_gcm_cmd()
769 				unsigned int nbytes = aes->src_len  in ccp_run_aes_gcm_cmd()
774 					op.u.aes.size = (nbytes * 8) - 1;  in ccp_run_aes_gcm_cmd()
778 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
797 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
816 	final[0] = cpu_to_be64(aes->aad_len * 8);  in ccp_run_aes_gcm_cmd()
819 	op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
820 	op.u.aes.action = CCP_AES_GHASHFINAL;  in ccp_run_aes_gcm_cmd()
828 	op.u.aes.size = 0;  in ccp_run_aes_gcm_cmd()
829 	ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
833 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
854 	if (aes->src_len && !in_place)  in ccp_run_aes_gcm_cmd()
858 	if (aes->src_len)  in ccp_run_aes_gcm_cmd()
862 	if (aes->aad_len)  in ccp_run_aes_gcm_cmd()
876 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmd()  local
884 	if (aes->mode == CCP_AES_MODE_CMAC)  in ccp_run_aes_cmd()
887 	if (aes->mode == CCP_AES_MODE_GCM)  in ccp_run_aes_cmd()
890 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmd()
891 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmd()
892 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmd()
895 	if (((aes->mode == CCP_AES_MODE_ECB) ||  in ccp_run_aes_cmd()
896 	     (aes->mode == CCP_AES_MODE_CBC) ||  in ccp_run_aes_cmd()
897 	     (aes->mode == CCP_AES_MODE_CFB)) &&  in ccp_run_aes_cmd()
898 	    (aes->src_len & (AES_BLOCK_SIZE - 1)))  in ccp_run_aes_cmd()
901 	if (!aes->key || !aes->src || !aes->dst)  in ccp_run_aes_cmd()
904 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
905 		if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmd()
908 		if (!aes->iv)  in ccp_run_aes_cmd()
921 	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;  in ccp_run_aes_cmd()
922 	op.u.aes.type = aes->type;  in ccp_run_aes_cmd()
923 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmd()
924 	op.u.aes.action = aes->action;  in ccp_run_aes_cmd()
937 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmd()
938 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmd()
958 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
961 		ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()
971 	switch (aes->mode) {  in ccp_run_aes_cmd()
974 		op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;  in ccp_run_aes_cmd()
977 		op.u.aes.size = 0;  in ccp_run_aes_cmd()
984 	if (sg_virt(aes->src) == sg_virt(aes->dst))  in ccp_run_aes_cmd()
987 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
996 		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,  in ccp_run_aes_cmd()
1012 			if (aes->mode == CCP_AES_MODE_ECB)  in ccp_run_aes_cmd()
1016 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmd()
1025 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
1038 		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()