475 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmac_cmd()  local
482 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmac_cmd()
483 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmac_cmd()
484 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmac_cmd()
487 	if (aes->src_len & (AES_BLOCK_SIZE - 1))  in ccp_run_aes_cmac_cmd()
490 	if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
493 	if (!aes->key || !aes->iv || !aes->src)  in ccp_run_aes_cmac_cmd()
496 	if (aes->cmac_final) {  in ccp_run_aes_cmac_cmd()
497 		if (aes->cmac_key_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
500 		if (!aes->cmac_key)  in ccp_run_aes_cmac_cmd()
514 	op.u.aes.type = aes->type;  in ccp_run_aes_cmac_cmd()
515 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmac_cmd()
516 	op.u.aes.action = aes->action;  in ccp_run_aes_cmac_cmd()
529 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmac_cmd()
530 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmac_cmd()
540 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_aes_cmac_cmd()
551 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
561 	/* Send data to the CCP AES engine */  in ccp_run_aes_cmac_cmd()
562 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
569 		if (aes->cmac_final && !src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
581 			ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,  in ccp_run_aes_cmac_cmd()
582 					      aes->cmac_key_len);  in ccp_run_aes_cmac_cmd()
593 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmac_cmd()
602 	/* Retrieve the AES context - convert from LE to BE using  in ccp_run_aes_cmac_cmd()
614 	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
631 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_gcm_cmd()  local
649 	if (!aes->iv)  in ccp_run_aes_gcm_cmd()
652 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_gcm_cmd()
653 		(aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_gcm_cmd()
654 		(aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_gcm_cmd()
657 	if (!aes->key) /* Gotta have a key SGL */  in ccp_run_aes_gcm_cmd()
661 	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;  in ccp_run_aes_gcm_cmd()
681 	p_aad = aes->src;  in ccp_run_aes_gcm_cmd()
682 	p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);  in ccp_run_aes_gcm_cmd()
683 	p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);  in ccp_run_aes_gcm_cmd()
684 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
685 		ilen = aes->src_len;  in ccp_run_aes_gcm_cmd()
689 		ilen = aes->src_len - authsize;  in ccp_run_aes_gcm_cmd()
701 	op.u.aes.type = aes->type;  in ccp_run_aes_gcm_cmd()
710 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_gcm_cmd()
711 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_gcm_cmd()
731 	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;  in ccp_run_aes_gcm_cmd()
732 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
744 	if (aes->aad_len > 0) {  in ccp_run_aes_gcm_cmd()
746 		ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,  in ccp_run_aes_gcm_cmd()
752 		op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
753 		op.u.aes.action = CCP_AES_GHASHAAD;  in ccp_run_aes_gcm_cmd()
758 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
769 	op.u.aes.mode = CCP_AES_MODE_GCTR;  in ccp_run_aes_gcm_cmd()
770 	op.u.aes.action = aes->action;  in ccp_run_aes_gcm_cmd()
802 					op.u.aes.size = (nbytes * 8) - 1;  in ccp_run_aes_gcm_cmd()
806 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
825 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
844 	final[0] = cpu_to_be64(aes->aad_len * 8);  in ccp_run_aes_gcm_cmd()
853 	op.u.aes.type = aes->type;  in ccp_run_aes_gcm_cmd()
854 	op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
855 	op.u.aes.action = CCP_AES_GHASHFINAL;  in ccp_run_aes_gcm_cmd()
863 	op.u.aes.size = 0;  in ccp_run_aes_gcm_cmd()
864 	ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
868 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
900 	if (aes->aad_len)  in ccp_run_aes_gcm_cmd()
915 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmd()  local
923 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmd()
924 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmd()
925 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmd()
928 	if (((aes->mode == CCP_AES_MODE_ECB) ||  in ccp_run_aes_cmd()
929 	     (aes->mode == CCP_AES_MODE_CBC)) &&  in ccp_run_aes_cmd()
930 	    (aes->src_len & (AES_BLOCK_SIZE - 1)))  in ccp_run_aes_cmd()
933 	if (!aes->key || !aes->src || !aes->dst)  in ccp_run_aes_cmd()
936 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
937 		if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmd()
940 		if (!aes->iv)  in ccp_run_aes_cmd()
953 	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;  in ccp_run_aes_cmd()
954 	op.u.aes.type = aes->type;  in ccp_run_aes_cmd()
955 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmd()
956 	op.u.aes.action = aes->action;  in ccp_run_aes_cmd()
969 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmd()
970 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmd()
980 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_aes_cmd()
990 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
991 		/* Load the AES context - convert to LE */  in ccp_run_aes_cmd()
993 		ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()
1003 	switch (aes->mode) {  in ccp_run_aes_cmd()
1006 		op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;  in ccp_run_aes_cmd()
1009 		op.u.aes.size = 0;  in ccp_run_aes_cmd()
1016 	if (sg_virt(aes->src) == sg_virt(aes->dst))  in ccp_run_aes_cmd()
1019 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
1028 		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,  in ccp_run_aes_cmd()
1034 	/* Send data to the CCP AES engine */  in ccp_run_aes_cmd()
1040 			/* Since we don't retrieve the AES context in ECB  in ccp_run_aes_cmd()
1044 			if (aes->mode == CCP_AES_MODE_ECB)  in ccp_run_aes_cmd()
1048 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmd()
1057 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
1058 		/* Retrieve the AES context - convert from LE to BE using  in ccp_run_aes_cmd()
1070 		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()
1202 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_xts_aes_cmd()
1244 	/* Send data to the CCP AES engine */  in ccp_run_xts_aes_cmd()
1259 	/* Retrieve the AES context - convert from LE to BE using  in ccp_run_xts_aes_cmd()
1343 	 * (like AES) must be in little endian format. Use the 256-bit byte  in ccp_run_des3_cmd()
2474 		switch (cmd->u.aes.mode) {  in ccp_run_cmd()