943 		       struct aead_edesc *edesc,  in aead_unmap()  argument
947 		   edesc->src_nents, edesc->dst_nents, 0, 0,  in aead_unmap()
948 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);  in aead_unmap()
951 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,  in skcipher_unmap()  argument
958 		   edesc->src_nents, edesc->dst_nents,  in skcipher_unmap()
959 		   edesc->iv_dma, ivsize,  in skcipher_unmap()
960 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);  in skcipher_unmap()
967 	struct aead_edesc *edesc;  in aead_encrypt_done()  local
972 	edesc = container_of(desc, struct aead_edesc, hw_desc[0]);  in aead_encrypt_done()
977 	aead_unmap(jrdev, edesc, req);  in aead_encrypt_done()
979 	kfree(edesc);  in aead_encrypt_done()
988 	struct aead_edesc *edesc;  in aead_decrypt_done()  local
993 	edesc = container_of(desc, struct aead_edesc, hw_desc[0]);  in aead_decrypt_done()
998 	aead_unmap(jrdev, edesc, req);  in aead_decrypt_done()
1000 	kfree(edesc);  in aead_decrypt_done()
1009 	struct skcipher_edesc *edesc;  in skcipher_encrypt_done()  local
1016 	edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);  in skcipher_encrypt_done()
1021 	skcipher_unmap(jrdev, edesc, req);  in skcipher_encrypt_done()
1029 		memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,  in skcipher_encrypt_done()
1033 				     edesc->src_nents > 1 ? 100 : ivsize, 1);  in skcipher_encrypt_done()
1038 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);  in skcipher_encrypt_done()
1040 	kfree(edesc);  in skcipher_encrypt_done()
1049 	struct skcipher_edesc *edesc;  in skcipher_decrypt_done()  local
1056 	edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);  in skcipher_decrypt_done()
1060 	skcipher_unmap(jrdev, edesc, req);  in skcipher_decrypt_done()
1068 		memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,  in skcipher_decrypt_done()
1078 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);  in skcipher_decrypt_done()
1080 	kfree(edesc);  in skcipher_decrypt_done()
1089 			  struct aead_edesc *edesc,  in init_aead_job()  argument
1095 	u32 *desc = edesc->hw_desc;  in init_aead_job()
1109 		src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :  in init_aead_job()
1113 		src_dma = edesc->sec4_sg_dma;  in init_aead_job()
1114 		sec4_sg_index += edesc->mapped_src_nents;  in init_aead_job()
1125 		if (!edesc->mapped_dst_nents) {  in init_aead_job()
1128 		} else if (edesc->mapped_dst_nents == 1) {  in init_aead_job()
1132 			dst_dma = edesc->sec4_sg_dma +  in init_aead_job()
1150 			 struct aead_edesc *edesc,  in init_gcm_job()  argument
1156 	u32 *desc = edesc->hw_desc;  in init_gcm_job()
1160 	init_aead_job(req, edesc, all_contig, encrypt);  in init_gcm_job()
1180 				struct aead_edesc *edesc, bool all_contig,  in init_chachapoly_job()  argument
1186 	u32 *desc = edesc->hw_desc;  in init_chachapoly_job()
1189 	init_aead_job(req, edesc, all_contig, encrypt);  in init_chachapoly_job()
1214 			     struct aead_edesc *edesc,  in init_authenc_job()  argument
1226 	u32 *desc = edesc->hw_desc;  in init_authenc_job()
1244 	init_aead_job(req, edesc, all_contig, encrypt);  in init_authenc_job()
1266 			      struct skcipher_edesc *edesc,  in init_skcipher_job()  argument
1273 	u32 *desc = edesc->hw_desc;  in init_skcipher_job()
1282 	       (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);  in init_skcipher_job()
1286 		     edesc->src_nents > 1 ? 100 : req->cryptlen, 1);  in init_skcipher_job()
1294 	if (ivsize || edesc->mapped_src_nents > 1) {  in init_skcipher_job()
1295 		src_dma = edesc->sec4_sg_dma;  in init_skcipher_job()
1296 		sec4_sg_index = edesc->mapped_src_nents + !!ivsize;  in init_skcipher_job()
1307 	} else if (!ivsize && edesc->mapped_dst_nents == 1) {  in init_skcipher_job()
1310 		dst_dma = edesc->sec4_sg_dma + sec4_sg_index *  in init_skcipher_job()
1332 	struct aead_edesc *edesc;  in aead_edesc_alloc()  local
1414 	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,  in aead_edesc_alloc()
1416 	if (!edesc) {  in aead_edesc_alloc()
1422 	edesc->src_nents = src_nents;  in aead_edesc_alloc()
1423 	edesc->dst_nents = dst_nents;  in aead_edesc_alloc()
1424 	edesc->mapped_src_nents = mapped_src_nents;  in aead_edesc_alloc()
1425 	edesc->mapped_dst_nents = mapped_dst_nents;  in aead_edesc_alloc()
1426 	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +  in aead_edesc_alloc()
1433 				   edesc->sec4_sg + sec4_sg_index, 0);  in aead_edesc_alloc()
1438 				   edesc->sec4_sg + sec4_sg_index, 0);  in aead_edesc_alloc()
1442 		return edesc;  in aead_edesc_alloc()
1444 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in aead_edesc_alloc()
1446 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in aead_edesc_alloc()
1448 		aead_unmap(jrdev, edesc, req);  in aead_edesc_alloc()
1449 		kfree(edesc);  in aead_edesc_alloc()
1453 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in aead_edesc_alloc()
1455 	return edesc;  in aead_edesc_alloc()
1460 	struct aead_edesc *edesc;  in gcm_encrypt()  local
1469 	edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);  in gcm_encrypt()
1470 	if (IS_ERR(edesc))  in gcm_encrypt()
1471 		return PTR_ERR(edesc);  in gcm_encrypt()
1474 	init_gcm_job(req, edesc, all_contig, true);  in gcm_encrypt()
1477 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in gcm_encrypt()
1478 			     desc_bytes(edesc->hw_desc), 1);  in gcm_encrypt()
1480 	desc = edesc->hw_desc;  in gcm_encrypt()
1485 		aead_unmap(jrdev, edesc, req);  in gcm_encrypt()
1486 		kfree(edesc);  in gcm_encrypt()
1494 	struct aead_edesc *edesc;  in chachapoly_encrypt()  local
1502 	edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,  in chachapoly_encrypt()
1504 	if (IS_ERR(edesc))  in chachapoly_encrypt()
1505 		return PTR_ERR(edesc);  in chachapoly_encrypt()
1507 	desc = edesc->hw_desc;  in chachapoly_encrypt()
1509 	init_chachapoly_job(req, edesc, all_contig, true);  in chachapoly_encrypt()
1518 		aead_unmap(jrdev, edesc, req);  in chachapoly_encrypt()
1519 		kfree(edesc);  in chachapoly_encrypt()
1527 	struct aead_edesc *edesc;  in chachapoly_decrypt()  local
1535 	edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,  in chachapoly_decrypt()
1537 	if (IS_ERR(edesc))  in chachapoly_decrypt()
1538 		return PTR_ERR(edesc);  in chachapoly_decrypt()
1540 	desc = edesc->hw_desc;  in chachapoly_decrypt()
1542 	init_chachapoly_job(req, edesc, all_contig, false);  in chachapoly_decrypt()
1551 		aead_unmap(jrdev, edesc, req);  in chachapoly_decrypt()
1552 		kfree(edesc);  in chachapoly_decrypt()
1565 	struct aead_edesc *edesc;  in aead_encrypt()  local
1574 	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,  in aead_encrypt()
1576 	if (IS_ERR(edesc))  in aead_encrypt()
1577 		return PTR_ERR(edesc);  in aead_encrypt()
1580 	init_authenc_job(req, edesc, all_contig, true);  in aead_encrypt()
1583 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in aead_encrypt()
1584 			     desc_bytes(edesc->hw_desc), 1);  in aead_encrypt()
1586 	desc = edesc->hw_desc;  in aead_encrypt()
1591 		aead_unmap(jrdev, edesc, req);  in aead_encrypt()
1592 		kfree(edesc);  in aead_encrypt()
1600 	struct aead_edesc *edesc;  in gcm_decrypt()  local
1609 	edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);  in gcm_decrypt()
1610 	if (IS_ERR(edesc))  in gcm_decrypt()
1611 		return PTR_ERR(edesc);  in gcm_decrypt()
1614 	init_gcm_job(req, edesc, all_contig, false);  in gcm_decrypt()
1617 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in gcm_decrypt()
1618 			     desc_bytes(edesc->hw_desc), 1);  in gcm_decrypt()
1620 	desc = edesc->hw_desc;  in gcm_decrypt()
1625 		aead_unmap(jrdev, edesc, req);  in gcm_decrypt()
1626 		kfree(edesc);  in gcm_decrypt()
1639 	struct aead_edesc *edesc;  in aead_decrypt()  local
1652 	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,  in aead_decrypt()
1654 	if (IS_ERR(edesc))  in aead_decrypt()
1655 		return PTR_ERR(edesc);  in aead_decrypt()
1658 	init_authenc_job(req, edesc, all_contig, false);  in aead_decrypt()
1661 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in aead_decrypt()
1662 			     desc_bytes(edesc->hw_desc), 1);  in aead_decrypt()
1664 	desc = edesc->hw_desc;  in aead_decrypt()
1669 		aead_unmap(jrdev, edesc, req);  in aead_decrypt()
1670 		kfree(edesc);  in aead_decrypt()
1688 	struct skcipher_edesc *edesc;  in skcipher_edesc_alloc()  local
1766 	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,  in skcipher_edesc_alloc()
1768 	if (!edesc) {  in skcipher_edesc_alloc()
1775 	edesc->src_nents = src_nents;  in skcipher_edesc_alloc()
1776 	edesc->dst_nents = dst_nents;  in skcipher_edesc_alloc()
1777 	edesc->mapped_src_nents = mapped_src_nents;  in skcipher_edesc_alloc()
1778 	edesc->mapped_dst_nents = mapped_dst_nents;  in skcipher_edesc_alloc()
1779 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in skcipher_edesc_alloc()
1780 	edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +  in skcipher_edesc_alloc()
1785 		iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;  in skcipher_edesc_alloc()
1793 			kfree(edesc);  in skcipher_edesc_alloc()
1797 		dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);  in skcipher_edesc_alloc()
1800 		sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +  in skcipher_edesc_alloc()
1804 		sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +  in skcipher_edesc_alloc()
1808 		dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +  in skcipher_edesc_alloc()
1812 		sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +  in skcipher_edesc_alloc()
1816 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in skcipher_edesc_alloc()
1819 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in skcipher_edesc_alloc()
1823 			kfree(edesc);  in skcipher_edesc_alloc()
1828 	edesc->iv_dma = iv_dma;  in skcipher_edesc_alloc()
1831 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,  in skcipher_edesc_alloc()
1834 	return edesc;  in skcipher_edesc_alloc()
1839 	struct skcipher_edesc *edesc;  in skcipher_encrypt()  local
1850 	edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);  in skcipher_encrypt()
1851 	if (IS_ERR(edesc))  in skcipher_encrypt()
1852 		return PTR_ERR(edesc);  in skcipher_encrypt()
1855 	init_skcipher_job(req, edesc, true);  in skcipher_encrypt()
1858 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in skcipher_encrypt()
1859 			     desc_bytes(edesc->hw_desc), 1);  in skcipher_encrypt()
1861 	desc = edesc->hw_desc;  in skcipher_encrypt()
1867 		skcipher_unmap(jrdev, edesc, req);  in skcipher_encrypt()
1868 		kfree(edesc);  in skcipher_encrypt()
1876 	struct skcipher_edesc *edesc;  in skcipher_decrypt()  local
1887 	edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);  in skcipher_decrypt()
1888 	if (IS_ERR(edesc))  in skcipher_decrypt()
1889 		return PTR_ERR(edesc);  in skcipher_decrypt()
1892 	init_skcipher_job(req, edesc, false);  in skcipher_decrypt()
1893 	desc = edesc->hw_desc;  in skcipher_decrypt()
1896 			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,  in skcipher_decrypt()
1897 			     desc_bytes(edesc->hw_desc), 1);  in skcipher_decrypt()
1903 		skcipher_unmap(jrdev, edesc, req);  in skcipher_decrypt()
1904 		kfree(edesc);  in skcipher_decrypt()