568 			struct ahash_edesc *edesc,  in ahash_unmap()  argument
573 	if (edesc->src_nents)  in ahash_unmap()
574 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);  in ahash_unmap()
576 	if (edesc->sec4_sg_bytes)  in ahash_unmap()
577 		dma_unmap_single(dev, edesc->sec4_sg_dma,  in ahash_unmap()
578 				 edesc->sec4_sg_bytes, DMA_TO_DEVICE);  in ahash_unmap()
588 			struct ahash_edesc *edesc,  in ahash_unmap_ctx()  argument
597 	ahash_unmap(dev, edesc, req, dst_len);  in ahash_unmap_ctx()
604 	struct ahash_edesc *edesc;  in ahash_done()  local
613 	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);  in ahash_done()
617 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_done()
619 	kfree(edesc);  in ahash_done()
632 	struct ahash_edesc *edesc;  in ahash_done_bi()  local
641 	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);  in ahash_done_bi()
645 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);  in ahash_done_bi()
647 	kfree(edesc);  in ahash_done_bi()
664 	struct ahash_edesc *edesc;  in ahash_done_ctx_src()  local
673 	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);  in ahash_done_ctx_src()
677 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);  in ahash_done_ctx_src()
679 	kfree(edesc);  in ahash_done_ctx_src()
692 	struct ahash_edesc *edesc;  in ahash_done_ctx_dst()  local
701 	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);  in ahash_done_ctx_dst()
705 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);  in ahash_done_ctx_dst()
707 	kfree(edesc);  in ahash_done_ctx_dst()
729 	struct ahash_edesc *edesc;  in ahash_edesc_alloc()  local
732 	edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);  in ahash_edesc_alloc()
733 	if (!edesc) {  in ahash_edesc_alloc()
738 	init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),  in ahash_edesc_alloc()
741 	return edesc;  in ahash_edesc_alloc()
745 			       struct ahash_edesc *edesc,  in ahash_edesc_add_src()  argument
754 		struct sec4_sg_entry *sg = edesc->sec4_sg;  in ahash_edesc_add_src()
766 		edesc->sec4_sg_bytes = sgsize;  in ahash_edesc_add_src()
767 		edesc->sec4_sg_dma = src_dma;  in ahash_edesc_add_src()
774 	append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,  in ahash_edesc_add_src()
797 	struct ahash_edesc *edesc;  in ahash_update_ctx()  local
844 		edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update,  in ahash_update_ctx()
846 		if (!edesc) {  in ahash_update_ctx()
851 		edesc->src_nents = src_nents;  in ahash_update_ctx()
852 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_ctx()
855 					 edesc->sec4_sg, DMA_BIDIRECTIONAL);  in ahash_update_ctx()
859 		ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_update_ctx()
865 					   edesc->sec4_sg + sec4_sg_src_index,  in ahash_update_ctx()
868 			sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -  in ahash_update_ctx()
875 		desc = edesc->hw_desc;  in ahash_update_ctx()
877 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_ctx()
880 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_ctx()
886 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +  in ahash_update_ctx()
915 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);  in ahash_update_ctx()
916 	kfree(edesc);  in ahash_update_ctx()
932 	struct ahash_edesc *edesc;  in ahash_final_ctx()  local
939 	edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin,  in ahash_final_ctx()
941 	if (!edesc)  in ahash_final_ctx()
944 	desc = edesc->hw_desc;  in ahash_final_ctx()
946 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_final_ctx()
949 				 edesc->sec4_sg, DMA_BIDIRECTIONAL);  in ahash_final_ctx()
953 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_final_ctx()
957 	sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));  in ahash_final_ctx()
959 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_final_ctx()
961 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_final_ctx()
967 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,  in ahash_final_ctx()
981 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);  in ahash_final_ctx()
982 	kfree(edesc);  in ahash_final_ctx()
999 	struct ahash_edesc *edesc;  in ahash_finup_ctx()  local
1022 	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,  in ahash_finup_ctx()
1025 	if (!edesc) {  in ahash_finup_ctx()
1030 	desc = edesc->hw_desc;  in ahash_finup_ctx()
1032 	edesc->src_nents = src_nents;  in ahash_finup_ctx()
1035 				 edesc->sec4_sg, DMA_BIDIRECTIONAL);  in ahash_finup_ctx()
1039 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_finup_ctx()
1043 	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,  in ahash_finup_ctx()
1061 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);  in ahash_finup_ctx()
1062 	kfree(edesc);  in ahash_finup_ctx()
1077 	struct ahash_edesc *edesc;  in ahash_digest()  local
1100 	edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,  in ahash_digest()
1103 	if (!edesc) {  in ahash_digest()
1108 	edesc->src_nents = src_nents;  in ahash_digest()
1110 	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,  in ahash_digest()
1113 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1114 		kfree(edesc);  in ahash_digest()
1118 	desc = edesc->hw_desc;  in ahash_digest()
1122 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1123 		kfree(edesc);  in ahash_digest()
1135 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_digest()
1136 		kfree(edesc);  in ahash_digest()
1155 	struct ahash_edesc *edesc;  in ahash_final_no_ctx()  local
1159 	edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,  in ahash_final_no_ctx()
1161 	if (!edesc)  in ahash_final_no_ctx()
1164 	desc = edesc->hw_desc;  in ahash_final_no_ctx()
1189 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_final_no_ctx()
1190 		kfree(edesc);  in ahash_final_no_ctx()
1195 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_final_no_ctx()
1196 	kfree(edesc);  in ahash_final_no_ctx()
1217 	struct ahash_edesc *edesc;  in ahash_update_no_ctx()  local
1263 		edesc = ahash_edesc_alloc(ctx, pad_nents,  in ahash_update_no_ctx()
1267 		if (!edesc) {  in ahash_update_no_ctx()
1272 		edesc->src_nents = src_nents;  in ahash_update_no_ctx()
1273 		edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_update_no_ctx()
1275 		ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);  in ahash_update_no_ctx()
1279 		sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);  in ahash_update_no_ctx()
1287 		desc = edesc->hw_desc;  in ahash_update_no_ctx()
1289 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1292 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_no_ctx()
1298 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);  in ahash_update_no_ctx()
1331 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);  in ahash_update_no_ctx()
1332 	kfree(edesc);  in ahash_update_no_ctx()
1349 	struct ahash_edesc *edesc;  in ahash_finup_no_ctx()  local
1374 	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,  in ahash_finup_no_ctx()
1377 	if (!edesc) {  in ahash_finup_no_ctx()
1382 	desc = edesc->hw_desc;  in ahash_finup_no_ctx()
1384 	edesc->src_nents = src_nents;  in ahash_finup_no_ctx()
1385 	edesc->sec4_sg_bytes = sec4_sg_bytes;  in ahash_finup_no_ctx()
1387 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);  in ahash_finup_no_ctx()
1391 	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,  in ahash_finup_no_ctx()
1410 		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);  in ahash_finup_no_ctx()
1411 		kfree(edesc);  in ahash_finup_no_ctx()
1416 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_finup_no_ctx()
1417 	kfree(edesc);  in ahash_finup_no_ctx()
1437 	struct ahash_edesc *edesc;  in ahash_update_first()  local
1477 		edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?  in ahash_update_first()
1482 		if (!edesc) {  in ahash_update_first()
1487 		edesc->src_nents = src_nents;  in ahash_update_first()
1489 		ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,  in ahash_update_first()
1498 		desc = edesc->hw_desc;  in ahash_update_first()
1531 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);  in ahash_update_first()
1532 	kfree(edesc);  in ahash_update_first()