326 	struct talitos_edesc *edesc;  in get_request_hdr()  local
334 	edesc = container_of(request->desc, struct talitos_edesc, desc);  in get_request_hdr()
336 	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;  in get_request_hdr()
491 		struct talitos_edesc *edesc;  local
493 		edesc = container_of(priv->chan[ch].fifo[iter].desc,
496 			(edesc->buf + edesc->dma_len))->hdr;
960 			     struct talitos_edesc *edesc,  in talitos_sg_unmap()  argument
967 	unsigned int src_nents = edesc->src_nents ? : 1;  in talitos_sg_unmap()
968 	unsigned int dst_nents = edesc->dst_nents ? : 1;  in talitos_sg_unmap()
971 		dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,  in talitos_sg_unmap()
973 		sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,  in talitos_sg_unmap()
988 			    struct talitos_edesc *edesc,  in ipsec_esp_unmap()  argument
996 	bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;  in ipsec_esp_unmap()
997 	struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];  in ipsec_esp_unmap()
1000 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],  in ipsec_esp_unmap()
1004 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst,  in ipsec_esp_unmap()
1007 	if (edesc->dma_len)  in ipsec_esp_unmap()
1008 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in ipsec_esp_unmap()
1012 		unsigned int dst_nents = edesc->dst_nents ? : 1;  in ipsec_esp_unmap()
1029 	struct talitos_edesc *edesc;  in ipsec_esp_encrypt_done()  local
1031 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_encrypt_done()
1033 	ipsec_esp_unmap(dev, edesc, areq, true);  in ipsec_esp_encrypt_done()
1035 	dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);  in ipsec_esp_encrypt_done()
1037 	kfree(edesc);  in ipsec_esp_encrypt_done()
1049 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_swauth_done()  local
1052 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_swauth_done()
1054 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_swauth_done()
1058 		oicv = edesc->buf + edesc->dma_len;  in ipsec_esp_decrypt_swauth_done()
1064 	kfree(edesc);  in ipsec_esp_decrypt_swauth_done()
1074 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_hwauth_done()  local
1076 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_hwauth_done()
1078 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_hwauth_done()
1085 	kfree(edesc);  in ipsec_esp_decrypt_hwauth_done()
1145 			      unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map_ext()  argument
1164 		to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1);  in talitos_sg_map_ext()
1168 					 &edesc->link_tbl[tbl_off], align);  in talitos_sg_map_ext()
1171 		copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);  in talitos_sg_map_ext()
1174 	to_talitos_ptr(ptr, edesc->dma_link_tbl +  in talitos_sg_map_ext()
1182 			  unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map()  argument
1186 	return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,  in talitos_sg_map()
1193 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
1203 	struct talitos_desc *desc = &edesc->desc;  in ipsec_esp()
1215 	dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;  in ipsec_esp()
1220 	sg_count = edesc->src_nents ?: 1;  in ipsec_esp()
1222 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in ipsec_esp()
1230 	ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,  in ipsec_esp()
1239 	to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);  in ipsec_esp()
1254 	ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],  in ipsec_esp()
1265 		sg_count = edesc->dst_nents ? : 1;  in ipsec_esp()
1274 	ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],  in ipsec_esp()
1280 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
1294 		talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],  in ipsec_esp()
1304 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
1305 					   edesc->dma_len,  in ipsec_esp()
1310 		ipsec_esp_unmap(dev, edesc, areq, encrypt);  in ipsec_esp()
1311 		kfree(edesc);  in ipsec_esp()
1331 	struct talitos_edesc *edesc;  in talitos_edesc_alloc()  local
1396 	edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);  in talitos_edesc_alloc()
1397 	if (!edesc)  in talitos_edesc_alloc()
1400 		iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);  in talitos_edesc_alloc()
1403 	memset(&edesc->desc, 0, sizeof(edesc->desc));  in talitos_edesc_alloc()
1405 	edesc->src_nents = src_nents;  in talitos_edesc_alloc()
1406 	edesc->dst_nents = dst_nents;  in talitos_edesc_alloc()
1407 	edesc->iv_dma = iv_dma;  in talitos_edesc_alloc()
1408 	edesc->dma_len = dma_len;  in talitos_edesc_alloc()
1410 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],  in talitos_edesc_alloc()
1411 						     edesc->dma_len,  in talitos_edesc_alloc()
1414 	return edesc;  in talitos_edesc_alloc()
1436 	struct talitos_edesc *edesc;  in aead_encrypt()  local
1439 	edesc = aead_edesc_alloc(req, req->iv, 0, true);  in aead_encrypt()
1440 	if (IS_ERR(edesc))  in aead_encrypt()
1441 		return PTR_ERR(edesc);  in aead_encrypt()
1444 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_encrypt()
1446 	return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);  in aead_encrypt()
1455 	struct talitos_edesc *edesc;  in aead_decrypt()  local
1459 	edesc = aead_edesc_alloc(req, req->iv, 1, false);  in aead_decrypt()
1460 	if (IS_ERR(edesc))  in aead_decrypt()
1461 		return PTR_ERR(edesc);  in aead_decrypt()
1463 	if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&  in aead_decrypt()
1465 	    ((!edesc->src_nents && !edesc->dst_nents) ||  in aead_decrypt()
1469 		edesc->desc.hdr = ctx->desc_hdr_template |  in aead_decrypt()
1475 		return ipsec_esp(edesc, req, false,  in aead_decrypt()
1480 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in aead_decrypt()
1483 	icvdata = edesc->buf + edesc->dma_len;  in aead_decrypt()
1485 	sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,  in aead_decrypt()
1488 	return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);  in aead_decrypt()
1533 				  struct talitos_edesc *edesc,  in common_nonsnoop_unmap()  argument
1536 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_unmap()
1538 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);  in common_nonsnoop_unmap()
1539 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1541 	if (edesc->dma_len)  in common_nonsnoop_unmap()
1542 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_unmap()
1554 	struct talitos_edesc *edesc;  in skcipher_done()  local
1556 	edesc = container_of(desc, struct talitos_edesc, desc);  in skcipher_done()
1558 	common_nonsnoop_unmap(dev, edesc, areq);  in skcipher_done()
1561 	kfree(edesc);  in skcipher_done()
1566 static int common_nonsnoop(struct talitos_edesc *edesc,  in common_nonsnoop()  argument
1575 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop()
1588 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);  in common_nonsnoop()
1593 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop()
1595 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in common_nonsnoop()
1604 	sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3],  in common_nonsnoop()
1611 		sg_count = edesc->dst_nents ? : 1;  in common_nonsnoop()
1616 	ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],  in common_nonsnoop()
1617 			     sg_count, 0, (edesc->src_nents + 1));  in common_nonsnoop()
1628 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop()
1629 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop()
1633 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1634 		kfree(edesc);  in common_nonsnoop()
1655 	struct talitos_edesc *edesc;  in skcipher_encrypt()  local
1666 	edesc = skcipher_edesc_alloc(areq, true);  in skcipher_encrypt()
1667 	if (IS_ERR(edesc))  in skcipher_encrypt()
1668 		return PTR_ERR(edesc);  in skcipher_encrypt()
1671 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in skcipher_encrypt()
1673 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_encrypt()
1680 	struct talitos_edesc *edesc;  in skcipher_decrypt()  local
1691 	edesc = skcipher_edesc_alloc(areq, false);  in skcipher_decrypt()
1692 	if (IS_ERR(edesc))  in skcipher_decrypt()
1693 		return PTR_ERR(edesc);  in skcipher_decrypt()
1695 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in skcipher_decrypt()
1697 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_decrypt()
1701 				       struct talitos_edesc *edesc,  in common_nonsnoop_hash_unmap()  argument
1708 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash_unmap()
1710 				     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash_unmap()
1721 		talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);  in common_nonsnoop_hash_unmap()
1735 	if (edesc->dma_len)  in common_nonsnoop_hash_unmap()
1736 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_hash_unmap()
1749 	struct talitos_edesc *edesc =  in ahash_done()  local
1758 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1760 	kfree(edesc);  in ahash_done()
1770 			       struct talitos_edesc *edesc,  in talitos_handle_buggy_hash()  argument
1781 	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;  in talitos_handle_buggy_hash()
1786 static int common_nonsnoop_hash(struct talitos_edesc *edesc,  in common_nonsnoop_hash()  argument
1796 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash()
1824 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop_hash()
1826 		sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);  in common_nonsnoop_hash()
1838 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1860 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);  in common_nonsnoop_hash()
1864 					     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash()
1884 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1901 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop_hash()
1902 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop_hash()
1906 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1907 		kfree(edesc);  in common_nonsnoop_hash()
1986 	struct talitos_edesc *edesc;  in ahash_process_req()  local
2068 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
2069 	if (IS_ERR(edesc))  in ahash_process_req()
2070 		return PTR_ERR(edesc);  in ahash_process_req()
2072 	edesc->desc.hdr = ctx->desc_hdr_template;  in ahash_process_req()
2076 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;  in ahash_process_req()
2078 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;  in ahash_process_req()
2082 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;  in ahash_process_req()
2088 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;  in ahash_process_req()
2090 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);  in ahash_process_req()