325 	struct talitos_edesc *edesc;  in get_request_hdr()  local
333 	edesc = container_of(request->desc, struct talitos_edesc, desc);  in get_request_hdr()
335 	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;  in get_request_hdr()
490 		struct talitos_edesc *edesc;  local
492 		edesc = container_of(priv->chan[ch].fifo[iter].desc,
495 			(edesc->buf + edesc->dma_len))->hdr;
959 			     struct talitos_edesc *edesc,  in talitos_sg_unmap()  argument
966 	unsigned int src_nents = edesc->src_nents ? : 1;  in talitos_sg_unmap()
967 	unsigned int dst_nents = edesc->dst_nents ? : 1;  in talitos_sg_unmap()
970 		dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,  in talitos_sg_unmap()
972 		sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,  in talitos_sg_unmap()
987 			    struct talitos_edesc *edesc,  in ipsec_esp_unmap()  argument
995 	bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;  in ipsec_esp_unmap()
996 	struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];  in ipsec_esp_unmap()
999 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],  in ipsec_esp_unmap()
1003 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst,  in ipsec_esp_unmap()
1006 	if (edesc->dma_len)  in ipsec_esp_unmap()
1007 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in ipsec_esp_unmap()
1011 		unsigned int dst_nents = edesc->dst_nents ? : 1;  in ipsec_esp_unmap()
1028 	struct talitos_edesc *edesc;  in ipsec_esp_encrypt_done()  local
1030 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_encrypt_done()
1032 	ipsec_esp_unmap(dev, edesc, areq, true);  in ipsec_esp_encrypt_done()
1034 	dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);  in ipsec_esp_encrypt_done()
1036 	kfree(edesc);  in ipsec_esp_encrypt_done()
1048 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_swauth_done()  local
1051 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_swauth_done()
1053 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_swauth_done()
1057 		oicv = edesc->buf + edesc->dma_len;  in ipsec_esp_decrypt_swauth_done()
1063 	kfree(edesc);  in ipsec_esp_decrypt_swauth_done()
1073 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_hwauth_done()  local
1075 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_hwauth_done()
1077 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_hwauth_done()
1084 	kfree(edesc);  in ipsec_esp_decrypt_hwauth_done()
1143 			      unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map_ext()  argument
1161 		to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);  in talitos_sg_map_ext()
1165 					 &edesc->link_tbl[tbl_off]);  in talitos_sg_map_ext()
1168 		copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);  in talitos_sg_map_ext()
1171 	to_talitos_ptr(ptr, edesc->dma_link_tbl +  in talitos_sg_map_ext()
1179 			  unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map()  argument
1183 	return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,  in talitos_sg_map()
1190 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
1200 	struct talitos_desc *desc = &edesc->desc;  in ipsec_esp()
1212 	dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;  in ipsec_esp()
1217 	sg_count = edesc->src_nents ?: 1;  in ipsec_esp()
1219 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in ipsec_esp()
1227 	ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,  in ipsec_esp()
1236 	to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);  in ipsec_esp()
1251 	ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],  in ipsec_esp()
1262 		sg_count = edesc->dst_nents ? : 1;  in ipsec_esp()
1271 	ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],  in ipsec_esp()
1277 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
1291 		talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],  in ipsec_esp()
1301 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
1302 					   edesc->dma_len,  in ipsec_esp()
1307 		ipsec_esp_unmap(dev, edesc, areq, encrypt);  in ipsec_esp()
1308 		kfree(edesc);  in ipsec_esp()
1328 	struct talitos_edesc *edesc;  in talitos_edesc_alloc()  local
1393 	edesc = kmalloc(alloc_len, GFP_DMA | flags);  in talitos_edesc_alloc()
1394 	if (!edesc)  in talitos_edesc_alloc()
1397 		iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);  in talitos_edesc_alloc()
1400 	memset(&edesc->desc, 0, sizeof(edesc->desc));  in talitos_edesc_alloc()
1402 	edesc->src_nents = src_nents;  in talitos_edesc_alloc()
1403 	edesc->dst_nents = dst_nents;  in talitos_edesc_alloc()
1404 	edesc->iv_dma = iv_dma;  in talitos_edesc_alloc()
1405 	edesc->dma_len = dma_len;  in talitos_edesc_alloc()
1407 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],  in talitos_edesc_alloc()
1408 						     edesc->dma_len,  in talitos_edesc_alloc()
1411 	return edesc;  in talitos_edesc_alloc()
1433 	struct talitos_edesc *edesc;  in aead_encrypt()  local
1436 	edesc = aead_edesc_alloc(req, req->iv, 0, true);  in aead_encrypt()
1437 	if (IS_ERR(edesc))  in aead_encrypt()
1438 		return PTR_ERR(edesc);  in aead_encrypt()
1441 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_encrypt()
1443 	return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);  in aead_encrypt()
1452 	struct talitos_edesc *edesc;  in aead_decrypt()  local
1456 	edesc = aead_edesc_alloc(req, req->iv, 1, false);  in aead_decrypt()
1457 	if (IS_ERR(edesc))  in aead_decrypt()
1458 		return PTR_ERR(edesc);  in aead_decrypt()
1460 	if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&  in aead_decrypt()
1462 	    ((!edesc->src_nents && !edesc->dst_nents) ||  in aead_decrypt()
1466 		edesc->desc.hdr = ctx->desc_hdr_template |  in aead_decrypt()
1472 		return ipsec_esp(edesc, req, false,  in aead_decrypt()
1477 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in aead_decrypt()
1480 	icvdata = edesc->buf + edesc->dma_len;  in aead_decrypt()
1482 	sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,  in aead_decrypt()
1485 	return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);  in aead_decrypt()
1530 				  struct talitos_edesc *edesc,  in common_nonsnoop_unmap()  argument
1533 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_unmap()
1535 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);  in common_nonsnoop_unmap()
1536 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1538 	if (edesc->dma_len)  in common_nonsnoop_unmap()
1539 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_unmap()
1551 	struct talitos_edesc *edesc;  in skcipher_done()  local
1553 	edesc = container_of(desc, struct talitos_edesc, desc);  in skcipher_done()
1555 	common_nonsnoop_unmap(dev, edesc, areq);  in skcipher_done()
1558 	kfree(edesc);  in skcipher_done()
1563 static int common_nonsnoop(struct talitos_edesc *edesc,  in common_nonsnoop()  argument
1572 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop()
1583 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);  in common_nonsnoop()
1588 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop()
1590 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in common_nonsnoop()
1599 	sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,  in common_nonsnoop()
1606 		sg_count = edesc->dst_nents ? : 1;  in common_nonsnoop()
1611 	ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],  in common_nonsnoop()
1612 			     sg_count, 0, (edesc->src_nents + 1));  in common_nonsnoop()
1623 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop()
1624 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop()
1628 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1629 		kfree(edesc);  in common_nonsnoop()
1650 	struct talitos_edesc *edesc;  in skcipher_encrypt()  local
1661 	edesc = skcipher_edesc_alloc(areq, true);  in skcipher_encrypt()
1662 	if (IS_ERR(edesc))  in skcipher_encrypt()
1663 		return PTR_ERR(edesc);  in skcipher_encrypt()
1666 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in skcipher_encrypt()
1668 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_encrypt()
1675 	struct talitos_edesc *edesc;  in skcipher_decrypt()  local
1686 	edesc = skcipher_edesc_alloc(areq, false);  in skcipher_decrypt()
1687 	if (IS_ERR(edesc))  in skcipher_decrypt()
1688 		return PTR_ERR(edesc);  in skcipher_decrypt()
1690 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in skcipher_decrypt()
1692 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_decrypt()
1696 				       struct talitos_edesc *edesc,  in common_nonsnoop_hash_unmap()  argument
1703 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash_unmap()
1705 				     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash_unmap()
1707 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_hash_unmap()
1716 		talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);  in common_nonsnoop_hash_unmap()
1719 	if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))  in common_nonsnoop_hash_unmap()
1720 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],  in common_nonsnoop_hash_unmap()
1730 	if (edesc->dma_len)  in common_nonsnoop_hash_unmap()
1731 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_hash_unmap()
1734 	if (edesc->desc.next_desc)  in common_nonsnoop_hash_unmap()
1735 		dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),  in common_nonsnoop_hash_unmap()
1744 	struct talitos_edesc *edesc =  in ahash_done()  local
1753 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1755 	kfree(edesc);  in ahash_done()
1765 			       struct talitos_edesc *edesc,  in talitos_handle_buggy_hash()  argument
1776 	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;  in talitos_handle_buggy_hash()
1781 static int common_nonsnoop_hash(struct talitos_edesc *edesc,  in common_nonsnoop_hash()  argument
1791 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash()
1819 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop_hash()
1821 		sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);  in common_nonsnoop_hash()
1833 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1855 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);  in common_nonsnoop_hash()
1859 					     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash()
1879 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1896 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop_hash()
1897 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop_hash()
1901 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1902 		kfree(edesc);  in common_nonsnoop_hash()
1981 	struct talitos_edesc *edesc;  in ahash_process_req()  local
2063 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
2064 	if (IS_ERR(edesc))  in ahash_process_req()
2065 		return PTR_ERR(edesc);  in ahash_process_req()
2067 	edesc->desc.hdr = ctx->desc_hdr_template;  in ahash_process_req()
2071 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;  in ahash_process_req()
2073 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;  in ahash_process_req()
2077 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;  in ahash_process_req()
2083 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;  in ahash_process_req()
2085 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);  in ahash_process_req()