325 	struct talitos_edesc *edesc;  in get_request_hdr()  local
333 	edesc = container_of(request->desc, struct talitos_edesc, desc);  in get_request_hdr()
335 	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;  in get_request_hdr()
490 		struct talitos_edesc *edesc;  local
492 		edesc = container_of(priv->chan[ch].fifo[iter].desc,
495 			(edesc->buf + edesc->dma_len))->hdr;
964 			     struct talitos_edesc *edesc,  in talitos_sg_unmap()  argument
971 	unsigned int src_nents = edesc->src_nents ? : 1;  in talitos_sg_unmap()
972 	unsigned int dst_nents = edesc->dst_nents ? : 1;  in talitos_sg_unmap()
975 		dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,  in talitos_sg_unmap()
977 		sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,  in talitos_sg_unmap()
992 			    struct talitos_edesc *edesc,  in ipsec_esp_unmap()  argument
1000 	bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;  in ipsec_esp_unmap()
1001 	struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];  in ipsec_esp_unmap()
1004 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],  in ipsec_esp_unmap()
1008 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst,  in ipsec_esp_unmap()
1011 	if (edesc->dma_len)  in ipsec_esp_unmap()
1012 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in ipsec_esp_unmap()
1016 		unsigned int dst_nents = edesc->dst_nents ? : 1;  in ipsec_esp_unmap()
1033 	struct talitos_edesc *edesc;  in ipsec_esp_encrypt_done()  local
1035 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_encrypt_done()
1037 	ipsec_esp_unmap(dev, edesc, areq, true);  in ipsec_esp_encrypt_done()
1039 	dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);  in ipsec_esp_encrypt_done()
1041 	kfree(edesc);  in ipsec_esp_encrypt_done()
1053 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_swauth_done()  local
1056 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_swauth_done()
1058 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_swauth_done()
1062 		oicv = edesc->buf + edesc->dma_len;  in ipsec_esp_decrypt_swauth_done()
1068 	kfree(edesc);  in ipsec_esp_decrypt_swauth_done()
1078 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_hwauth_done()  local
1080 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_hwauth_done()
1082 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_hwauth_done()
1089 	kfree(edesc);  in ipsec_esp_decrypt_hwauth_done()
1148 			      unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map_ext()  argument
1166 		to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);  in talitos_sg_map_ext()
1170 					 &edesc->link_tbl[tbl_off]);  in talitos_sg_map_ext()
1173 		copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);  in talitos_sg_map_ext()
1176 	to_talitos_ptr(ptr, edesc->dma_link_tbl +  in talitos_sg_map_ext()
1184 			  unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map()  argument
1188 	return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,  in talitos_sg_map()
1195 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
1205 	struct talitos_desc *desc = &edesc->desc;  in ipsec_esp()
1217 	dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;  in ipsec_esp()
1222 	sg_count = edesc->src_nents ?: 1;  in ipsec_esp()
1224 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in ipsec_esp()
1232 	ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,  in ipsec_esp()
1241 	to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);  in ipsec_esp()
1256 	ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],  in ipsec_esp()
1267 		sg_count = edesc->dst_nents ? : 1;  in ipsec_esp()
1276 	ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],  in ipsec_esp()
1282 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
1296 		talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],  in ipsec_esp()
1306 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
1307 					   edesc->dma_len,  in ipsec_esp()
1312 		ipsec_esp_unmap(dev, edesc, areq, encrypt);  in ipsec_esp()
1313 		kfree(edesc);  in ipsec_esp()
1333 	struct talitos_edesc *edesc;  in talitos_edesc_alloc()  local
1398 	edesc = kmalloc(alloc_len, GFP_DMA | flags);  in talitos_edesc_alloc()
1399 	if (!edesc)  in talitos_edesc_alloc()
1402 		iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);  in talitos_edesc_alloc()
1405 	memset(&edesc->desc, 0, sizeof(edesc->desc));  in talitos_edesc_alloc()
1407 	edesc->src_nents = src_nents;  in talitos_edesc_alloc()
1408 	edesc->dst_nents = dst_nents;  in talitos_edesc_alloc()
1409 	edesc->iv_dma = iv_dma;  in talitos_edesc_alloc()
1410 	edesc->dma_len = dma_len;  in talitos_edesc_alloc()
1412 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],  in talitos_edesc_alloc()
1413 						     edesc->dma_len,  in talitos_edesc_alloc()
1416 	return edesc;  in talitos_edesc_alloc()
1438 	struct talitos_edesc *edesc;  in aead_encrypt()  local
1441 	edesc = aead_edesc_alloc(req, req->iv, 0, true);  in aead_encrypt()
1442 	if (IS_ERR(edesc))  in aead_encrypt()
1443 		return PTR_ERR(edesc);  in aead_encrypt()
1446 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_encrypt()
1448 	return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);  in aead_encrypt()
1457 	struct talitos_edesc *edesc;  in aead_decrypt()  local
1461 	edesc = aead_edesc_alloc(req, req->iv, 1, false);  in aead_decrypt()
1462 	if (IS_ERR(edesc))  in aead_decrypt()
1463 		return PTR_ERR(edesc);  in aead_decrypt()
1465 	if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&  in aead_decrypt()
1467 	    ((!edesc->src_nents && !edesc->dst_nents) ||  in aead_decrypt()
1471 		edesc->desc.hdr = ctx->desc_hdr_template |  in aead_decrypt()
1477 		return ipsec_esp(edesc, req, false,  in aead_decrypt()
1482 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in aead_decrypt()
1485 	icvdata = edesc->buf + edesc->dma_len;  in aead_decrypt()
1487 	sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,  in aead_decrypt()
1490 	return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);  in aead_decrypt()
1537 				  struct talitos_edesc *edesc,  in common_nonsnoop_unmap()  argument
1540 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_unmap()
1542 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);  in common_nonsnoop_unmap()
1543 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1545 	if (edesc->dma_len)  in common_nonsnoop_unmap()
1546 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_unmap()
1558 	struct talitos_edesc *edesc;  in ablkcipher_done()  local
1560 	edesc = container_of(desc, struct talitos_edesc, desc);  in ablkcipher_done()
1562 	common_nonsnoop_unmap(dev, edesc, areq);  in ablkcipher_done()
1565 	kfree(edesc);  in ablkcipher_done()
1570 static int common_nonsnoop(struct talitos_edesc *edesc,  in common_nonsnoop()  argument
1579 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop()
1590 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);  in common_nonsnoop()
1595 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop()
1597 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in common_nonsnoop()
1606 	sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,  in common_nonsnoop()
1613 		sg_count = edesc->dst_nents ? : 1;  in common_nonsnoop()
1618 	ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],  in common_nonsnoop()
1619 			     sg_count, 0, (edesc->src_nents + 1));  in common_nonsnoop()
1630 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop()
1631 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop()
1635 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1636 		kfree(edesc);  in common_nonsnoop()
1657 	struct talitos_edesc *edesc;  in ablkcipher_encrypt()  local
1668 	edesc = ablkcipher_edesc_alloc(areq, true);  in ablkcipher_encrypt()
1669 	if (IS_ERR(edesc))  in ablkcipher_encrypt()
1670 		return PTR_ERR(edesc);  in ablkcipher_encrypt()
1673 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in ablkcipher_encrypt()
1675 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_encrypt()
1682 	struct talitos_edesc *edesc;  in ablkcipher_decrypt()  local
1693 	edesc = ablkcipher_edesc_alloc(areq, false);  in ablkcipher_decrypt()
1694 	if (IS_ERR(edesc))  in ablkcipher_decrypt()
1695 		return PTR_ERR(edesc);  in ablkcipher_decrypt()
1697 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in ablkcipher_decrypt()
1699 	return common_nonsnoop(edesc, areq, ablkcipher_done);  in ablkcipher_decrypt()
1703 				       struct talitos_edesc *edesc,  in common_nonsnoop_hash_unmap()  argument
1709 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash_unmap()
1711 				     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash_unmap()
1713 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_hash_unmap()
1719 		talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);  in common_nonsnoop_hash_unmap()
1722 	if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))  in common_nonsnoop_hash_unmap()
1723 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],  in common_nonsnoop_hash_unmap()
1733 	if (edesc->dma_len)  in common_nonsnoop_hash_unmap()
1734 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_hash_unmap()
1737 	if (edesc->desc.next_desc)  in common_nonsnoop_hash_unmap()
1738 		dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),  in common_nonsnoop_hash_unmap()
1747 	struct talitos_edesc *edesc =  in ahash_done()  local
1756 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1758 	kfree(edesc);  in ahash_done()
1768 			       struct talitos_edesc *edesc,  in talitos_handle_buggy_hash()  argument
1779 	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;  in talitos_handle_buggy_hash()
1784 static int common_nonsnoop_hash(struct talitos_edesc *edesc,  in common_nonsnoop_hash()  argument
1794 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash()
1822 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop_hash()
1824 		sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);  in common_nonsnoop_hash()
1836 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1858 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);  in common_nonsnoop_hash()
1862 					     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash()
1882 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1899 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop_hash()
1900 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop_hash()
1904 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1905 		kfree(edesc);  in common_nonsnoop_hash()
1984 	struct talitos_edesc *edesc;  in ahash_process_req()  local
2066 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
2067 	if (IS_ERR(edesc))  in ahash_process_req()
2068 		return PTR_ERR(edesc);  in ahash_process_req()
2070 	edesc->desc.hdr = ctx->desc_hdr_template;  in ahash_process_req()
2074 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;  in ahash_process_req()
2076 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;  in ahash_process_req()
2080 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;  in ahash_process_req()
2086 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;  in ahash_process_req()
2088 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);  in ahash_process_req()