54 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)  in safexcel_queued_len()  argument
56 	return req->len - req->processed;  in safexcel_queued_len()
79 				     struct safexcel_ahash_req *req,  in safexcel_context_control()  argument
92 	if (!req->processed) {  in safexcel_context_control()
94 		if (req->finish) {  in safexcel_context_control()
112 	memcpy(ctx->base.ctxr->data, req->state, req->state_sz);  in safexcel_context_control()
114 	if (req->finish) {  in safexcel_context_control()
116 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||  in safexcel_context_control()
117 		    req->hmac_zlen || (req->processed != req->block_sz)) {  in safexcel_context_control()
118 			count = req->processed / EIP197_COUNTER_BLOCK_SIZE;  in safexcel_context_control()
132 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||  in safexcel_context_control()
134 		    req->hmac_zlen ||  in safexcel_context_control()
136 		    (req->processed != req->block_sz)) {  in safexcel_context_control()
139 				CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |  in safexcel_context_control()
143 			if (req->hmac_zlen)  in safexcel_context_control()
148 			ctx->base.ctxr->data[req->state_sz >> 2] =  in safexcel_context_control()
150 			req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_context_control()
153 			req->hmac_zlen = false;  in safexcel_context_control()
156 			memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),  in safexcel_context_control()
157 			       ctx->opad, req->state_sz);  in safexcel_context_control()
161 				CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |  in safexcel_context_control()
167 			CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |  in safexcel_context_control()
258 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send_req()  local
267 	queued = len = safexcel_queued_len(req);  in safexcel_ahash_send_req()
273 	if (!req->finish && !req->last_req) {  in safexcel_ahash_send_req()
286 				   req->cache_next, extra,  in safexcel_ahash_send_req()
301 		req->cache_dma = dma_map_single(priv->dev, req->cache,  in safexcel_ahash_send_req()
303 		if (dma_mapping_error(priv->dev, req->cache_dma))  in safexcel_ahash_send_req()
306 		req->cache_sz = cache_len;  in safexcel_ahash_send_req()
309 						 req->cache_dma, cache_len, len,  in safexcel_ahash_send_req()
327 	req->nents = dma_map_sg(priv->dev, areq->src,  in safexcel_ahash_send_req()
331 	if (!req->nents) {  in safexcel_ahash_send_req()
336 	for_each_sg(areq->src, sg, req->nents, i) {  in safexcel_ahash_send_req()
363 	safexcel_context_control(ctx, req, first_cdesc);  in safexcel_ahash_send_req()
366 	safexcel_hash_token(first_cdesc, len, req->state_sz);  in safexcel_ahash_send_req()
368 	req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz,  in safexcel_ahash_send_req()
370 	if (dma_mapping_error(priv->dev, req->result_dma)) {  in safexcel_ahash_send_req()
376 	rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,  in safexcel_ahash_send_req()
377 				   req->state_sz);  in safexcel_ahash_send_req()
385 	req->processed += len;  in safexcel_ahash_send_req()
392 	dma_unmap_single(priv->dev, req->result_dma, req->state_sz,  in safexcel_ahash_send_req()
395 	dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);  in safexcel_ahash_send_req()
400 	if (req->cache_dma) {  in safexcel_ahash_send_req()
401 		dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,  in safexcel_ahash_send_req()
403 		req->cache_dma = 0;  in safexcel_ahash_send_req()
404 		req->cache_sz = 0;  in safexcel_ahash_send_req()
465 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_handle_result()  local
468 	BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);  in safexcel_handle_result()
470 	if (req->needs_inv) {  in safexcel_handle_result()
471 		req->needs_inv = false;  in safexcel_handle_result()
504 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send()  local
507 	if (req->needs_inv)  in safexcel_ahash_send()
519 	EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);  in safexcel_ahash_exit_inv()
520 	struct safexcel_ahash_req *rctx = ahash_request_ctx(req);  in safexcel_ahash_exit_inv()
524 	memset(req, 0, EIP197_AHASH_REQ_SIZE);  in safexcel_ahash_exit_inv()
528 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in safexcel_ahash_exit_inv()
531 	ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));  in safexcel_ahash_exit_inv()
532 	ctx = crypto_tfm_ctx(req->base.tfm);  in safexcel_ahash_exit_inv()
537 	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);  in safexcel_ahash_exit_inv()
559 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_cache()  local
565 	cache_len = safexcel_queued_len(req);  in safexcel_ahash_cache()
573 				   req->cache + cache_len,  in safexcel_ahash_cache()
585 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_enqueue()  local
589 	req->needs_inv = false;  in safexcel_ahash_enqueue()
593 		    req->processed &&  in safexcel_ahash_enqueue()
595 		     (req->finish &&  in safexcel_ahash_enqueue()
596 		      (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)) ||  in safexcel_ahash_enqueue()
598 		     memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||  in safexcel_ahash_enqueue()
600 		     (req->finish && (req->processed != req->block_sz)) ||  in safexcel_ahash_enqueue()
602 		     (req->finish &&  in safexcel_ahash_enqueue()
603 		      memcmp(ctx->base.ctxr->data + (req->state_sz>>2),  in safexcel_ahash_enqueue()
604 			     ctx->opad, req->state_sz))))  in safexcel_ahash_enqueue()
615 			req->needs_inv = true;  in safexcel_ahash_enqueue()
640 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_update()  local
651 	req->len += areq->nbytes;  in safexcel_ahash_update()
657 	if ((ret && !req->finish) || req->last_req)  in safexcel_ahash_update()
665 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_final()  local
668 	req->finish = true;  in safexcel_ahash_final()
670 	if (unlikely(!req->len && !areq->nbytes)) {  in safexcel_ahash_final()
696 	} else if (unlikely(req->hmac &&  in safexcel_ahash_final()
697 			    (req->len == req->block_sz) &&  in safexcel_ahash_final()
707 		memset(req->cache, 0, req->block_sz);  in safexcel_ahash_final()
709 		req->cache[0] = 0x80;  in safexcel_ahash_final()
711 		if (req->len_is_le) {  in safexcel_ahash_final()
713 			req->cache[req->block_sz-8] = (req->block_sz << 3) &  in safexcel_ahash_final()
715 			req->cache[req->block_sz-7] = (req->block_sz >> 5);  in safexcel_ahash_final()
718 			req->cache[req->block_sz-2] = (req->block_sz >> 5);  in safexcel_ahash_final()
719 			req->cache[req->block_sz-1] = (req->block_sz << 3) &  in safexcel_ahash_final()
723 		req->len += req->block_sz; /* plus 1 hash block */  in safexcel_ahash_final()
726 		req->hmac_zlen = true;  in safexcel_ahash_final()
729 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_ahash_final()
730 	} else if (req->hmac) {  in safexcel_ahash_final()
732 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_ahash_final()
740 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_finup()  local
742 	req->finish = true;  in safexcel_ahash_finup()
750 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_export()  local
753 	export->len = req->len;  in safexcel_ahash_export()
754 	export->processed = req->processed;  in safexcel_ahash_export()
756 	export->digest = req->digest;  in safexcel_ahash_export()
758 	memcpy(export->state, req->state, req->state_sz);  in safexcel_ahash_export()
759 	memcpy(export->cache, req->cache, HASH_CACHE_SIZE);  in safexcel_ahash_export()
766 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_import()  local
774 	req->len = export->len;  in safexcel_ahash_import()
775 	req->processed = export->processed;  in safexcel_ahash_import()
777 	req->digest = export->digest;  in safexcel_ahash_import()
779 	memcpy(req->cache, export->cache, HASH_CACHE_SIZE);  in safexcel_ahash_import()
780 	memcpy(req->state, export->state, req->state_sz);  in safexcel_ahash_import()
804 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha1_init()  local
806 	memset(req, 0, sizeof(*req));  in safexcel_sha1_init()
809 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha1_init()
810 	req->state_sz = SHA1_DIGEST_SIZE;  in safexcel_sha1_init()
811 	req->block_sz = SHA1_BLOCK_SIZE;  in safexcel_sha1_init()
879 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha1_init()  local
881 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha1_init()
884 	memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE);  in safexcel_hmac_sha1_init()
886 	req->len	= SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
887 	req->processed	= SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
890 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha1_init()
891 	req->state_sz = SHA1_DIGEST_SIZE;  in safexcel_hmac_sha1_init()
892 	req->block_sz = SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
893 	req->hmac = true;  in safexcel_hmac_sha1_init()
913 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)  in safexcel_ahash_complete()  argument
915 	struct safexcel_ahash_result *result = req->data;  in safexcel_ahash_complete()
977 	struct safexcel_ahash_req *req;  in safexcel_hmac_init_iv()  local
991 	req = ahash_request_ctx(areq);  in safexcel_hmac_init_iv()
992 	req->hmac = true;  in safexcel_hmac_init_iv()
993 	req->last_req = true;  in safexcel_hmac_init_iv()
1121 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha256_init()  local
1123 	memset(req, 0, sizeof(*req));  in safexcel_sha256_init()
1126 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha256_init()
1127 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_sha256_init()
1128 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_sha256_init()
1176 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha224_init()  local
1178 	memset(req, 0, sizeof(*req));  in safexcel_sha224_init()
1181 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha224_init()
1182 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_sha224_init()
1183 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_sha224_init()
1238 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha224_init()  local
1240 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha224_init()
1243 	memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);  in safexcel_hmac_sha224_init()
1245 	req->len	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1246 	req->processed	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1249 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha224_init()
1250 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha224_init()
1251 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1252 	req->hmac = true;  in safexcel_hmac_sha224_init()
1308 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha256_init()  local
1310 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha256_init()
1313 	memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);  in safexcel_hmac_sha256_init()
1315 	req->len	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1316 	req->processed	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1319 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha256_init()
1320 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha256_init()
1321 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1322 	req->hmac = true;  in safexcel_hmac_sha256_init()
1371 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha512_init()  local
1373 	memset(req, 0, sizeof(*req));  in safexcel_sha512_init()
1376 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha512_init()
1377 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_sha512_init()
1378 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_sha512_init()
1426 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha384_init()  local
1428 	memset(req, 0, sizeof(*req));  in safexcel_sha384_init()
1431 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha384_init()
1432 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_sha384_init()
1433 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_sha384_init()
1488 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha512_init()  local
1490 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha512_init()
1493 	memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);  in safexcel_hmac_sha512_init()
1495 	req->len	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1496 	req->processed	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1499 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha512_init()
1500 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha512_init()
1501 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1502 	req->hmac = true;  in safexcel_hmac_sha512_init()
1558 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha384_init()  local
1560 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha384_init()
1563 	memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);  in safexcel_hmac_sha384_init()
1565 	req->len	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1566 	req->processed	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1569 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha384_init()
1570 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha384_init()
1571 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1572 	req->hmac = true;  in safexcel_hmac_sha384_init()
1621 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_md5_init()  local
1623 	memset(req, 0, sizeof(*req));  in safexcel_md5_init()
1626 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_md5_init()
1627 	req->state_sz = MD5_DIGEST_SIZE;  in safexcel_md5_init()
1628 	req->block_sz = MD5_HMAC_BLOCK_SIZE;  in safexcel_md5_init()
1676 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_md5_init()  local
1678 	memset(req, 0, sizeof(*req));  in safexcel_hmac_md5_init()
1681 	memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE);  in safexcel_hmac_md5_init()
1683 	req->len	= MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1684 	req->processed	= MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1687 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_md5_init()
1688 	req->state_sz = MD5_DIGEST_SIZE;  in safexcel_hmac_md5_init()
1689 	req->block_sz = MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1690 	req->len_is_le = true; /* MD5 is little endian! ... */  in safexcel_hmac_md5_init()
1691 	req->hmac = true;  in safexcel_hmac_md5_init()