28 			    struct ahash_request *req)  in mv_cesa_ahash_req_iter_init()  argument
30 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_iter_init()
31 	unsigned int len = req->nbytes + creq->cache_ptr;  in mv_cesa_ahash_req_iter_init()
37 	mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE);  in mv_cesa_ahash_req_iter_init()
50 mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_dma_req *req, gfp_t flags)  in mv_cesa_ahash_dma_alloc_cache()  argument
52 	req->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags,  in mv_cesa_ahash_dma_alloc_cache()
53 				    &req->cache_dma);  in mv_cesa_ahash_dma_alloc_cache()
54 	if (!req->cache)  in mv_cesa_ahash_dma_alloc_cache()
61 mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_dma_req *req)  in mv_cesa_ahash_dma_free_cache()  argument
63 	if (!req->cache)  in mv_cesa_ahash_dma_free_cache()
66 	dma_pool_free(cesa_dev->dma->cache_pool, req->cache,  in mv_cesa_ahash_dma_free_cache()
67 		      req->cache_dma);  in mv_cesa_ahash_dma_free_cache()
70 static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req,  in mv_cesa_ahash_dma_alloc_padding()  argument
73 	if (req->padding)  in mv_cesa_ahash_dma_alloc_padding()
76 	req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags,  in mv_cesa_ahash_dma_alloc_padding()
77 				      &req->padding_dma);  in mv_cesa_ahash_dma_alloc_padding()
78 	if (!req->padding)  in mv_cesa_ahash_dma_alloc_padding()
84 static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req)  in mv_cesa_ahash_dma_free_padding()  argument
86 	if (!req->padding)  in mv_cesa_ahash_dma_free_padding()
89 	dma_pool_free(cesa_dev->dma->padding_pool, req->padding,  in mv_cesa_ahash_dma_free_padding()
90 		      req->padding_dma);  in mv_cesa_ahash_dma_free_padding()
91 	req->padding = NULL;  in mv_cesa_ahash_dma_free_padding()
94 static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_last_cleanup()  argument
96 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_last_cleanup()
98 	mv_cesa_ahash_dma_free_padding(&creq->req.dma);  in mv_cesa_ahash_dma_last_cleanup()
101 static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_cleanup()  argument
103 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_cleanup()
105 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_cleanup()
106 	mv_cesa_ahash_dma_free_cache(&creq->req.dma);  in mv_cesa_ahash_dma_cleanup()
110 static inline void mv_cesa_ahash_cleanup(struct ahash_request *req)  in mv_cesa_ahash_cleanup()  argument
112 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cleanup()
115 		mv_cesa_ahash_dma_cleanup(req);  in mv_cesa_ahash_cleanup()
118 static void mv_cesa_ahash_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_last_cleanup()  argument
120 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_last_cleanup()
123 		mv_cesa_ahash_dma_last_cleanup(req);  in mv_cesa_ahash_last_cleanup()
158 static void mv_cesa_ahash_std_step(struct ahash_request *req)  in mv_cesa_ahash_std_step()  argument
160 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_step()
161 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_step()
179 		digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));  in mv_cesa_ahash_std_step()
194 	len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset,  in mv_cesa_ahash_std_step()
204 			engine, req->src, creq->src_nents,  in mv_cesa_ahash_std_step()
212 	if (creq->last_req && sreq->offset == req->nbytes &&  in mv_cesa_ahash_std_step()
284 static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status)  in mv_cesa_ahash_std_process()  argument
286 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_process()
287 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_process()
289 	if (sreq->offset < (req->nbytes - creq->cache_ptr))  in mv_cesa_ahash_std_process()
295 static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req)  in mv_cesa_ahash_dma_prepare()  argument
297 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_prepare()
303 static void mv_cesa_ahash_std_prepare(struct ahash_request *req)  in mv_cesa_ahash_std_prepare()  argument
305 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_prepare()
306 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_prepare()
311 static void mv_cesa_ahash_dma_step(struct ahash_request *req)  in mv_cesa_ahash_dma_step()  argument
313 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_step()
330 static void mv_cesa_ahash_step(struct crypto_async_request *req)  in mv_cesa_ahash_step()  argument
332 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_step()
341 static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status)  in mv_cesa_ahash_process()  argument
343 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_process()
352 static void mv_cesa_ahash_complete(struct crypto_async_request *req)  in mv_cesa_ahash_complete()  argument
354 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_complete()
402 static void mv_cesa_ahash_prepare(struct crypto_async_request *req,  in mv_cesa_ahash_prepare()  argument
405 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_prepare()
416 static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req)  in mv_cesa_ahash_req_cleanup()  argument
418 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_req_cleanup()
440 static void mv_cesa_ahash_init(struct ahash_request *req,  in mv_cesa_ahash_init()  argument
443 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_init()
469 static bool mv_cesa_ahash_cache_req(struct ahash_request *req)  in mv_cesa_ahash_cache_req()  argument
471 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cache_req()
474 	if (creq->cache_ptr + req->nbytes < CESA_MAX_HASH_BLOCK_SIZE &&  in mv_cesa_ahash_cache_req()
478 		if (!req->nbytes)  in mv_cesa_ahash_cache_req()
481 		sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_cache_req()
483 				   req->nbytes, 0);  in mv_cesa_ahash_cache_req()
485 		creq->cache_ptr += req->nbytes;  in mv_cesa_ahash_cache_req()
524 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_add_cache()
550 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_last_req()
627 static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)  in mv_cesa_ahash_dma_req_init()  argument
629 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_req_init()
630 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  in mv_cesa_ahash_dma_req_init()
647 		ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents,  in mv_cesa_ahash_dma_req_init()
656 	mv_cesa_ahash_req_iter_init(&iter, req);  in mv_cesa_ahash_dma_req_init()
730 		creq->cache_ptr = req->nbytes + creq->cache_ptr -  in mv_cesa_ahash_dma_req_init()
753 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_req_init()
756 	mv_cesa_ahash_last_cleanup(req);  in mv_cesa_ahash_dma_req_init()
761 static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)  in mv_cesa_ahash_req_init()  argument
763 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_init()
765 	creq->src_nents = sg_nents_for_len(req->src, req->nbytes);  in mv_cesa_ahash_req_init()
771 	*cached = mv_cesa_ahash_cache_req(req);  in mv_cesa_ahash_req_init()
777 		return mv_cesa_ahash_dma_req_init(req);  in mv_cesa_ahash_req_init()
782 static int mv_cesa_ahash_queue_req(struct ahash_request *req)  in mv_cesa_ahash_queue_req()  argument
784 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_queue_req()
789 	ret = mv_cesa_ahash_req_init(req, &cached);  in mv_cesa_ahash_queue_req()
796 	engine = mv_cesa_select_engine(req->nbytes);  in mv_cesa_ahash_queue_req()
797 	mv_cesa_ahash_prepare(&req->base, engine);  in mv_cesa_ahash_queue_req()
799 	ret = mv_cesa_queue_req(&req->base, &creq->base);  in mv_cesa_ahash_queue_req()
801 	if (mv_cesa_req_needs_cleanup(&req->base, ret))  in mv_cesa_ahash_queue_req()
802 		mv_cesa_ahash_cleanup(req);  in mv_cesa_ahash_queue_req()
807 static int mv_cesa_ahash_update(struct ahash_request *req)  in mv_cesa_ahash_update()  argument
809 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_update()
811 	creq->len += req->nbytes;  in mv_cesa_ahash_update()
813 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_update()
816 static int mv_cesa_ahash_final(struct ahash_request *req)  in mv_cesa_ahash_final()  argument
818 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_final()
823 	req->nbytes = 0;  in mv_cesa_ahash_final()
825 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_final()
828 static int mv_cesa_ahash_finup(struct ahash_request *req)  in mv_cesa_ahash_finup()  argument
830 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_finup()
833 	creq->len += req->nbytes;  in mv_cesa_ahash_finup()
837 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_finup()
840 static int mv_cesa_ahash_export(struct ahash_request *req, void *hash,  in mv_cesa_ahash_export()  argument
843 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_export()
844 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_export()
858 static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash,  in mv_cesa_ahash_import()  argument
861 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_import()
862 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_import()
868 	ret = crypto_ahash_init(req);  in mv_cesa_ahash_import()
892 static int mv_cesa_md5_init(struct ahash_request *req)  in mv_cesa_md5_init()  argument
894 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_md5_init()
899 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_md5_init()
909 static int mv_cesa_md5_export(struct ahash_request *req, void *out)  in mv_cesa_md5_export()  argument
913 	return mv_cesa_ahash_export(req, out_state->hash,  in mv_cesa_md5_export()
917 static int mv_cesa_md5_import(struct ahash_request *req, const void *in)  in mv_cesa_md5_import()  argument
921 	return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count,  in mv_cesa_md5_import()
925 static int mv_cesa_md5_digest(struct ahash_request *req)  in mv_cesa_md5_digest()  argument
929 	ret = mv_cesa_md5_init(req);  in mv_cesa_md5_digest()
933 	return mv_cesa_ahash_finup(req);  in mv_cesa_md5_digest()
962 static int mv_cesa_sha1_init(struct ahash_request *req)  in mv_cesa_sha1_init()  argument
964 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_sha1_init()
969 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha1_init()
980 static int mv_cesa_sha1_export(struct ahash_request *req, void *out)  in mv_cesa_sha1_export()  argument
984 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha1_export()
988 static int mv_cesa_sha1_import(struct ahash_request *req, const void *in)  in mv_cesa_sha1_import()  argument
992 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha1_import()
996 static int mv_cesa_sha1_digest(struct ahash_request *req)  in mv_cesa_sha1_digest()  argument
1000 	ret = mv_cesa_sha1_init(req);  in mv_cesa_sha1_digest()
1004 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha1_digest()
1033 static int mv_cesa_sha256_init(struct ahash_request *req)  in mv_cesa_sha256_init()  argument
1035 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_sha256_init()
1040 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha256_init()
1054 static int mv_cesa_sha256_digest(struct ahash_request *req)  in mv_cesa_sha256_digest()  argument
1058 	ret = mv_cesa_sha256_init(req);  in mv_cesa_sha256_digest()
1062 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha256_digest()
1065 static int mv_cesa_sha256_export(struct ahash_request *req, void *out)  in mv_cesa_sha256_export()  argument
1069 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha256_export()
1073 static int mv_cesa_sha256_import(struct ahash_request *req, const void *in)  in mv_cesa_sha256_import()  argument
1077 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha256_import()
1112 static void mv_cesa_hmac_ahash_complete(struct crypto_async_request *req,  in mv_cesa_hmac_ahash_complete()  argument
1115 	struct mv_cesa_ahash_result *result = req->data;  in mv_cesa_hmac_ahash_complete()
1124 static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad,  in mv_cesa_ahmac_iv_state_init()  argument
1131 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_iv_state_init()
1134 	ahash_request_set_crypt(req, &sg, pad, blocksize);  in mv_cesa_ahmac_iv_state_init()
1137 	ret = crypto_ahash_init(req);  in mv_cesa_ahmac_iv_state_init()
1141 	ret = crypto_ahash_update(req);  in mv_cesa_ahmac_iv_state_init()
1149 	ret = crypto_ahash_export(req, state);  in mv_cesa_ahmac_iv_state_init()
1156 static int mv_cesa_ahmac_pad_init(struct ahash_request *req,  in mv_cesa_ahmac_pad_init()  argument
1174 		ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_pad_init()
1178 		ahash_request_set_crypt(req, &sg, ipad, keylen);  in mv_cesa_ahmac_pad_init()
1181 		ret = crypto_ahash_digest(req);  in mv_cesa_ahmac_pad_init()
1193 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));  in mv_cesa_ahmac_pad_init()
1211 	struct ahash_request *req;  in mv_cesa_ahmac_setkey()  local
1222 	req = ahash_request_alloc(tfm, GFP_KERNEL);  in mv_cesa_ahmac_setkey()
1223 	if (!req) {  in mv_cesa_ahmac_setkey()
1240 	ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize);  in mv_cesa_ahmac_setkey()
1244 	ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize);  in mv_cesa_ahmac_setkey()
1248 	ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize);  in mv_cesa_ahmac_setkey()
1253 	ahash_request_free(req);  in mv_cesa_ahmac_setkey()
1271 static int mv_cesa_ahmac_md5_init(struct ahash_request *req)  in mv_cesa_ahmac_md5_init()  argument
1273 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_md5_init()
1279 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_ahmac_md5_init()
1304 static int mv_cesa_ahmac_md5_digest(struct ahash_request *req)  in mv_cesa_ahmac_md5_digest()  argument
1308 	ret = mv_cesa_ahmac_md5_init(req);  in mv_cesa_ahmac_md5_digest()
1312 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_md5_digest()
1342 static int mv_cesa_ahmac_sha1_init(struct ahash_request *req)  in mv_cesa_ahmac_sha1_init()  argument
1344 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha1_init()
1350 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha1_init()
1375 static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha1_digest()  argument
1379 	ret = mv_cesa_ahmac_sha1_init(req);  in mv_cesa_ahmac_sha1_digest()
1383 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha1_digest()
1433 static int mv_cesa_ahmac_sha256_init(struct ahash_request *req)  in mv_cesa_ahmac_sha256_init()  argument
1435 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha256_init()
1441 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha256_init()
1446 static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha256_digest()  argument
1450 	ret = mv_cesa_ahmac_sha256_init(req);  in mv_cesa_ahmac_sha256_digest()
1454 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha256_digest()