26 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
29 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)  in omap_aes_gcm_finish_req()  argument
31 	struct aead_request *req = dd->aead_req;  in omap_aes_gcm_finish_req()
33 	dd->in_sg = NULL;  in omap_aes_gcm_finish_req()
34 	dd->out_sg = NULL;  in omap_aes_gcm_finish_req()
36 	crypto_finalize_aead_request(dd->engine, req, ret);  in omap_aes_gcm_finish_req()
38 	pm_runtime_mark_last_busy(dd->dev);  in omap_aes_gcm_finish_req()
39 	pm_runtime_put_autosuspend(dd->dev);  in omap_aes_gcm_finish_req()
42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)  in omap_aes_gcm_done_task()  argument
48 	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);  in omap_aes_gcm_done_task()
49 	clen = ALIGN(dd->total, AES_BLOCK_SIZE);  in omap_aes_gcm_done_task()
50 	rctx = aead_request_ctx(dd->aead_req);  in omap_aes_gcm_done_task()
52 	nsg = !!(dd->assoc_len && dd->total);  in omap_aes_gcm_done_task()
54 	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_gcm_done_task()
56 	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_aes_gcm_done_task()
57 	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);  in omap_aes_gcm_done_task()
58 	omap_aes_crypt_dma_stop(dd);  in omap_aes_gcm_done_task()
60 	omap_crypto_cleanup(dd->out_sg, dd->orig_out,  in omap_aes_gcm_done_task()
61 			    dd->aead_req->assoclen, dd->total,  in omap_aes_gcm_done_task()
62 			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
64 	if (dd->flags & FLAGS_ENCRYPT)  in omap_aes_gcm_done_task()
66 					 dd->aead_req->dst,  in omap_aes_gcm_done_task()
67 					 dd->total + dd->aead_req->assoclen,  in omap_aes_gcm_done_task()
68 					 dd->authsize, 1);  in omap_aes_gcm_done_task()
70 	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,  in omap_aes_gcm_done_task()
71 			    FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
73 	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,  in omap_aes_gcm_done_task()
74 			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
76 	if (!(dd->flags & FLAGS_ENCRYPT)) {  in omap_aes_gcm_done_task()
78 		for (i = 0; i < dd->authsize; i++) {  in omap_aes_gcm_done_task()
85 	omap_aes_gcm_finish_req(dd, ret);  in omap_aes_gcm_done_task()
88 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,  in omap_aes_gcm_copy_buffers()  argument
101 	if (dd->flags & FLAGS_RFC4106_GCM)  in omap_aes_gcm_copy_buffers()
104 	if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_copy_buffers()
112 	omap_aes_clear_copy_flags(dd);  in omap_aes_gcm_copy_buffers()
114 	sg_init_table(dd->in_sgl, nsg + 1);  in omap_aes_gcm_copy_buffers()
118 					   AES_BLOCK_SIZE, dd->in_sgl,  in omap_aes_gcm_copy_buffers()
123 					   &dd->flags);  in omap_aes_gcm_copy_buffers()
132 			sg_unmark_end(dd->in_sgl);  in omap_aes_gcm_copy_buffers()
135 					   AES_BLOCK_SIZE, &dd->in_sgl[nsg],  in omap_aes_gcm_copy_buffers()
140 					   &dd->flags);  in omap_aes_gcm_copy_buffers()
145 	dd->in_sg = dd->in_sgl;  in omap_aes_gcm_copy_buffers()
146 	dd->total = cryptlen;  in omap_aes_gcm_copy_buffers()
147 	dd->assoc_len = assoclen;  in omap_aes_gcm_copy_buffers()
148 	dd->authsize = authlen;  in omap_aes_gcm_copy_buffers()
150 	dd->out_sg = req->dst;  in omap_aes_gcm_copy_buffers()
151 	dd->orig_out = req->dst;  in omap_aes_gcm_copy_buffers()
153 	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);  in omap_aes_gcm_copy_buffers()
156 	if (req->src == req->dst || dd->out_sg == sg_arr)  in omap_aes_gcm_copy_buffers()
160 		ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,  in omap_aes_gcm_copy_buffers()
161 					   AES_BLOCK_SIZE, &dd->out_sgl,  in omap_aes_gcm_copy_buffers()
163 					   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);  in omap_aes_gcm_copy_buffers()
168 	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);  in omap_aes_gcm_copy_buffers()
169 	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);  in omap_aes_gcm_copy_buffers()
184 	struct omap_aes_dev *dd = data;  in omap_aes_gcm_dma_out_callback()  local
189 	if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_dma_out_callback()
190 		scatterwalk_map_and_copy(tag, dd->aead_req->src,  in omap_aes_gcm_dma_out_callback()
191 					 dd->total + dd->aead_req->assoclen,  in omap_aes_gcm_dma_out_callback()
192 					 dd->authsize, 0);  in omap_aes_gcm_dma_out_callback()
194 	rctx = aead_request_ctx(dd->aead_req);  in omap_aes_gcm_dma_out_callback()
197 		val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));  in omap_aes_gcm_dma_out_callback()
199 		if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_dma_out_callback()
203 	omap_aes_gcm_done_task(dd);  in omap_aes_gcm_dma_out_callback()
206 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,  in omap_aes_gcm_handle_queue()  argument
210 		return crypto_transfer_aead_request_to_engine(dd->engine, req);  in omap_aes_gcm_handle_queue()
220 	struct omap_aes_dev *dd = rctx->dd;  in omap_aes_gcm_prepare_req()  local
224 	dd->aead_req = req;  in omap_aes_gcm_prepare_req()
227 	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in omap_aes_gcm_prepare_req()
229 	err = omap_aes_gcm_copy_buffers(dd, req);  in omap_aes_gcm_prepare_req()
233 	dd->ctx = &ctx->octx;  in omap_aes_gcm_prepare_req()
235 	return omap_aes_write_ctrl(dd);  in omap_aes_gcm_prepare_req()
243 	struct omap_aes_dev *dd;  in omap_aes_gcm_crypt()  local
264 	dd = omap_aes_find_dev(rctx);  in omap_aes_gcm_crypt()
265 	if (!dd)  in omap_aes_gcm_crypt()
269 	return omap_aes_gcm_handle_queue(dd, req);  in omap_aes_gcm_crypt()
364 	struct omap_aes_dev *dd = rctx->dd;  in omap_aes_gcm_crypt_req()  local
367 	if (!dd)  in omap_aes_gcm_crypt_req()
370 	if (dd->in_sg_len)  in omap_aes_gcm_crypt_req()
371 		ret = omap_aes_crypt_dma_start(dd);  in omap_aes_gcm_crypt_req()
373 		omap_aes_gcm_dma_out_callback(dd);  in omap_aes_gcm_crypt_req()