222 static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,  in mtk_aes_set_mode()  argument
226 	aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;  in mtk_aes_set_mode()
263 				   struct mtk_aes_rec *aes,  in mtk_aes_complete()  argument
266 	aes->flags &= ~AES_FLAGS_BUSY;  in mtk_aes_complete()
267 	aes->areq->complete(aes->areq, err);  in mtk_aes_complete()
269 	tasklet_schedule(&aes->queue_task);  in mtk_aes_complete()
277 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_xmit()  argument
279 	struct mtk_ring *ring = cryp->ring[aes->id];  in mtk_aes_xmit()
281 	struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;  in mtk_aes_xmit()
282 	u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;  in mtk_aes_xmit()
293 				    MTK_DESC_CT_LEN(aes->ctx->ct_size);  in mtk_aes_xmit()
294 			cmd->ct = cpu_to_le32(aes->ctx->ct_dma);  in mtk_aes_xmit()
295 			cmd->ct_hdr = aes->ctx->ct_hdr;  in mtk_aes_xmit()
296 			cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);  in mtk_aes_xmit()
324 	if (aes->flags & AES_FLAGS_GCM)  in mtk_aes_xmit()
333 	mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));  in mtk_aes_xmit()
334 	mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));  in mtk_aes_xmit()
339 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_unmap()  argument
341 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_unmap()
346 	if (aes->src.sg == aes->dst.sg) {  in mtk_aes_unmap()
347 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
350 		if (aes->src.sg != &aes->aligned_sg)  in mtk_aes_unmap()
351 			mtk_aes_restore_sg(&aes->src);  in mtk_aes_unmap()
353 		dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,  in mtk_aes_unmap()
356 		if (aes->dst.sg != &aes->aligned_sg)  in mtk_aes_unmap()
357 			mtk_aes_restore_sg(&aes->dst);  in mtk_aes_unmap()
359 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
362 		if (aes->src.sg != &aes->aligned_sg)  in mtk_aes_unmap()
363 			mtk_aes_restore_sg(&aes->src);  in mtk_aes_unmap()
366 	if (aes->dst.sg == &aes->aligned_sg)  in mtk_aes_unmap()
367 		sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),  in mtk_aes_unmap()
368 				    aes->buf, aes->total);  in mtk_aes_unmap()
371 static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_map()  argument
373 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_map()
383 	if (aes->src.sg == aes->dst.sg) {  in mtk_aes_map()
384 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
385 					     aes->src.nents,  in mtk_aes_map()
387 		aes->dst.sg_len = aes->src.sg_len;  in mtk_aes_map()
388 		if (unlikely(!aes->src.sg_len))  in mtk_aes_map()
391 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
392 					     aes->src.nents, DMA_TO_DEVICE);  in mtk_aes_map()
393 		if (unlikely(!aes->src.sg_len))  in mtk_aes_map()
396 		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,  in mtk_aes_map()
397 					     aes->dst.nents, DMA_FROM_DEVICE);  in mtk_aes_map()
398 		if (unlikely(!aes->dst.sg_len)) {  in mtk_aes_map()
399 			dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_map()
405 	return mtk_aes_xmit(cryp, aes);  in mtk_aes_map()
410 	return mtk_aes_complete(cryp, aes, -EINVAL);  in mtk_aes_map()
414 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_info_init()  argument
417 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_info_init()
418 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_info_init()
427 	if (aes->flags & AES_FLAGS_ENCRYPT)  in mtk_aes_info_init()
432 	switch (aes->flags & AES_FLAGS_CIPHER_MSK) {  in mtk_aes_info_init()
463 static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_dma()  argument
470 	aes->total = len;  in mtk_aes_dma()
471 	aes->src.sg = src;  in mtk_aes_dma()
472 	aes->dst.sg = dst;  in mtk_aes_dma()
473 	aes->real_dst = dst;  in mtk_aes_dma()
475 	src_aligned = mtk_aes_check_aligned(src, len, &aes->src);  in mtk_aes_dma()
479 		dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);  in mtk_aes_dma()
485 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_dma()
488 			sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);  in mtk_aes_dma()
489 			aes->src.sg = &aes->aligned_sg;  in mtk_aes_dma()
490 			aes->src.nents = 1;  in mtk_aes_dma()
491 			aes->src.remainder = 0;  in mtk_aes_dma()
495 			aes->dst.sg = &aes->aligned_sg;  in mtk_aes_dma()
496 			aes->dst.nents = 1;  in mtk_aes_dma()
497 			aes->dst.remainder = 0;  in mtk_aes_dma()
500 		sg_init_table(&aes->aligned_sg, 1);  in mtk_aes_dma()
501 		sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);  in mtk_aes_dma()
504 	mtk_aes_info_init(cryp, aes, len + padlen);  in mtk_aes_dma()
506 	return mtk_aes_map(cryp, aes);  in mtk_aes_dma()
512 	struct mtk_aes_rec *aes = cryp->aes[id];  in mtk_aes_handle_queue()  local
518 	spin_lock_irqsave(&aes->lock, flags);  in mtk_aes_handle_queue()
520 		ret = crypto_enqueue_request(&aes->queue, new_areq);  in mtk_aes_handle_queue()
521 	if (aes->flags & AES_FLAGS_BUSY) {  in mtk_aes_handle_queue()
522 		spin_unlock_irqrestore(&aes->lock, flags);  in mtk_aes_handle_queue()
525 	backlog = crypto_get_backlog(&aes->queue);  in mtk_aes_handle_queue()
526 	areq = crypto_dequeue_request(&aes->queue);  in mtk_aes_handle_queue()
528 		aes->flags |= AES_FLAGS_BUSY;  in mtk_aes_handle_queue()
529 	spin_unlock_irqrestore(&aes->lock, flags);  in mtk_aes_handle_queue()
541 	aes->areq = areq;  in mtk_aes_handle_queue()
542 	aes->ctx = ctx;  in mtk_aes_handle_queue()
544 	return ctx->start(cryp, aes);  in mtk_aes_handle_queue()
548 				     struct mtk_aes_rec *aes)  in mtk_aes_transfer_complete()  argument
550 	return mtk_aes_complete(cryp, aes, 0);  in mtk_aes_transfer_complete()
553 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_start()  argument
555 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_start()
558 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_start()
559 	aes->resume = mtk_aes_transfer_complete;  in mtk_aes_start()
561 	return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);  in mtk_aes_start()
570 static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_transfer()  argument
572 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_ctr_transfer()
574 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_ctr_transfer()
581 	cctx->offset += aes->total;  in mtk_aes_ctr_transfer()
583 		return mtk_aes_transfer_complete(cryp, aes);  in mtk_aes_ctr_transfer()
617 	return mtk_aes_dma(cryp, aes, src, dst, datalen);  in mtk_aes_ctr_transfer()
620 static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_start()  argument
622 	struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);  in mtk_aes_ctr_start()
623 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_ctr_start()
626 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_ctr_start()
630 	aes->total = 0;  in mtk_aes_ctr_start()
631 	aes->resume = mtk_aes_ctr_transfer;  in mtk_aes_ctr_start()
633 	return mtk_aes_ctr_transfer(cryp, aes);  in mtk_aes_ctr_start()
868 				  struct mtk_aes_rec *aes)  in mtk_aes_gcm_tag_verify()  argument
870 	u32 status = cryp->ring[aes->id]->res_prev->ct;  in mtk_aes_gcm_tag_verify()
872 	return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?  in mtk_aes_gcm_tag_verify()
878 				  struct mtk_aes_rec *aes,  in mtk_aes_gcm_info_init()  argument
881 	struct aead_request *req = aead_request_cast(aes->areq);  in mtk_aes_gcm_info_init()
882 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_gcm_info_init()
895 	if (aes->flags & AES_FLAGS_ENCRYPT) {  in mtk_aes_gcm_info_init()
915 static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_gcm_dma()  argument
921 	aes->src.sg = src;  in mtk_aes_gcm_dma()
922 	aes->dst.sg = dst;  in mtk_aes_gcm_dma()
923 	aes->real_dst = dst;  in mtk_aes_gcm_dma()
925 	src_aligned = mtk_aes_check_aligned(src, len, &aes->src);  in mtk_aes_gcm_dma()
929 		dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);  in mtk_aes_gcm_dma()
932 		if (aes->total > AES_BUF_SIZE)  in mtk_aes_gcm_dma()
933 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_gcm_dma()
936 			sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);  in mtk_aes_gcm_dma()
937 			aes->src.sg = &aes->aligned_sg;  in mtk_aes_gcm_dma()
938 			aes->src.nents = 1;  in mtk_aes_gcm_dma()
939 			aes->src.remainder = 0;  in mtk_aes_gcm_dma()
943 			aes->dst.sg = &aes->aligned_sg;  in mtk_aes_gcm_dma()
944 			aes->dst.nents = 1;  in mtk_aes_gcm_dma()
945 			aes->dst.remainder = 0;  in mtk_aes_gcm_dma()
948 		sg_init_table(&aes->aligned_sg, 1);  in mtk_aes_gcm_dma()
949 		sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);  in mtk_aes_gcm_dma()
952 	mtk_aes_gcm_info_init(cryp, aes, len);  in mtk_aes_gcm_dma()
954 	return mtk_aes_map(cryp, aes);  in mtk_aes_gcm_dma()
958 static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_gcm_start()  argument
960 	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);  in mtk_aes_gcm_start()
961 	struct aead_request *req = aead_request_cast(aes->areq);  in mtk_aes_gcm_start()
965 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_gcm_start()
967 	if (aes->flags & AES_FLAGS_ENCRYPT) {  in mtk_aes_gcm_start()
970 		aes->resume = mtk_aes_transfer_complete;  in mtk_aes_gcm_start()
972 		aes->total = len + gctx->authsize;  in mtk_aes_gcm_start()
976 		aes->resume = mtk_aes_gcm_tag_verify;  in mtk_aes_gcm_start()
977 		aes->total = len;  in mtk_aes_gcm_start()
980 	return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);  in mtk_aes_gcm_start()
1161 	struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;  in mtk_aes_queue_task()  local
1163 	mtk_aes_handle_queue(aes->cryp, aes->id, NULL);  in mtk_aes_queue_task()
1168 	struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;  in mtk_aes_done_task()  local
1169 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_done_task()
1171 	mtk_aes_unmap(cryp, aes);  in mtk_aes_done_task()
1172 	aes->resume(cryp, aes);  in mtk_aes_done_task()
1177 	struct mtk_aes_rec *aes  = (struct mtk_aes_rec *)dev_id;  in mtk_aes_irq()  local
1178 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_irq()
1179 	u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));  in mtk_aes_irq()
1181 	mtk_aes_write(cryp, RDR_STAT(aes->id), val);  in mtk_aes_irq()
1183 	if (likely(AES_FLAGS_BUSY & aes->flags)) {  in mtk_aes_irq()
1184 		mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);  in mtk_aes_irq()
1185 		mtk_aes_write(cryp, RDR_THRESH(aes->id),  in mtk_aes_irq()
1188 		tasklet_schedule(&aes->done_task);  in mtk_aes_irq()
1203 	struct mtk_aes_rec **aes = cryp->aes;  in mtk_aes_record_init()  local
1207 		aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);  in mtk_aes_record_init()
1208 		if (!aes[i])  in mtk_aes_record_init()
1211 		aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,  in mtk_aes_record_init()
1213 		if (!aes[i]->buf)  in mtk_aes_record_init()
1216 		aes[i]->cryp = cryp;  in mtk_aes_record_init()
1218 		spin_lock_init(&aes[i]->lock);  in mtk_aes_record_init()
1219 		crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);  in mtk_aes_record_init()
1221 		tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task,  in mtk_aes_record_init()
1222 			     (unsigned long)aes[i]);  in mtk_aes_record_init()
1223 		tasklet_init(&aes[i]->done_task, mtk_aes_done_task,  in mtk_aes_record_init()
1224 			     (unsigned long)aes[i]);  in mtk_aes_record_init()
1228 	aes[0]->id = MTK_RING0;  in mtk_aes_record_init()
1229 	aes[1]->id = MTK_RING1;  in mtk_aes_record_init()
1235 		free_page((unsigned long)aes[i]->buf);  in mtk_aes_record_init()
1236 		kfree(aes[i]);  in mtk_aes_record_init()
1247 		tasklet_kill(&cryp->aes[i]->done_task);  in mtk_aes_record_free()
1248 		tasklet_kill(&cryp->aes[i]->queue_task);  in mtk_aes_record_free()
1250 		free_page((unsigned long)cryp->aes[i]->buf);  in mtk_aes_record_free()
1251 		kfree(cryp->aes[i]);  in mtk_aes_record_free()
1300 			       0, "mtk-aes", cryp->aes[0]);  in mtk_cipher_alg_register()
1307 			       0, "mtk-aes", cryp->aes[1]);  in mtk_cipher_alg_register()