220 static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,  in mtk_aes_set_mode()  argument
224 	aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;  in mtk_aes_set_mode()
261 				   struct mtk_aes_rec *aes,  in mtk_aes_complete()  argument
264 	aes->flags &= ~AES_FLAGS_BUSY;  in mtk_aes_complete()
265 	aes->areq->complete(aes->areq, err);  in mtk_aes_complete()
267 	tasklet_schedule(&aes->queue_task);  in mtk_aes_complete()
275 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_xmit()  argument
277 	struct mtk_ring *ring = cryp->ring[aes->id];  in mtk_aes_xmit()
279 	struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;  in mtk_aes_xmit()
280 	u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;  in mtk_aes_xmit()
291 				    MTK_DESC_CT_LEN(aes->ctx->ct_size);  in mtk_aes_xmit()
292 			cmd->ct = cpu_to_le32(aes->ctx->ct_dma);  in mtk_aes_xmit()
293 			cmd->ct_hdr = aes->ctx->ct_hdr;  in mtk_aes_xmit()
294 			cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);  in mtk_aes_xmit()
322 	if (aes->flags & AES_FLAGS_GCM)  in mtk_aes_xmit()
331 	mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));  in mtk_aes_xmit()
332 	mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));  in mtk_aes_xmit()
337 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_unmap()  argument
339 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_unmap()
344 	if (aes->src.sg == aes->dst.sg) {  in mtk_aes_unmap()
345 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
348 		if (aes->src.sg != &aes->aligned_sg)  in mtk_aes_unmap()
349 			mtk_aes_restore_sg(&aes->src);  in mtk_aes_unmap()
351 		dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,  in mtk_aes_unmap()
354 		if (aes->dst.sg != &aes->aligned_sg)  in mtk_aes_unmap()
355 			mtk_aes_restore_sg(&aes->dst);  in mtk_aes_unmap()
357 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
360 		if (aes->src.sg != &aes->aligned_sg)  in mtk_aes_unmap()
361 			mtk_aes_restore_sg(&aes->src);  in mtk_aes_unmap()
364 	if (aes->dst.sg == &aes->aligned_sg)  in mtk_aes_unmap()
365 		sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),  in mtk_aes_unmap()
366 				    aes->buf, aes->total);  in mtk_aes_unmap()
369 static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_map()  argument
371 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_map()
381 	if (aes->src.sg == aes->dst.sg) {  in mtk_aes_map()
382 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
383 					     aes->src.nents,  in mtk_aes_map()
385 		aes->dst.sg_len = aes->src.sg_len;  in mtk_aes_map()
386 		if (unlikely(!aes->src.sg_len))  in mtk_aes_map()
389 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
390 					     aes->src.nents, DMA_TO_DEVICE);  in mtk_aes_map()
391 		if (unlikely(!aes->src.sg_len))  in mtk_aes_map()
394 		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,  in mtk_aes_map()
395 					     aes->dst.nents, DMA_FROM_DEVICE);  in mtk_aes_map()
396 		if (unlikely(!aes->dst.sg_len)) {  in mtk_aes_map()
397 			dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_map()
403 	return mtk_aes_xmit(cryp, aes);  in mtk_aes_map()
408 	return mtk_aes_complete(cryp, aes, -EINVAL);  in mtk_aes_map()
412 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_info_init()  argument
415 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_info_init()
416 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_info_init()
425 	if (aes->flags & AES_FLAGS_ENCRYPT)  in mtk_aes_info_init()
430 	switch (aes->flags & AES_FLAGS_CIPHER_MSK) {  in mtk_aes_info_init()
456 static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_dma()  argument
463 	aes->total = len;  in mtk_aes_dma()
464 	aes->src.sg = src;  in mtk_aes_dma()
465 	aes->dst.sg = dst;  in mtk_aes_dma()
466 	aes->real_dst = dst;  in mtk_aes_dma()
468 	src_aligned = mtk_aes_check_aligned(src, len, &aes->src);  in mtk_aes_dma()
472 		dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);  in mtk_aes_dma()
478 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_dma()
481 			sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);  in mtk_aes_dma()
482 			aes->src.sg = &aes->aligned_sg;  in mtk_aes_dma()
483 			aes->src.nents = 1;  in mtk_aes_dma()
484 			aes->src.remainder = 0;  in mtk_aes_dma()
488 			aes->dst.sg = &aes->aligned_sg;  in mtk_aes_dma()
489 			aes->dst.nents = 1;  in mtk_aes_dma()
490 			aes->dst.remainder = 0;  in mtk_aes_dma()
493 		sg_init_table(&aes->aligned_sg, 1);  in mtk_aes_dma()
494 		sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);  in mtk_aes_dma()
497 	mtk_aes_info_init(cryp, aes, len + padlen);  in mtk_aes_dma()
499 	return mtk_aes_map(cryp, aes);  in mtk_aes_dma()
505 	struct mtk_aes_rec *aes = cryp->aes[id];  in mtk_aes_handle_queue()  local
511 	spin_lock_irqsave(&aes->lock, flags);  in mtk_aes_handle_queue()
513 		ret = crypto_enqueue_request(&aes->queue, new_areq);  in mtk_aes_handle_queue()
514 	if (aes->flags & AES_FLAGS_BUSY) {  in mtk_aes_handle_queue()
515 		spin_unlock_irqrestore(&aes->lock, flags);  in mtk_aes_handle_queue()
518 	backlog = crypto_get_backlog(&aes->queue);  in mtk_aes_handle_queue()
519 	areq = crypto_dequeue_request(&aes->queue);  in mtk_aes_handle_queue()
521 		aes->flags |= AES_FLAGS_BUSY;  in mtk_aes_handle_queue()
522 	spin_unlock_irqrestore(&aes->lock, flags);  in mtk_aes_handle_queue()
532 	aes->areq = areq;  in mtk_aes_handle_queue()
533 	aes->ctx = ctx;  in mtk_aes_handle_queue()
535 	return ctx->start(cryp, aes);  in mtk_aes_handle_queue()
539 				     struct mtk_aes_rec *aes)  in mtk_aes_transfer_complete()  argument
541 	return mtk_aes_complete(cryp, aes, 0);  in mtk_aes_transfer_complete()
544 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_start()  argument
546 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_start()
549 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_start()
550 	aes->resume = mtk_aes_transfer_complete;  in mtk_aes_start()
552 	return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);  in mtk_aes_start()
561 static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_transfer()  argument
563 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_ctr_transfer()
565 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_ctr_transfer()
572 	cctx->offset += aes->total;  in mtk_aes_ctr_transfer()
574 		return mtk_aes_transfer_complete(cryp, aes);  in mtk_aes_ctr_transfer()
608 	return mtk_aes_dma(cryp, aes, src, dst, datalen);  in mtk_aes_ctr_transfer()
611 static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_start()  argument
613 	struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);  in mtk_aes_ctr_start()
614 	struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);  in mtk_aes_ctr_start()
617 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_ctr_start()
621 	aes->total = 0;  in mtk_aes_ctr_start()
622 	aes->resume = mtk_aes_ctr_transfer;  in mtk_aes_ctr_start()
624 	return mtk_aes_ctr_transfer(cryp, aes);  in mtk_aes_ctr_start()
806 				  struct mtk_aes_rec *aes)  in mtk_aes_gcm_tag_verify()  argument
808 	u32 status = cryp->ring[aes->id]->res_prev->ct;  in mtk_aes_gcm_tag_verify()
810 	return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?  in mtk_aes_gcm_tag_verify()
816 				  struct mtk_aes_rec *aes,  in mtk_aes_gcm_info_init()  argument
819 	struct aead_request *req = aead_request_cast(aes->areq);  in mtk_aes_gcm_info_init()
820 	struct mtk_aes_base_ctx *ctx = aes->ctx;  in mtk_aes_gcm_info_init()
833 	if (aes->flags & AES_FLAGS_ENCRYPT) {  in mtk_aes_gcm_info_init()
853 static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_gcm_dma()  argument
859 	aes->src.sg = src;  in mtk_aes_gcm_dma()
860 	aes->dst.sg = dst;  in mtk_aes_gcm_dma()
861 	aes->real_dst = dst;  in mtk_aes_gcm_dma()
863 	src_aligned = mtk_aes_check_aligned(src, len, &aes->src);  in mtk_aes_gcm_dma()
867 		dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);  in mtk_aes_gcm_dma()
870 		if (aes->total > AES_BUF_SIZE)  in mtk_aes_gcm_dma()
871 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_gcm_dma()
874 			sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);  in mtk_aes_gcm_dma()
875 			aes->src.sg = &aes->aligned_sg;  in mtk_aes_gcm_dma()
876 			aes->src.nents = 1;  in mtk_aes_gcm_dma()
877 			aes->src.remainder = 0;  in mtk_aes_gcm_dma()
881 			aes->dst.sg = &aes->aligned_sg;  in mtk_aes_gcm_dma()
882 			aes->dst.nents = 1;  in mtk_aes_gcm_dma()
883 			aes->dst.remainder = 0;  in mtk_aes_gcm_dma()
886 		sg_init_table(&aes->aligned_sg, 1);  in mtk_aes_gcm_dma()
887 		sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);  in mtk_aes_gcm_dma()
890 	mtk_aes_gcm_info_init(cryp, aes, len);  in mtk_aes_gcm_dma()
892 	return mtk_aes_map(cryp, aes);  in mtk_aes_gcm_dma()
896 static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_gcm_start()  argument
898 	struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);  in mtk_aes_gcm_start()
899 	struct aead_request *req = aead_request_cast(aes->areq);  in mtk_aes_gcm_start()
903 	mtk_aes_set_mode(aes, rctx);  in mtk_aes_gcm_start()
905 	if (aes->flags & AES_FLAGS_ENCRYPT) {  in mtk_aes_gcm_start()
908 		aes->resume = mtk_aes_transfer_complete;  in mtk_aes_gcm_start()
910 		aes->total = len + gctx->authsize;  in mtk_aes_gcm_start()
916 		aes->resume = mtk_aes_gcm_tag_verify;  in mtk_aes_gcm_start()
917 		aes->total = len;  in mtk_aes_gcm_start()
921 	return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);  in mtk_aes_gcm_start()
1103 	struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;  in mtk_aes_queue_task()  local
1105 	mtk_aes_handle_queue(aes->cryp, aes->id, NULL);  in mtk_aes_queue_task()
1110 	struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;  in mtk_aes_done_task()  local
1111 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_done_task()
1113 	mtk_aes_unmap(cryp, aes);  in mtk_aes_done_task()
1114 	aes->resume(cryp, aes);  in mtk_aes_done_task()
1119 	struct mtk_aes_rec *aes  = (struct mtk_aes_rec *)dev_id;  in mtk_aes_irq()  local
1120 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_irq()
1121 	u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));  in mtk_aes_irq()
1123 	mtk_aes_write(cryp, RDR_STAT(aes->id), val);  in mtk_aes_irq()
1125 	if (likely(AES_FLAGS_BUSY & aes->flags)) {  in mtk_aes_irq()
1126 		mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);  in mtk_aes_irq()
1127 		mtk_aes_write(cryp, RDR_THRESH(aes->id),  in mtk_aes_irq()
1130 		tasklet_schedule(&aes->done_task);  in mtk_aes_irq()
1145 	struct mtk_aes_rec **aes = cryp->aes;  in mtk_aes_record_init()  local
1149 		aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);  in mtk_aes_record_init()
1150 		if (!aes[i])  in mtk_aes_record_init()
1153 		aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,  in mtk_aes_record_init()
1155 		if (!aes[i]->buf)  in mtk_aes_record_init()
1158 		aes[i]->cryp = cryp;  in mtk_aes_record_init()
1160 		spin_lock_init(&aes[i]->lock);  in mtk_aes_record_init()
1161 		crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);  in mtk_aes_record_init()
1163 		tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task,  in mtk_aes_record_init()
1164 			     (unsigned long)aes[i]);  in mtk_aes_record_init()
1165 		tasklet_init(&aes[i]->done_task, mtk_aes_done_task,  in mtk_aes_record_init()
1166 			     (unsigned long)aes[i]);  in mtk_aes_record_init()
1170 	aes[0]->id = MTK_RING0;  in mtk_aes_record_init()
1171 	aes[1]->id = MTK_RING1;  in mtk_aes_record_init()
1177 		free_page((unsigned long)aes[i]->buf);  in mtk_aes_record_init()
1178 		kfree(aes[i]);  in mtk_aes_record_init()
1189 		tasklet_kill(&cryp->aes[i]->done_task);  in mtk_aes_record_free()
1190 		tasklet_kill(&cryp->aes[i]->queue_task);  in mtk_aes_record_free()
1192 		free_page((unsigned long)cryp->aes[i]->buf);  in mtk_aes_record_free()
1193 		kfree(cryp->aes[i]);  in mtk_aes_record_free()
1242 			       0, "mtk-aes", cryp->aes[0]);  in mtk_cipher_alg_register()
1249 			       0, "mtk-aes", cryp->aes[1]);  in mtk_cipher_alg_register()