104 	struct atmel_aes_dev	*dd;  member
347 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)  in atmel_aes_read()  argument
349 	u32 value = readl_relaxed(dd->io_base + offset);  in atmel_aes_read()
352 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_read()
355 		dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,  in atmel_aes_read()
363 static inline void atmel_aes_write(struct atmel_aes_dev *dd,  in atmel_aes_write()  argument
367 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_write()
370 		dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,  in atmel_aes_write()
375 	writel_relaxed(value, dd->io_base + offset);  in atmel_aes_write()
378 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_n()  argument
382 		*value = atmel_aes_read(dd, offset);  in atmel_aes_read_n()
385 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_n()  argument
389 		atmel_aes_write(dd, offset, *value);  in atmel_aes_write_n()
392 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_block()  argument
395 	atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_read_block()
398 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_block()  argument
401 	atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_write_block()
404 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,  in atmel_aes_wait_for_data_ready()  argument
407 	u32 isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_wait_for_data_ready()
410 		return resume(dd);  in atmel_aes_wait_for_data_ready()
412 	dd->resume = resume;  in atmel_aes_wait_for_data_ready()
413 	atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_wait_for_data_ready()
435 static int atmel_aes_hw_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_init()  argument
439 	err = clk_enable(dd->iclk);  in atmel_aes_hw_init()
443 	atmel_aes_write(dd, AES_CR, AES_CR_SWRST);  in atmel_aes_hw_init()
444 	atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);  in atmel_aes_hw_init()
449 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)  in atmel_aes_get_version()  argument
451 	return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;  in atmel_aes_get_version()
454 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_version_init()  argument
458 	err = atmel_aes_hw_init(dd);  in atmel_aes_hw_version_init()
462 	dd->hw_version = atmel_aes_get_version(dd);  in atmel_aes_hw_version_init()
464 	dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);  in atmel_aes_hw_version_init()
466 	clk_disable(dd->iclk);  in atmel_aes_hw_version_init()
470 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,  in atmel_aes_set_mode()  argument
474 	dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;  in atmel_aes_set_mode()
477 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)  in atmel_aes_is_encrypt()  argument
479 	return (dd->flags & AES_FLAGS_ENCRYPT);  in atmel_aes_is_encrypt()
483 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
486 static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd)  in atmel_aes_set_iv_as_last_ciphertext_block()  argument
488 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_set_iv_as_last_ciphertext_block()
515 static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd)  in atmel_aes_ctr_update_req_iv()  argument
517 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_update_req_iv()
518 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_update_req_iv()
535 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_complete()  argument
537 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_complete()
541 	if (dd->ctx->is_aead)  in atmel_aes_complete()
542 		atmel_aes_authenc_complete(dd, err);  in atmel_aes_complete()
545 	clk_disable(dd->iclk);  in atmel_aes_complete()
546 	dd->flags &= ~AES_FLAGS_BUSY;  in atmel_aes_complete()
548 	if (!err && !dd->ctx->is_aead &&  in atmel_aes_complete()
551 			atmel_aes_set_iv_as_last_ciphertext_block(dd);  in atmel_aes_complete()
553 			atmel_aes_ctr_update_req_iv(dd);  in atmel_aes_complete()
556 	if (dd->is_async)  in atmel_aes_complete()
557 		dd->areq->complete(dd->areq, err);  in atmel_aes_complete()
559 	tasklet_schedule(&dd->queue_task);  in atmel_aes_complete()
564 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl_key()  argument
577 	valmr |= dd->flags & AES_FLAGS_MODE_MASK;  in atmel_aes_write_ctrl_key()
581 		if (dd->caps.has_dualbuff)  in atmel_aes_write_ctrl_key()
587 	atmel_aes_write(dd, AES_MR, valmr);  in atmel_aes_write_ctrl_key()
589 	atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));  in atmel_aes_write_ctrl_key()
592 		atmel_aes_write_block(dd, AES_IVR(0), iv);  in atmel_aes_write_ctrl_key()
595 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl()  argument
599 	atmel_aes_write_ctrl_key(dd, use_dma, iv,  in atmel_aes_write_ctrl()
600 				 dd->ctx->key, dd->ctx->keylen);  in atmel_aes_write_ctrl()
605 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)  in atmel_aes_cpu_transfer()  argument
611 		atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);  in atmel_aes_cpu_transfer()
612 		dd->data += 4;  in atmel_aes_cpu_transfer()
613 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_cpu_transfer()
615 		if (dd->datalen < AES_BLOCK_SIZE)  in atmel_aes_cpu_transfer()
618 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_transfer()
620 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_cpu_transfer()
622 			dd->resume = atmel_aes_cpu_transfer;  in atmel_aes_cpu_transfer()
623 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_cpu_transfer()
628 	if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_cpu_transfer()
629 				 dd->buf, dd->total))  in atmel_aes_cpu_transfer()
633 		return atmel_aes_complete(dd, err);  in atmel_aes_cpu_transfer()
635 	return dd->cpu_transfer_complete(dd);  in atmel_aes_cpu_transfer()
638 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,  in atmel_aes_cpu_start()  argument
649 	sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_cpu_start()
651 	dd->total = len;  in atmel_aes_cpu_start()
652 	dd->real_dst = dst;  in atmel_aes_cpu_start()
653 	dd->cpu_transfer_complete = resume;  in atmel_aes_cpu_start()
654 	dd->datalen = len + padlen;  in atmel_aes_cpu_start()
655 	dd->data = (u32 *)dd->buf;  in atmel_aes_cpu_start()
656 	atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_start()
657 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);  in atmel_aes_cpu_start()
665 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,  in atmel_aes_check_aligned()  argument
672 	if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
680 			if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
689 		if (!IS_ALIGNED(sg->length, dd->ctx->block_size))  in atmel_aes_check_aligned()
715 static int atmel_aes_map(struct atmel_aes_dev *dd,  in atmel_aes_map()  argument
723 	dd->total = len;  in atmel_aes_map()
724 	dd->src.sg = src;  in atmel_aes_map()
725 	dd->dst.sg = dst;  in atmel_aes_map()
726 	dd->real_dst = dst;  in atmel_aes_map()
728 	src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);  in atmel_aes_map()
732 		dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);  in atmel_aes_map()
734 		padlen = atmel_aes_padlen(len, dd->ctx->block_size);  in atmel_aes_map()
736 		if (dd->buflen < len + padlen)  in atmel_aes_map()
740 			sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_map()
741 			dd->src.sg = &dd->aligned_sg;  in atmel_aes_map()
742 			dd->src.nents = 1;  in atmel_aes_map()
743 			dd->src.remainder = 0;  in atmel_aes_map()
747 			dd->dst.sg = &dd->aligned_sg;  in atmel_aes_map()
748 			dd->dst.nents = 1;  in atmel_aes_map()
749 			dd->dst.remainder = 0;  in atmel_aes_map()
752 		sg_init_table(&dd->aligned_sg, 1);  in atmel_aes_map()
753 		sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);  in atmel_aes_map()
756 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_map()
757 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
759 		dd->dst.sg_len = dd->src.sg_len;  in atmel_aes_map()
760 		if (!dd->src.sg_len)  in atmel_aes_map()
763 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
765 		if (!dd->src.sg_len)  in atmel_aes_map()
768 		dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_map()
770 		if (!dd->dst.sg_len) {  in atmel_aes_map()
771 			dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
780 static void atmel_aes_unmap(struct atmel_aes_dev *dd)  in atmel_aes_unmap()  argument
782 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_unmap()
783 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
786 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
787 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
789 		dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_unmap()
792 		if (dd->dst.sg != &dd->aligned_sg)  in atmel_aes_unmap()
793 			atmel_aes_restore_sg(&dd->dst);  in atmel_aes_unmap()
795 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
798 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
799 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
802 	if (dd->dst.sg == &dd->aligned_sg)  in atmel_aes_unmap()
803 		sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_unmap()
804 				    dd->buf, dd->total);  in atmel_aes_unmap()
807 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_transfer_start()  argument
826 		dma = &dd->src;  in atmel_aes_dma_transfer_start()
828 		config.dst_addr = dd->phys_base + AES_IDATAR(0);  in atmel_aes_dma_transfer_start()
832 		dma = &dd->dst;  in atmel_aes_dma_transfer_start()
834 		config.src_addr = dd->phys_base + AES_ODATAR(0);  in atmel_aes_dma_transfer_start()
851 	desc->callback_param = dd;  in atmel_aes_dma_transfer_start()
858 static int atmel_aes_dma_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_start()  argument
868 	switch (dd->ctx->block_size) {  in atmel_aes_dma_start()
887 		maxburst = dd->caps.max_burst_size;  in atmel_aes_dma_start()
895 	err = atmel_aes_map(dd, src, dst, len);  in atmel_aes_dma_start()
899 	dd->resume = resume;  in atmel_aes_dma_start()
902 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,  in atmel_aes_dma_start()
908 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,  in atmel_aes_dma_start()
916 	dmaengine_terminate_sync(dd->dst.chan);  in atmel_aes_dma_start()
918 	atmel_aes_unmap(dd);  in atmel_aes_dma_start()
920 	return atmel_aes_complete(dd, err);  in atmel_aes_dma_start()
925 	struct atmel_aes_dev *dd = data;  in atmel_aes_dma_callback()  local
927 	atmel_aes_unmap(dd);  in atmel_aes_dma_callback()
928 	dd->is_async = true;  in atmel_aes_dma_callback()
929 	(void)dd->resume(dd);  in atmel_aes_dma_callback()
932 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,  in atmel_aes_handle_queue()  argument
941 	spin_lock_irqsave(&dd->lock, flags);  in atmel_aes_handle_queue()
943 		ret = crypto_enqueue_request(&dd->queue, new_areq);  in atmel_aes_handle_queue()
944 	if (dd->flags & AES_FLAGS_BUSY) {  in atmel_aes_handle_queue()
945 		spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
948 	backlog = crypto_get_backlog(&dd->queue);  in atmel_aes_handle_queue()
949 	areq = crypto_dequeue_request(&dd->queue);  in atmel_aes_handle_queue()
951 		dd->flags |= AES_FLAGS_BUSY;  in atmel_aes_handle_queue()
952 	spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
962 	dd->areq = areq;  in atmel_aes_handle_queue()
963 	dd->ctx = ctx;  in atmel_aes_handle_queue()
965 	dd->is_async = start_async;  in atmel_aes_handle_queue()
968 	err = ctx->start(dd);  in atmel_aes_handle_queue()
975 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)  in atmel_aes_transfer_complete()  argument
977 	return atmel_aes_complete(dd, 0);  in atmel_aes_transfer_complete()
980 static int atmel_aes_start(struct atmel_aes_dev *dd)  in atmel_aes_start()  argument
982 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_start()
985 			dd->ctx->block_size != AES_BLOCK_SIZE);  in atmel_aes_start()
988 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_start()
990 	err = atmel_aes_hw_init(dd);  in atmel_aes_start()
992 		return atmel_aes_complete(dd, err);  in atmel_aes_start()
994 	atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv);  in atmel_aes_start()
996 		return atmel_aes_dma_start(dd, req->src, req->dst,  in atmel_aes_start()
1000 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,  in atmel_aes_start()
1004 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)  in atmel_aes_ctr_transfer()  argument
1006 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_transfer()
1007 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_transfer()
1015 	ctx->offset += dd->total;  in atmel_aes_ctr_transfer()
1017 		return atmel_aes_transfer_complete(dd);  in atmel_aes_ctr_transfer()
1042 	atmel_aes_write_ctrl(dd, use_dma, ctx->iv);  in atmel_aes_ctr_transfer()
1053 		return atmel_aes_dma_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1056 	return atmel_aes_cpu_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1060 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)  in atmel_aes_ctr_start()  argument
1062 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_start()
1063 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_start()
1067 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_ctr_start()
1069 	err = atmel_aes_hw_init(dd);  in atmel_aes_ctr_start()
1071 		return atmel_aes_complete(dd, err);  in atmel_aes_ctr_start()
1075 	dd->total = 0;  in atmel_aes_ctr_start()
1076 	return atmel_aes_ctr_transfer(dd);  in atmel_aes_ctr_start()
1158 	return atmel_aes_handle_queue(ctx->dd, &req->base);  in atmel_aes_crypt()
1270 	struct atmel_aes_dev *dd;  in atmel_aes_init_tfm()  local
1272 	dd = atmel_aes_dev_alloc(&ctx->base);  in atmel_aes_init_tfm()
1273 	if (!dd)  in atmel_aes_init_tfm()
1277 	ctx->base.dd = dd;  in atmel_aes_init_tfm()
1286 	struct atmel_aes_dev *dd;  in atmel_aes_ctr_init_tfm()  local
1288 	dd = atmel_aes_dev_alloc(&ctx->base);  in atmel_aes_ctr_init_tfm()
1289 	if (!dd)  in atmel_aes_ctr_init_tfm()
1293 	ctx->base.dd = dd;  in atmel_aes_ctr_init_tfm()
1431 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
1435 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
1436 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
1438 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
1439 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
1440 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
1441 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
1442 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
1443 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
1444 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
1452 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,  in atmel_aes_gcm_ghash()  argument
1457 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash()
1459 	dd->data = (u32 *)data;  in atmel_aes_gcm_ghash()
1460 	dd->datalen = datalen;  in atmel_aes_gcm_ghash()
1465 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_ghash()
1466 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);  in atmel_aes_gcm_ghash()
1469 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_init()  argument
1471 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_init()
1474 	atmel_aes_write(dd, AES_AADLENR, dd->total);  in atmel_aes_gcm_ghash_init()
1475 	atmel_aes_write(dd, AES_CLENR, 0);  in atmel_aes_gcm_ghash_init()
1479 		atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);  in atmel_aes_gcm_ghash_init()
1481 	return atmel_aes_gcm_ghash_finalize(dd);  in atmel_aes_gcm_ghash_init()
1484 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_finalize()  argument
1486 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_finalize()
1490 	while (dd->datalen > 0) {  in atmel_aes_gcm_ghash_finalize()
1491 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_ghash_finalize()
1492 		dd->data += 4;  in atmel_aes_gcm_ghash_finalize()
1493 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_ghash_finalize()
1495 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_ghash_finalize()
1497 			dd->resume = atmel_aes_gcm_ghash_finalize;  in atmel_aes_gcm_ghash_finalize()
1498 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_ghash_finalize()
1504 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);  in atmel_aes_gcm_ghash_finalize()
1506 	return ctx->ghash_resume(dd);  in atmel_aes_gcm_ghash_finalize()
1510 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)  in atmel_aes_gcm_start()  argument
1512 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_start()
1513 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_start()
1519 	u8 *data = dd->buf;  in atmel_aes_gcm_start()
1522 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_gcm_start()
1524 	err = atmel_aes_hw_init(dd);  in atmel_aes_gcm_start()
1526 		return atmel_aes_complete(dd, err);  in atmel_aes_gcm_start()
1531 		return atmel_aes_gcm_process(dd);  in atmel_aes_gcm_start()
1536 	if (datalen > dd->buflen)  in atmel_aes_gcm_start()
1537 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_start()
1543 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,  in atmel_aes_gcm_start()
1547 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)  in atmel_aes_gcm_process()  argument
1549 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_process()
1550 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_process()
1552 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_process()
1564 		dd->flags |= AES_FLAGS_GTAGEN;  in atmel_aes_gcm_process()
1566 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_process()
1567 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);  in atmel_aes_gcm_process()
1570 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)  in atmel_aes_gcm_length()  argument
1572 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_length()
1573 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_length()
1580 	atmel_aes_write_block(dd, AES_IVR(0), j0);  in atmel_aes_gcm_length()
1584 	atmel_aes_write(dd, AES_AADLENR, req->assoclen);  in atmel_aes_gcm_length()
1585 	atmel_aes_write(dd, AES_CLENR, ctx->textlen);  in atmel_aes_gcm_length()
1589 		dd->datalen = 0;  in atmel_aes_gcm_length()
1590 		return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1595 	if (unlikely(req->assoclen + padlen > dd->buflen))  in atmel_aes_gcm_length()
1596 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_length()
1597 	sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);  in atmel_aes_gcm_length()
1600 	dd->data = (u32 *)dd->buf;  in atmel_aes_gcm_length()
1601 	dd->datalen = req->assoclen + padlen;  in atmel_aes_gcm_length()
1602 	return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1605 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)  in atmel_aes_gcm_data()  argument
1607 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_data()
1608 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_data()
1614 	while (dd->datalen > 0) {  in atmel_aes_gcm_data()
1615 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_data()
1616 		dd->data += 4;  in atmel_aes_gcm_data()
1617 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_data()
1619 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_data()
1621 			dd->resume = atmel_aes_gcm_data;  in atmel_aes_gcm_data()
1622 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_data()
1629 		return atmel_aes_gcm_tag_init(dd);  in atmel_aes_gcm_data()
1638 		mr = atmel_aes_read(dd, AES_MR);  in atmel_aes_gcm_data()
1641 		if (dd->caps.has_dualbuff)  in atmel_aes_gcm_data()
1643 		atmel_aes_write(dd, AES_MR, mr);  in atmel_aes_gcm_data()
1645 		return atmel_aes_dma_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1649 	return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1653 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag_init()  argument
1655 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag_init()
1656 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_tag_init()
1657 	__be64 *data = dd->buf;  in atmel_aes_gcm_tag_init()
1659 	if (likely(dd->flags & AES_FLAGS_GTAGEN)) {  in atmel_aes_gcm_tag_init()
1660 		if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {  in atmel_aes_gcm_tag_init()
1661 			dd->resume = atmel_aes_gcm_tag_init;  in atmel_aes_gcm_tag_init()
1662 			atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);  in atmel_aes_gcm_tag_init()
1666 		return atmel_aes_gcm_finalize(dd);  in atmel_aes_gcm_tag_init()
1670 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);  in atmel_aes_gcm_tag_init()
1675 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,  in atmel_aes_gcm_tag_init()
1679 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag()  argument
1681 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag()
1688 	flags = dd->flags;  in atmel_aes_gcm_tag()
1689 	dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);  in atmel_aes_gcm_tag()
1690 	dd->flags |= AES_FLAGS_CTR;  in atmel_aes_gcm_tag()
1691 	atmel_aes_write_ctrl(dd, false, ctx->j0);  in atmel_aes_gcm_tag()
1692 	dd->flags = flags;  in atmel_aes_gcm_tag()
1694 	atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);  in atmel_aes_gcm_tag()
1695 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);  in atmel_aes_gcm_tag()
1698 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_finalize()  argument
1700 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_finalize()
1701 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_finalize()
1703 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_finalize()
1708 	if (likely(dd->flags & AES_FLAGS_GTAGEN))  in atmel_aes_gcm_finalize()
1709 		atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1711 		atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1723 	return atmel_aes_complete(dd, err);  in atmel_aes_gcm_finalize()
1739 	return atmel_aes_handle_queue(ctx->dd, &req->base);  in atmel_aes_gcm_crypt()
1777 	struct atmel_aes_dev *dd;  in atmel_aes_gcm_init()  local
1779 	dd = atmel_aes_dev_alloc(&ctx->base);  in atmel_aes_gcm_init()
1780 	if (!dd)  in atmel_aes_gcm_init()
1784 	ctx->base.dd = dd;  in atmel_aes_gcm_init()
1816 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
1818 static int atmel_aes_xts_start(struct atmel_aes_dev *dd)  in atmel_aes_xts_start()  argument
1820 	struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);  in atmel_aes_xts_start()
1821 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_xts_start()
1826 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_xts_start()
1828 	err = atmel_aes_hw_init(dd);  in atmel_aes_xts_start()
1830 		return atmel_aes_complete(dd, err);  in atmel_aes_xts_start()
1833 	flags = dd->flags;  in atmel_aes_xts_start()
1834 	dd->flags &= ~AES_FLAGS_MODE_MASK;  in atmel_aes_xts_start()
1835 	dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);  in atmel_aes_xts_start()
1836 	atmel_aes_write_ctrl_key(dd, false, NULL,  in atmel_aes_xts_start()
1838 	dd->flags = flags;  in atmel_aes_xts_start()
1840 	atmel_aes_write_block(dd, AES_IDATAR(0), req->iv);  in atmel_aes_xts_start()
1841 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);  in atmel_aes_xts_start()
1844 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)  in atmel_aes_xts_process_data()  argument
1846 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_xts_process_data()
1854 	atmel_aes_read_block(dd, AES_ODATAR(0), tweak);  in atmel_aes_xts_process_data()
1864 	atmel_aes_write_ctrl(dd, use_dma, NULL);  in atmel_aes_xts_process_data()
1865 	atmel_aes_write_block(dd, AES_TWR(0), tweak);  in atmel_aes_xts_process_data()
1866 	atmel_aes_write_block(dd, AES_ALPHAR(0), one);  in atmel_aes_xts_process_data()
1868 		return atmel_aes_dma_start(dd, req->src, req->dst,  in atmel_aes_xts_process_data()
1872 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,  in atmel_aes_xts_process_data()
1913 	struct atmel_aes_dev *dd;  in atmel_aes_xts_init_tfm()  local
1916 	dd = atmel_aes_dev_alloc(&ctx->base);  in atmel_aes_xts_init_tfm()
1917 	if (!dd)  in atmel_aes_xts_init_tfm()
1927 	ctx->base.dd = dd;  in atmel_aes_xts_init_tfm()
1960 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
1961 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
1963 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
1965 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
1966 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
1969 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_authenc_complete()  argument
1971 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_complete()
1974 	if (err && (dd->flags & AES_FLAGS_OWN_SHA))  in atmel_aes_authenc_complete()
1976 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_complete()
1979 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)  in atmel_aes_authenc_start()  argument
1981 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_start()
1987 	atmel_aes_set_mode(dd, &rctx->base);  in atmel_aes_authenc_start()
1989 	err = atmel_aes_hw_init(dd);  in atmel_aes_authenc_start()
1991 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_start()
1994 					  atmel_aes_authenc_init, dd);  in atmel_aes_authenc_start()
1997 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_init()  argument
2000 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_init()
2004 		dd->is_async = true;  in atmel_aes_authenc_init()
2006 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_init()
2009 	dd->flags |= AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_init()
2015 				      atmel_aes_authenc_transfer, dd);  in atmel_aes_authenc_init()
2018 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_transfer()  argument
2021 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_transfer()
2023 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_transfer()
2029 		dd->is_async = true;  in atmel_aes_authenc_transfer()
2031 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_transfer()
2050 	atmel_aes_write_ctrl(dd, true, iv);  in atmel_aes_authenc_transfer()
2054 	atmel_aes_write(dd, AES_EMR, emr);  in atmel_aes_authenc_transfer()
2057 	return atmel_aes_dma_start(dd, src, dst, rctx->textlen,  in atmel_aes_authenc_transfer()
2061 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)  in atmel_aes_authenc_digest()  argument
2063 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_digest()
2067 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_digest()
2070 				       atmel_aes_authenc_final, dd);  in atmel_aes_authenc_digest()
2073 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_final()  argument
2076 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_final()
2079 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_final()
2084 		dd->is_async = true;  in atmel_aes_authenc_final()
2099 	return atmel_aes_complete(dd, err);  in atmel_aes_authenc_final()
2141 	struct atmel_aes_dev *dd;  in atmel_aes_authenc_init_tfm()  local
2143 	dd = atmel_aes_dev_alloc(&ctx->base);  in atmel_aes_authenc_init_tfm()
2144 	if (!dd)  in atmel_aes_authenc_init_tfm()
2153 	ctx->base.dd = dd;  in atmel_aes_authenc_init_tfm()
2217 	return atmel_aes_handle_queue(ctx->dd, &req->base);  in atmel_aes_authenc_crypt()
2316 static int atmel_aes_buff_init(struct atmel_aes_dev *dd)  in atmel_aes_buff_init()  argument
2318 	dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);  in atmel_aes_buff_init()
2319 	dd->buflen = ATMEL_AES_BUFFER_SIZE;  in atmel_aes_buff_init()
2320 	dd->buflen &= ~(AES_BLOCK_SIZE - 1);  in atmel_aes_buff_init()
2322 	if (!dd->buf) {  in atmel_aes_buff_init()
2323 		dev_err(dd->dev, "unable to alloc pages.\n");  in atmel_aes_buff_init()
2330 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_buff_cleanup()  argument
2332 	free_page((unsigned long)dd->buf);  in atmel_aes_buff_cleanup()
2335 static int atmel_aes_dma_init(struct atmel_aes_dev *dd)  in atmel_aes_dma_init()  argument
2340 	dd->src.chan = dma_request_chan(dd->dev, "tx");  in atmel_aes_dma_init()
2341 	if (IS_ERR(dd->src.chan)) {  in atmel_aes_dma_init()
2342 		ret = PTR_ERR(dd->src.chan);  in atmel_aes_dma_init()
2346 	dd->dst.chan = dma_request_chan(dd->dev, "rx");  in atmel_aes_dma_init()
2347 	if (IS_ERR(dd->dst.chan)) {  in atmel_aes_dma_init()
2348 		ret = PTR_ERR(dd->dst.chan);  in atmel_aes_dma_init()
2355 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_init()
2357 	dev_err(dd->dev, "no DMA channel available\n");  in atmel_aes_dma_init()
2361 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_dma_cleanup()  argument
2363 	dma_release_channel(dd->dst.chan);  in atmel_aes_dma_cleanup()
2364 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_cleanup()
2369 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_queue_task()  local
2371 	atmel_aes_handle_queue(dd, NULL);  in atmel_aes_queue_task()
2376 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_done_task()  local
2378 	dd->is_async = true;  in atmel_aes_done_task()
2379 	(void)dd->resume(dd);  in atmel_aes_done_task()
2400 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)  in atmel_aes_unregister_algs()  argument
2405 	if (dd->caps.has_authenc)  in atmel_aes_unregister_algs()
2410 	if (dd->caps.has_xts)  in atmel_aes_unregister_algs()
2413 	if (dd->caps.has_gcm)  in atmel_aes_unregister_algs()
2416 	if (dd->caps.has_cfb64)  in atmel_aes_unregister_algs()
2431 static int atmel_aes_register_algs(struct atmel_aes_dev *dd)  in atmel_aes_register_algs()  argument
2443 	if (dd->caps.has_cfb64) {  in atmel_aes_register_algs()
2451 	if (dd->caps.has_gcm) {  in atmel_aes_register_algs()
2459 	if (dd->caps.has_xts) {  in atmel_aes_register_algs()
2468 	if (dd->caps.has_authenc) {  in atmel_aes_register_algs()
2501 static void atmel_aes_get_cap(struct atmel_aes_dev *dd)  in atmel_aes_get_cap()  argument
2503 	dd->caps.has_dualbuff = 0;  in atmel_aes_get_cap()
2504 	dd->caps.has_cfb64 = 0;  in atmel_aes_get_cap()
2505 	dd->caps.has_gcm = 0;  in atmel_aes_get_cap()
2506 	dd->caps.has_xts = 0;  in atmel_aes_get_cap()
2507 	dd->caps.has_authenc = 0;  in atmel_aes_get_cap()
2508 	dd->caps.max_burst_size = 1;  in atmel_aes_get_cap()
2511 	switch (dd->hw_version & 0xff0) {  in atmel_aes_get_cap()
2514 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2515 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2516 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2517 		dd->caps.has_xts = 1;  in atmel_aes_get_cap()
2518 		dd->caps.has_authenc = 1;  in atmel_aes_get_cap()
2519 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2522 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2523 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2524 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2525 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2528 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2529 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2530 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2535 		dev_warn(dd->dev,  in atmel_aes_get_cap()