105 	struct atmel_aes_dev	*dd;  member
345 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)  in atmel_aes_read()  argument
347 	u32 value = readl_relaxed(dd->io_base + offset);  in atmel_aes_read()
350 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_read()
353 		dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,  in atmel_aes_read()
361 static inline void atmel_aes_write(struct atmel_aes_dev *dd,  in atmel_aes_write()  argument
365 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_write()
368 		dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,  in atmel_aes_write()
373 	writel_relaxed(value, dd->io_base + offset);  in atmel_aes_write()
376 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_n()  argument
380 		*value = atmel_aes_read(dd, offset);  in atmel_aes_read_n()
383 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_n()  argument
387 		atmel_aes_write(dd, offset, *value);  in atmel_aes_write_n()
390 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_block()  argument
393 	atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_read_block()
396 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_block()  argument
399 	atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_write_block()
402 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,  in atmel_aes_wait_for_data_ready()  argument
405 	u32 isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_wait_for_data_ready()
408 		return resume(dd);  in atmel_aes_wait_for_data_ready()
410 	dd->resume = resume;  in atmel_aes_wait_for_data_ready()
411 	atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_wait_for_data_ready()
427 	if (!ctx->dd) {  in atmel_aes_find_dev()
432 		ctx->dd = aes_dd;  in atmel_aes_find_dev()
434 		aes_dd = ctx->dd;  in atmel_aes_find_dev()
442 static int atmel_aes_hw_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_init()  argument
446 	err = clk_enable(dd->iclk);  in atmel_aes_hw_init()
450 	atmel_aes_write(dd, AES_CR, AES_CR_SWRST);  in atmel_aes_hw_init()
451 	atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);  in atmel_aes_hw_init()
456 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)  in atmel_aes_get_version()  argument
458 	return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;  in atmel_aes_get_version()
461 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_version_init()  argument
465 	err = atmel_aes_hw_init(dd);  in atmel_aes_hw_version_init()
469 	dd->hw_version = atmel_aes_get_version(dd);  in atmel_aes_hw_version_init()
471 	dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);  in atmel_aes_hw_version_init()
473 	clk_disable(dd->iclk);  in atmel_aes_hw_version_init()
477 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,  in atmel_aes_set_mode()  argument
481 	dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;  in atmel_aes_set_mode()
484 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)  in atmel_aes_is_encrypt()  argument
486 	return (dd->flags & AES_FLAGS_ENCRYPT);  in atmel_aes_is_encrypt()
490 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
493 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_complete()  argument
496 	if (dd->ctx->is_aead)  in atmel_aes_complete()
497 		atmel_aes_authenc_complete(dd, err);  in atmel_aes_complete()
500 	clk_disable(dd->iclk);  in atmel_aes_complete()
501 	dd->flags &= ~AES_FLAGS_BUSY;  in atmel_aes_complete()
503 	if (!dd->ctx->is_aead) {  in atmel_aes_complete()
505 			ablkcipher_request_cast(dd->areq);  in atmel_aes_complete()
524 	if (dd->is_async)  in atmel_aes_complete()
525 		dd->areq->complete(dd->areq, err);  in atmel_aes_complete()
527 	tasklet_schedule(&dd->queue_task);  in atmel_aes_complete()
532 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl_key()  argument
545 	valmr |= dd->flags & AES_FLAGS_MODE_MASK;  in atmel_aes_write_ctrl_key()
549 		if (dd->caps.has_dualbuff)  in atmel_aes_write_ctrl_key()
555 	atmel_aes_write(dd, AES_MR, valmr);  in atmel_aes_write_ctrl_key()
557 	atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));  in atmel_aes_write_ctrl_key()
560 		atmel_aes_write_block(dd, AES_IVR(0), iv);  in atmel_aes_write_ctrl_key()
563 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl()  argument
567 	atmel_aes_write_ctrl_key(dd, use_dma, iv,  in atmel_aes_write_ctrl()
568 				 dd->ctx->key, dd->ctx->keylen);  in atmel_aes_write_ctrl()
573 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)  in atmel_aes_cpu_transfer()  argument
579 		atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);  in atmel_aes_cpu_transfer()
580 		dd->data += 4;  in atmel_aes_cpu_transfer()
581 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_cpu_transfer()
583 		if (dd->datalen < AES_BLOCK_SIZE)  in atmel_aes_cpu_transfer()
586 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_transfer()
588 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_cpu_transfer()
590 			dd->resume = atmel_aes_cpu_transfer;  in atmel_aes_cpu_transfer()
591 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_cpu_transfer()
596 	if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_cpu_transfer()
597 				 dd->buf, dd->total))  in atmel_aes_cpu_transfer()
601 		return atmel_aes_complete(dd, err);  in atmel_aes_cpu_transfer()
603 	return dd->cpu_transfer_complete(dd);  in atmel_aes_cpu_transfer()
606 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,  in atmel_aes_cpu_start()  argument
617 	sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_cpu_start()
619 	dd->total = len;  in atmel_aes_cpu_start()
620 	dd->real_dst = dst;  in atmel_aes_cpu_start()
621 	dd->cpu_transfer_complete = resume;  in atmel_aes_cpu_start()
622 	dd->datalen = len + padlen;  in atmel_aes_cpu_start()
623 	dd->data = (u32 *)dd->buf;  in atmel_aes_cpu_start()
624 	atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_start()
625 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);  in atmel_aes_cpu_start()
633 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,  in atmel_aes_check_aligned()  argument
640 	if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
648 			if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
657 		if (!IS_ALIGNED(sg->length, dd->ctx->block_size))  in atmel_aes_check_aligned()
683 static int atmel_aes_map(struct atmel_aes_dev *dd,  in atmel_aes_map()  argument
691 	dd->total = len;  in atmel_aes_map()
692 	dd->src.sg = src;  in atmel_aes_map()
693 	dd->dst.sg = dst;  in atmel_aes_map()
694 	dd->real_dst = dst;  in atmel_aes_map()
696 	src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);  in atmel_aes_map()
700 		dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);  in atmel_aes_map()
702 		padlen = atmel_aes_padlen(len, dd->ctx->block_size);  in atmel_aes_map()
704 		if (dd->buflen < len + padlen)  in atmel_aes_map()
708 			sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_map()
709 			dd->src.sg = &dd->aligned_sg;  in atmel_aes_map()
710 			dd->src.nents = 1;  in atmel_aes_map()
711 			dd->src.remainder = 0;  in atmel_aes_map()
715 			dd->dst.sg = &dd->aligned_sg;  in atmel_aes_map()
716 			dd->dst.nents = 1;  in atmel_aes_map()
717 			dd->dst.remainder = 0;  in atmel_aes_map()
720 		sg_init_table(&dd->aligned_sg, 1);  in atmel_aes_map()
721 		sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);  in atmel_aes_map()
724 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_map()
725 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
727 		dd->dst.sg_len = dd->src.sg_len;  in atmel_aes_map()
728 		if (!dd->src.sg_len)  in atmel_aes_map()
731 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
733 		if (!dd->src.sg_len)  in atmel_aes_map()
736 		dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_map()
738 		if (!dd->dst.sg_len) {  in atmel_aes_map()
739 			dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
748 static void atmel_aes_unmap(struct atmel_aes_dev *dd)  in atmel_aes_unmap()  argument
750 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_unmap()
751 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
754 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
755 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
757 		dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_unmap()
760 		if (dd->dst.sg != &dd->aligned_sg)  in atmel_aes_unmap()
761 			atmel_aes_restore_sg(&dd->dst);  in atmel_aes_unmap()
763 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
766 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
767 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
770 	if (dd->dst.sg == &dd->aligned_sg)  in atmel_aes_unmap()
771 		sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_unmap()
772 				    dd->buf, dd->total);  in atmel_aes_unmap()
775 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_transfer_start()  argument
795 		dma = &dd->src;  in atmel_aes_dma_transfer_start()
797 		config.dst_addr = dd->phys_base + AES_IDATAR(0);  in atmel_aes_dma_transfer_start()
801 		dma = &dd->dst;  in atmel_aes_dma_transfer_start()
803 		config.src_addr = dd->phys_base + AES_ODATAR(0);  in atmel_aes_dma_transfer_start()
820 	desc->callback_param = dd;  in atmel_aes_dma_transfer_start()
827 static void atmel_aes_dma_transfer_stop(struct atmel_aes_dev *dd,  in atmel_aes_dma_transfer_stop()  argument
834 		dma = &dd->src;  in atmel_aes_dma_transfer_stop()
838 		dma = &dd->dst;  in atmel_aes_dma_transfer_stop()
848 static int atmel_aes_dma_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_start()  argument
858 	switch (dd->ctx->block_size) {  in atmel_aes_dma_start()
877 		maxburst = dd->caps.max_burst_size;  in atmel_aes_dma_start()
885 	err = atmel_aes_map(dd, src, dst, len);  in atmel_aes_dma_start()
889 	dd->resume = resume;  in atmel_aes_dma_start()
892 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,  in atmel_aes_dma_start()
898 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,  in atmel_aes_dma_start()
906 	atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM);  in atmel_aes_dma_start()
908 	atmel_aes_unmap(dd);  in atmel_aes_dma_start()
910 	return atmel_aes_complete(dd, err);  in atmel_aes_dma_start()
913 static void atmel_aes_dma_stop(struct atmel_aes_dev *dd)  in atmel_aes_dma_stop()  argument
915 	atmel_aes_dma_transfer_stop(dd, DMA_MEM_TO_DEV);  in atmel_aes_dma_stop()
916 	atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM);  in atmel_aes_dma_stop()
917 	atmel_aes_unmap(dd);  in atmel_aes_dma_stop()
922 	struct atmel_aes_dev *dd = data;  in atmel_aes_dma_callback()  local
924 	atmel_aes_dma_stop(dd);  in atmel_aes_dma_callback()
925 	dd->is_async = true;  in atmel_aes_dma_callback()
926 	(void)dd->resume(dd);  in atmel_aes_dma_callback()
929 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,  in atmel_aes_handle_queue()  argument
938 	spin_lock_irqsave(&dd->lock, flags);  in atmel_aes_handle_queue()
940 		ret = crypto_enqueue_request(&dd->queue, new_areq);  in atmel_aes_handle_queue()
941 	if (dd->flags & AES_FLAGS_BUSY) {  in atmel_aes_handle_queue()
942 		spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
945 	backlog = crypto_get_backlog(&dd->queue);  in atmel_aes_handle_queue()
946 	areq = crypto_dequeue_request(&dd->queue);  in atmel_aes_handle_queue()
948 		dd->flags |= AES_FLAGS_BUSY;  in atmel_aes_handle_queue()
949 	spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
959 	dd->areq = areq;  in atmel_aes_handle_queue()
960 	dd->ctx = ctx;  in atmel_aes_handle_queue()
962 	dd->is_async = start_async;  in atmel_aes_handle_queue()
965 	err = ctx->start(dd);  in atmel_aes_handle_queue()
972 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)  in atmel_aes_transfer_complete()  argument
974 	return atmel_aes_complete(dd, 0);  in atmel_aes_transfer_complete()
977 static int atmel_aes_start(struct atmel_aes_dev *dd)  in atmel_aes_start()  argument
979 	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);  in atmel_aes_start()
982 			dd->ctx->block_size != AES_BLOCK_SIZE);  in atmel_aes_start()
985 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_start()
987 	err = atmel_aes_hw_init(dd);  in atmel_aes_start()
989 		return atmel_aes_complete(dd, err);  in atmel_aes_start()
991 	atmel_aes_write_ctrl(dd, use_dma, req->info);  in atmel_aes_start()
993 		return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes,  in atmel_aes_start()
996 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes,  in atmel_aes_start()
1006 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)  in atmel_aes_ctr_transfer()  argument
1008 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_transfer()
1009 	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);  in atmel_aes_ctr_transfer()
1016 	ctx->offset += dd->total;  in atmel_aes_ctr_transfer()
1018 		return atmel_aes_transfer_complete(dd);  in atmel_aes_ctr_transfer()
1024 	if (dd->caps.has_ctr32) {  in atmel_aes_ctr_transfer()
1053 	atmel_aes_write_ctrl(dd, use_dma, ctx->iv);  in atmel_aes_ctr_transfer()
1064 		return atmel_aes_dma_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1067 	return atmel_aes_cpu_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1071 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)  in atmel_aes_ctr_start()  argument
1073 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_start()
1074 	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);  in atmel_aes_ctr_start()
1078 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_ctr_start()
1080 	err = atmel_aes_hw_init(dd);  in atmel_aes_ctr_start()
1082 		return atmel_aes_complete(dd, err);  in atmel_aes_ctr_start()
1086 	dd->total = 0;  in atmel_aes_ctr_start()
1087 	return atmel_aes_ctr_transfer(dd);  in atmel_aes_ctr_start()
1095 	struct atmel_aes_dev *dd;  in atmel_aes_crypt()  local
1120 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_crypt()
1121 	if (!dd)  in atmel_aes_crypt()
1134 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_crypt()
1451 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
1455 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
1456 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
1458 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
1459 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
1460 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
1461 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
1462 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
1463 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
1464 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
1472 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,  in atmel_aes_gcm_ghash()  argument
1477 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash()
1479 	dd->data = (u32 *)data;  in atmel_aes_gcm_ghash()
1480 	dd->datalen = datalen;  in atmel_aes_gcm_ghash()
1485 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_ghash()
1486 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);  in atmel_aes_gcm_ghash()
1489 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_init()  argument
1491 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_init()
1494 	atmel_aes_write(dd, AES_AADLENR, dd->total);  in atmel_aes_gcm_ghash_init()
1495 	atmel_aes_write(dd, AES_CLENR, 0);  in atmel_aes_gcm_ghash_init()
1499 		atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);  in atmel_aes_gcm_ghash_init()
1501 	return atmel_aes_gcm_ghash_finalize(dd);  in atmel_aes_gcm_ghash_init()
1504 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_finalize()  argument
1506 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_finalize()
1510 	while (dd->datalen > 0) {  in atmel_aes_gcm_ghash_finalize()
1511 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_ghash_finalize()
1512 		dd->data += 4;  in atmel_aes_gcm_ghash_finalize()
1513 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_ghash_finalize()
1515 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_ghash_finalize()
1517 			dd->resume = atmel_aes_gcm_ghash_finalize;  in atmel_aes_gcm_ghash_finalize()
1518 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_ghash_finalize()
1524 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);  in atmel_aes_gcm_ghash_finalize()
1526 	return ctx->ghash_resume(dd);  in atmel_aes_gcm_ghash_finalize()
1530 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)  in atmel_aes_gcm_start()  argument
1532 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_start()
1533 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_start()
1539 	u8 *data = dd->buf;  in atmel_aes_gcm_start()
1542 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_gcm_start()
1544 	err = atmel_aes_hw_init(dd);  in atmel_aes_gcm_start()
1546 		return atmel_aes_complete(dd, err);  in atmel_aes_gcm_start()
1551 		return atmel_aes_gcm_process(dd);  in atmel_aes_gcm_start()
1556 	if (datalen > dd->buflen)  in atmel_aes_gcm_start()
1557 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_start()
1563 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,  in atmel_aes_gcm_start()
1567 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)  in atmel_aes_gcm_process()  argument
1569 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_process()
1570 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_process()
1572 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_process()
1584 		dd->flags |= AES_FLAGS_GTAGEN;  in atmel_aes_gcm_process()
1586 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_process()
1587 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);  in atmel_aes_gcm_process()
1590 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)  in atmel_aes_gcm_length()  argument
1592 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_length()
1593 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_length()
1600 	atmel_aes_write_block(dd, AES_IVR(0), j0);  in atmel_aes_gcm_length()
1604 	atmel_aes_write(dd, AES_AADLENR, req->assoclen);  in atmel_aes_gcm_length()
1605 	atmel_aes_write(dd, AES_CLENR, ctx->textlen);  in atmel_aes_gcm_length()
1609 		dd->datalen = 0;  in atmel_aes_gcm_length()
1610 		return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1615 	if (unlikely(req->assoclen + padlen > dd->buflen))  in atmel_aes_gcm_length()
1616 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_length()
1617 	sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);  in atmel_aes_gcm_length()
1620 	dd->data = (u32 *)dd->buf;  in atmel_aes_gcm_length()
1621 	dd->datalen = req->assoclen + padlen;  in atmel_aes_gcm_length()
1622 	return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1625 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)  in atmel_aes_gcm_data()  argument
1627 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_data()
1628 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_data()
1634 	while (dd->datalen > 0) {  in atmel_aes_gcm_data()
1635 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_data()
1636 		dd->data += 4;  in atmel_aes_gcm_data()
1637 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_data()
1639 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_data()
1641 			dd->resume = atmel_aes_gcm_data;  in atmel_aes_gcm_data()
1642 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_data()
1649 		return atmel_aes_gcm_tag_init(dd);  in atmel_aes_gcm_data()
1658 		mr = atmel_aes_read(dd, AES_MR);  in atmel_aes_gcm_data()
1661 		if (dd->caps.has_dualbuff)  in atmel_aes_gcm_data()
1663 		atmel_aes_write(dd, AES_MR, mr);  in atmel_aes_gcm_data()
1665 		return atmel_aes_dma_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1669 	return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1673 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag_init()  argument
1675 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag_init()
1676 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_tag_init()
1677 	u64 *data = dd->buf;  in atmel_aes_gcm_tag_init()
1679 	if (likely(dd->flags & AES_FLAGS_GTAGEN)) {  in atmel_aes_gcm_tag_init()
1680 		if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {  in atmel_aes_gcm_tag_init()
1681 			dd->resume = atmel_aes_gcm_tag_init;  in atmel_aes_gcm_tag_init()
1682 			atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);  in atmel_aes_gcm_tag_init()
1686 		return atmel_aes_gcm_finalize(dd);  in atmel_aes_gcm_tag_init()
1690 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);  in atmel_aes_gcm_tag_init()
1695 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,  in atmel_aes_gcm_tag_init()
1699 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag()  argument
1701 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag()
1708 	flags = dd->flags;  in atmel_aes_gcm_tag()
1709 	dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);  in atmel_aes_gcm_tag()
1710 	dd->flags |= AES_FLAGS_CTR;  in atmel_aes_gcm_tag()
1711 	atmel_aes_write_ctrl(dd, false, ctx->j0);  in atmel_aes_gcm_tag()
1712 	dd->flags = flags;  in atmel_aes_gcm_tag()
1714 	atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);  in atmel_aes_gcm_tag()
1715 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);  in atmel_aes_gcm_tag()
1718 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_finalize()  argument
1720 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_finalize()
1721 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_finalize()
1723 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_finalize()
1728 	if (likely(dd->flags & AES_FLAGS_GTAGEN))  in atmel_aes_gcm_finalize()
1729 		atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1731 		atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1743 	return atmel_aes_complete(dd, err);  in atmel_aes_gcm_finalize()
1751 	struct atmel_aes_dev *dd;  in atmel_aes_gcm_crypt()  local
1757 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_gcm_crypt()
1758 	if (!dd)  in atmel_aes_gcm_crypt()
1764 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_gcm_crypt()
1855 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
1857 static int atmel_aes_xts_start(struct atmel_aes_dev *dd)  in atmel_aes_xts_start()  argument
1859 	struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);  in atmel_aes_xts_start()
1860 	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);  in atmel_aes_xts_start()
1865 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_xts_start()
1867 	err = atmel_aes_hw_init(dd);  in atmel_aes_xts_start()
1869 		return atmel_aes_complete(dd, err);  in atmel_aes_xts_start()
1872 	flags = dd->flags;  in atmel_aes_xts_start()
1873 	dd->flags &= ~AES_FLAGS_MODE_MASK;  in atmel_aes_xts_start()
1874 	dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);  in atmel_aes_xts_start()
1875 	atmel_aes_write_ctrl_key(dd, false, NULL,  in atmel_aes_xts_start()
1877 	dd->flags = flags;  in atmel_aes_xts_start()
1879 	atmel_aes_write_block(dd, AES_IDATAR(0), req->info);  in atmel_aes_xts_start()
1880 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);  in atmel_aes_xts_start()
1883 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)  in atmel_aes_xts_process_data()  argument
1885 	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);  in atmel_aes_xts_process_data()
1893 	atmel_aes_read_block(dd, AES_ODATAR(0), tweak);  in atmel_aes_xts_process_data()
1907 	atmel_aes_write_ctrl(dd, use_dma, NULL);  in atmel_aes_xts_process_data()
1908 	atmel_aes_write_block(dd, AES_TWR(0), tweak);  in atmel_aes_xts_process_data()
1909 	atmel_aes_write_block(dd, AES_ALPHAR(0), one);  in atmel_aes_xts_process_data()
1911 		return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes,  in atmel_aes_xts_process_data()
1914 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes,  in atmel_aes_xts_process_data()
1979 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
1980 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
1982 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
1984 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
1985 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
1988 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_authenc_complete()  argument
1990 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_complete()
1993 	if (err && (dd->flags & AES_FLAGS_OWN_SHA))  in atmel_aes_authenc_complete()
1995 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_complete()
1998 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)  in atmel_aes_authenc_start()  argument
2000 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_start()
2006 	atmel_aes_set_mode(dd, &rctx->base);  in atmel_aes_authenc_start()
2008 	err = atmel_aes_hw_init(dd);  in atmel_aes_authenc_start()
2010 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_start()
2013 					  atmel_aes_authenc_init, dd);  in atmel_aes_authenc_start()
2016 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_init()  argument
2019 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_init()
2023 		dd->is_async = true;  in atmel_aes_authenc_init()
2025 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_init()
2028 	dd->flags |= AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_init()
2034 				      atmel_aes_authenc_transfer, dd);  in atmel_aes_authenc_init()
2037 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_transfer()  argument
2040 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_transfer()
2042 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_transfer()
2048 		dd->is_async = true;  in atmel_aes_authenc_transfer()
2050 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_transfer()
2069 	atmel_aes_write_ctrl(dd, true, iv);  in atmel_aes_authenc_transfer()
2073 	atmel_aes_write(dd, AES_EMR, emr);  in atmel_aes_authenc_transfer()
2076 	return atmel_aes_dma_start(dd, src, dst, rctx->textlen,  in atmel_aes_authenc_transfer()
2080 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)  in atmel_aes_authenc_digest()  argument
2082 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_digest()
2086 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_digest()
2089 				       atmel_aes_authenc_final, dd);  in atmel_aes_authenc_digest()
2092 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_final()  argument
2095 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_final()
2098 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_final()
2103 		dd->is_async = true;  in atmel_aes_authenc_final()
2118 	return atmel_aes_complete(dd, err);  in atmel_aes_authenc_final()
2216 	struct atmel_aes_dev *dd;  in atmel_aes_authenc_crypt()  local
2235 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_authenc_crypt()
2236 	if (!dd)  in atmel_aes_authenc_crypt()
2239 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_authenc_crypt()
2358 static int atmel_aes_buff_init(struct atmel_aes_dev *dd)  in atmel_aes_buff_init()  argument
2360 	dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);  in atmel_aes_buff_init()
2361 	dd->buflen = ATMEL_AES_BUFFER_SIZE;  in atmel_aes_buff_init()
2362 	dd->buflen &= ~(AES_BLOCK_SIZE - 1);  in atmel_aes_buff_init()
2364 	if (!dd->buf) {  in atmel_aes_buff_init()
2365 		dev_err(dd->dev, "unable to alloc pages.\n");  in atmel_aes_buff_init()
2372 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_buff_cleanup()  argument
2374 	free_page((unsigned long)dd->buf);  in atmel_aes_buff_cleanup()
2389 static int atmel_aes_dma_init(struct atmel_aes_dev *dd,  in atmel_aes_dma_init()  argument
2400 	dd->src.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter,  in atmel_aes_dma_init()
2401 							slave, dd->dev, "tx");  in atmel_aes_dma_init()
2402 	if (!dd->src.chan)  in atmel_aes_dma_init()
2406 	dd->dst.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter,  in atmel_aes_dma_init()
2407 							slave, dd->dev, "rx");  in atmel_aes_dma_init()
2408 	if (!dd->dst.chan)  in atmel_aes_dma_init()
2414 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_init()
2416 	dev_warn(dd->dev, "no DMA channel available\n");  in atmel_aes_dma_init()
2420 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_dma_cleanup()  argument
2422 	dma_release_channel(dd->dst.chan);  in atmel_aes_dma_cleanup()
2423 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_cleanup()
2428 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_queue_task()  local
2430 	atmel_aes_handle_queue(dd, NULL);  in atmel_aes_queue_task()
2435 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_done_task()  local
2437 	dd->is_async = true;  in atmel_aes_done_task()
2438 	(void)dd->resume(dd);  in atmel_aes_done_task()
2459 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)  in atmel_aes_unregister_algs()  argument
2464 	if (dd->caps.has_authenc)  in atmel_aes_unregister_algs()
2469 	if (dd->caps.has_xts)  in atmel_aes_unregister_algs()
2472 	if (dd->caps.has_gcm)  in atmel_aes_unregister_algs()
2475 	if (dd->caps.has_cfb64)  in atmel_aes_unregister_algs()
2482 static int atmel_aes_register_algs(struct atmel_aes_dev *dd)  in atmel_aes_register_algs()  argument
2492 	if (dd->caps.has_cfb64) {  in atmel_aes_register_algs()
2498 	if (dd->caps.has_gcm) {  in atmel_aes_register_algs()
2504 	if (dd->caps.has_xts) {  in atmel_aes_register_algs()
2511 	if (dd->caps.has_authenc) {  in atmel_aes_register_algs()
2542 static void atmel_aes_get_cap(struct atmel_aes_dev *dd)  in atmel_aes_get_cap()  argument
2544 	dd->caps.has_dualbuff = 0;  in atmel_aes_get_cap()
2545 	dd->caps.has_cfb64 = 0;  in atmel_aes_get_cap()
2546 	dd->caps.has_ctr32 = 0;  in atmel_aes_get_cap()
2547 	dd->caps.has_gcm = 0;  in atmel_aes_get_cap()
2548 	dd->caps.has_xts = 0;  in atmel_aes_get_cap()
2549 	dd->caps.has_authenc = 0;  in atmel_aes_get_cap()
2550 	dd->caps.max_burst_size = 1;  in atmel_aes_get_cap()
2553 	switch (dd->hw_version & 0xff0) {  in atmel_aes_get_cap()
2555 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2556 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2557 		dd->caps.has_ctr32 = 1;  in atmel_aes_get_cap()
2558 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2559 		dd->caps.has_xts = 1;  in atmel_aes_get_cap()
2560 		dd->caps.has_authenc = 1;  in atmel_aes_get_cap()
2561 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2564 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2565 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2566 		dd->caps.has_ctr32 = 1;  in atmel_aes_get_cap()
2567 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2568 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2571 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2572 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2573 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2578 		dev_warn(dd->dev,  in atmel_aes_get_cap()