104 	struct atmel_aes_dev	*dd;  member
345 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)  in atmel_aes_read()  argument
347 	u32 value = readl_relaxed(dd->io_base + offset);  in atmel_aes_read()
350 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_read()
353 		dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,  in atmel_aes_read()
361 static inline void atmel_aes_write(struct atmel_aes_dev *dd,  in atmel_aes_write()  argument
365 	if (dd->flags & AES_FLAGS_DUMP_REG) {  in atmel_aes_write()
368 		dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,  in atmel_aes_write()
373 	writel_relaxed(value, dd->io_base + offset);  in atmel_aes_write()
376 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_n()  argument
380 		*value = atmel_aes_read(dd, offset);  in atmel_aes_read_n()
383 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_n()  argument
387 		atmel_aes_write(dd, offset, *value);  in atmel_aes_write_n()
390 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_read_block()  argument
393 	atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_read_block()
396 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,  in atmel_aes_write_block()  argument
399 	atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));  in atmel_aes_write_block()
402 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,  in atmel_aes_wait_for_data_ready()  argument
405 	u32 isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_wait_for_data_ready()
408 		return resume(dd);  in atmel_aes_wait_for_data_ready()
410 	dd->resume = resume;  in atmel_aes_wait_for_data_ready()
411 	atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_wait_for_data_ready()
427 	if (!ctx->dd) {  in atmel_aes_find_dev()
432 		ctx->dd = aes_dd;  in atmel_aes_find_dev()
434 		aes_dd = ctx->dd;  in atmel_aes_find_dev()
442 static int atmel_aes_hw_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_init()  argument
446 	err = clk_enable(dd->iclk);  in atmel_aes_hw_init()
450 	atmel_aes_write(dd, AES_CR, AES_CR_SWRST);  in atmel_aes_hw_init()
451 	atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);  in atmel_aes_hw_init()
456 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)  in atmel_aes_get_version()  argument
458 	return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;  in atmel_aes_get_version()
461 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)  in atmel_aes_hw_version_init()  argument
465 	err = atmel_aes_hw_init(dd);  in atmel_aes_hw_version_init()
469 	dd->hw_version = atmel_aes_get_version(dd);  in atmel_aes_hw_version_init()
471 	dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);  in atmel_aes_hw_version_init()
473 	clk_disable(dd->iclk);  in atmel_aes_hw_version_init()
477 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,  in atmel_aes_set_mode()  argument
481 	dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;  in atmel_aes_set_mode()
484 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)  in atmel_aes_is_encrypt()  argument
486 	return (dd->flags & AES_FLAGS_ENCRYPT);  in atmel_aes_is_encrypt()
490 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
493 static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd)  in atmel_aes_set_iv_as_last_ciphertext_block()  argument
495 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_set_iv_as_last_ciphertext_block()
522 static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd)  in atmel_aes_ctr_update_req_iv()  argument
524 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_update_req_iv()
525 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_update_req_iv()
542 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_complete()  argument
544 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_complete()
548 	if (dd->ctx->is_aead)  in atmel_aes_complete()
549 		atmel_aes_authenc_complete(dd, err);  in atmel_aes_complete()
552 	clk_disable(dd->iclk);  in atmel_aes_complete()
553 	dd->flags &= ~AES_FLAGS_BUSY;  in atmel_aes_complete()
555 	if (!err && !dd->ctx->is_aead &&  in atmel_aes_complete()
558 			atmel_aes_set_iv_as_last_ciphertext_block(dd);  in atmel_aes_complete()
560 			atmel_aes_ctr_update_req_iv(dd);  in atmel_aes_complete()
563 	if (dd->is_async)  in atmel_aes_complete()
564 		dd->areq->complete(dd->areq, err);  in atmel_aes_complete()
566 	tasklet_schedule(&dd->queue_task);  in atmel_aes_complete()
571 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl_key()  argument
584 	valmr |= dd->flags & AES_FLAGS_MODE_MASK;  in atmel_aes_write_ctrl_key()
588 		if (dd->caps.has_dualbuff)  in atmel_aes_write_ctrl_key()
594 	atmel_aes_write(dd, AES_MR, valmr);  in atmel_aes_write_ctrl_key()
596 	atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));  in atmel_aes_write_ctrl_key()
599 		atmel_aes_write_block(dd, AES_IVR(0), iv);  in atmel_aes_write_ctrl_key()
602 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,  in atmel_aes_write_ctrl()  argument
606 	atmel_aes_write_ctrl_key(dd, use_dma, iv,  in atmel_aes_write_ctrl()
607 				 dd->ctx->key, dd->ctx->keylen);  in atmel_aes_write_ctrl()
612 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)  in atmel_aes_cpu_transfer()  argument
618 		atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);  in atmel_aes_cpu_transfer()
619 		dd->data += 4;  in atmel_aes_cpu_transfer()
620 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_cpu_transfer()
622 		if (dd->datalen < AES_BLOCK_SIZE)  in atmel_aes_cpu_transfer()
625 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_transfer()
627 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_cpu_transfer()
629 			dd->resume = atmel_aes_cpu_transfer;  in atmel_aes_cpu_transfer()
630 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_cpu_transfer()
635 	if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_cpu_transfer()
636 				 dd->buf, dd->total))  in atmel_aes_cpu_transfer()
640 		return atmel_aes_complete(dd, err);  in atmel_aes_cpu_transfer()
642 	return dd->cpu_transfer_complete(dd);  in atmel_aes_cpu_transfer()
645 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,  in atmel_aes_cpu_start()  argument
656 	sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_cpu_start()
658 	dd->total = len;  in atmel_aes_cpu_start()
659 	dd->real_dst = dst;  in atmel_aes_cpu_start()
660 	dd->cpu_transfer_complete = resume;  in atmel_aes_cpu_start()
661 	dd->datalen = len + padlen;  in atmel_aes_cpu_start()
662 	dd->data = (u32 *)dd->buf;  in atmel_aes_cpu_start()
663 	atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_cpu_start()
664 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);  in atmel_aes_cpu_start()
672 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,  in atmel_aes_check_aligned()  argument
679 	if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
687 			if (!IS_ALIGNED(len, dd->ctx->block_size))  in atmel_aes_check_aligned()
696 		if (!IS_ALIGNED(sg->length, dd->ctx->block_size))  in atmel_aes_check_aligned()
722 static int atmel_aes_map(struct atmel_aes_dev *dd,  in atmel_aes_map()  argument
730 	dd->total = len;  in atmel_aes_map()
731 	dd->src.sg = src;  in atmel_aes_map()
732 	dd->dst.sg = dst;  in atmel_aes_map()
733 	dd->real_dst = dst;  in atmel_aes_map()
735 	src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);  in atmel_aes_map()
739 		dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);  in atmel_aes_map()
741 		padlen = atmel_aes_padlen(len, dd->ctx->block_size);  in atmel_aes_map()
743 		if (dd->buflen < len + padlen)  in atmel_aes_map()
747 			sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);  in atmel_aes_map()
748 			dd->src.sg = &dd->aligned_sg;  in atmel_aes_map()
749 			dd->src.nents = 1;  in atmel_aes_map()
750 			dd->src.remainder = 0;  in atmel_aes_map()
754 			dd->dst.sg = &dd->aligned_sg;  in atmel_aes_map()
755 			dd->dst.nents = 1;  in atmel_aes_map()
756 			dd->dst.remainder = 0;  in atmel_aes_map()
759 		sg_init_table(&dd->aligned_sg, 1);  in atmel_aes_map()
760 		sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);  in atmel_aes_map()
763 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_map()
764 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
766 		dd->dst.sg_len = dd->src.sg_len;  in atmel_aes_map()
767 		if (!dd->src.sg_len)  in atmel_aes_map()
770 		dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
772 		if (!dd->src.sg_len)  in atmel_aes_map()
775 		dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_map()
777 		if (!dd->dst.sg_len) {  in atmel_aes_map()
778 			dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_map()
787 static void atmel_aes_unmap(struct atmel_aes_dev *dd)  in atmel_aes_unmap()  argument
789 	if (dd->src.sg == dd->dst.sg) {  in atmel_aes_unmap()
790 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
793 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
794 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
796 		dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,  in atmel_aes_unmap()
799 		if (dd->dst.sg != &dd->aligned_sg)  in atmel_aes_unmap()
800 			atmel_aes_restore_sg(&dd->dst);  in atmel_aes_unmap()
802 		dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,  in atmel_aes_unmap()
805 		if (dd->src.sg != &dd->aligned_sg)  in atmel_aes_unmap()
806 			atmel_aes_restore_sg(&dd->src);  in atmel_aes_unmap()
809 	if (dd->dst.sg == &dd->aligned_sg)  in atmel_aes_unmap()
810 		sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),  in atmel_aes_unmap()
811 				    dd->buf, dd->total);  in atmel_aes_unmap()
814 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_transfer_start()  argument
833 		dma = &dd->src;  in atmel_aes_dma_transfer_start()
835 		config.dst_addr = dd->phys_base + AES_IDATAR(0);  in atmel_aes_dma_transfer_start()
839 		dma = &dd->dst;  in atmel_aes_dma_transfer_start()
841 		config.src_addr = dd->phys_base + AES_ODATAR(0);  in atmel_aes_dma_transfer_start()
858 	desc->callback_param = dd;  in atmel_aes_dma_transfer_start()
865 static int atmel_aes_dma_start(struct atmel_aes_dev *dd,  in atmel_aes_dma_start()  argument
875 	switch (dd->ctx->block_size) {  in atmel_aes_dma_start()
894 		maxburst = dd->caps.max_burst_size;  in atmel_aes_dma_start()
902 	err = atmel_aes_map(dd, src, dst, len);  in atmel_aes_dma_start()
906 	dd->resume = resume;  in atmel_aes_dma_start()
909 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,  in atmel_aes_dma_start()
915 	err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,  in atmel_aes_dma_start()
923 	dmaengine_terminate_sync(dd->dst.chan);  in atmel_aes_dma_start()
925 	atmel_aes_unmap(dd);  in atmel_aes_dma_start()
927 	return atmel_aes_complete(dd, err);  in atmel_aes_dma_start()
932 	struct atmel_aes_dev *dd = data;  in atmel_aes_dma_callback()  local
934 	atmel_aes_unmap(dd);  in atmel_aes_dma_callback()
935 	dd->is_async = true;  in atmel_aes_dma_callback()
936 	(void)dd->resume(dd);  in atmel_aes_dma_callback()
939 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,  in atmel_aes_handle_queue()  argument
948 	spin_lock_irqsave(&dd->lock, flags);  in atmel_aes_handle_queue()
950 		ret = crypto_enqueue_request(&dd->queue, new_areq);  in atmel_aes_handle_queue()
951 	if (dd->flags & AES_FLAGS_BUSY) {  in atmel_aes_handle_queue()
952 		spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
955 	backlog = crypto_get_backlog(&dd->queue);  in atmel_aes_handle_queue()
956 	areq = crypto_dequeue_request(&dd->queue);  in atmel_aes_handle_queue()
958 		dd->flags |= AES_FLAGS_BUSY;  in atmel_aes_handle_queue()
959 	spin_unlock_irqrestore(&dd->lock, flags);  in atmel_aes_handle_queue()
969 	dd->areq = areq;  in atmel_aes_handle_queue()
970 	dd->ctx = ctx;  in atmel_aes_handle_queue()
972 	dd->is_async = start_async;  in atmel_aes_handle_queue()
975 	err = ctx->start(dd);  in atmel_aes_handle_queue()
982 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)  in atmel_aes_transfer_complete()  argument
984 	return atmel_aes_complete(dd, 0);  in atmel_aes_transfer_complete()
987 static int atmel_aes_start(struct atmel_aes_dev *dd)  in atmel_aes_start()  argument
989 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_start()
992 			dd->ctx->block_size != AES_BLOCK_SIZE);  in atmel_aes_start()
995 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_start()
997 	err = atmel_aes_hw_init(dd);  in atmel_aes_start()
999 		return atmel_aes_complete(dd, err);  in atmel_aes_start()
1001 	atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv);  in atmel_aes_start()
1003 		return atmel_aes_dma_start(dd, req->src, req->dst,  in atmel_aes_start()
1007 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,  in atmel_aes_start()
1011 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)  in atmel_aes_ctr_transfer()  argument
1013 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_transfer()
1014 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_transfer()
1022 	ctx->offset += dd->total;  in atmel_aes_ctr_transfer()
1024 		return atmel_aes_transfer_complete(dd);  in atmel_aes_ctr_transfer()
1049 	atmel_aes_write_ctrl(dd, use_dma, ctx->iv);  in atmel_aes_ctr_transfer()
1060 		return atmel_aes_dma_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1063 	return atmel_aes_cpu_start(dd, src, dst, datalen,  in atmel_aes_ctr_transfer()
1067 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)  in atmel_aes_ctr_start()  argument
1069 	struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);  in atmel_aes_ctr_start()
1070 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_ctr_start()
1074 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_ctr_start()
1076 	err = atmel_aes_hw_init(dd);  in atmel_aes_ctr_start()
1078 		return atmel_aes_complete(dd, err);  in atmel_aes_ctr_start()
1082 	dd->total = 0;  in atmel_aes_ctr_start()
1083 	return atmel_aes_ctr_transfer(dd);  in atmel_aes_ctr_start()
1091 	struct atmel_aes_dev *dd;  in atmel_aes_crypt()  local
1116 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_crypt()
1117 	if (!dd)  in atmel_aes_crypt()
1133 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_crypt()
1394 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
1398 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
1399 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
1401 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
1402 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
1403 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
1404 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
1405 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
1406 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
1407 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
1415 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,  in atmel_aes_gcm_ghash()  argument
1420 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash()
1422 	dd->data = (u32 *)data;  in atmel_aes_gcm_ghash()
1423 	dd->datalen = datalen;  in atmel_aes_gcm_ghash()
1428 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_ghash()
1429 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);  in atmel_aes_gcm_ghash()
1432 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_init()  argument
1434 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_init()
1437 	atmel_aes_write(dd, AES_AADLENR, dd->total);  in atmel_aes_gcm_ghash_init()
1438 	atmel_aes_write(dd, AES_CLENR, 0);  in atmel_aes_gcm_ghash_init()
1442 		atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);  in atmel_aes_gcm_ghash_init()
1444 	return atmel_aes_gcm_ghash_finalize(dd);  in atmel_aes_gcm_ghash_init()
1447 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_ghash_finalize()  argument
1449 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_ghash_finalize()
1453 	while (dd->datalen > 0) {  in atmel_aes_gcm_ghash_finalize()
1454 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_ghash_finalize()
1455 		dd->data += 4;  in atmel_aes_gcm_ghash_finalize()
1456 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_ghash_finalize()
1458 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_ghash_finalize()
1460 			dd->resume = atmel_aes_gcm_ghash_finalize;  in atmel_aes_gcm_ghash_finalize()
1461 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_ghash_finalize()
1467 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);  in atmel_aes_gcm_ghash_finalize()
1469 	return ctx->ghash_resume(dd);  in atmel_aes_gcm_ghash_finalize()
1473 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)  in atmel_aes_gcm_start()  argument
1475 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_start()
1476 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_start()
1482 	u8 *data = dd->buf;  in atmel_aes_gcm_start()
1485 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_gcm_start()
1487 	err = atmel_aes_hw_init(dd);  in atmel_aes_gcm_start()
1489 		return atmel_aes_complete(dd, err);  in atmel_aes_gcm_start()
1494 		return atmel_aes_gcm_process(dd);  in atmel_aes_gcm_start()
1499 	if (datalen > dd->buflen)  in atmel_aes_gcm_start()
1500 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_start()
1506 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,  in atmel_aes_gcm_start()
1510 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)  in atmel_aes_gcm_process()  argument
1512 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_process()
1513 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_process()
1515 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_process()
1527 		dd->flags |= AES_FLAGS_GTAGEN;  in atmel_aes_gcm_process()
1529 	atmel_aes_write_ctrl(dd, false, NULL);  in atmel_aes_gcm_process()
1530 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);  in atmel_aes_gcm_process()
1533 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)  in atmel_aes_gcm_length()  argument
1535 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_length()
1536 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_length()
1543 	atmel_aes_write_block(dd, AES_IVR(0), j0);  in atmel_aes_gcm_length()
1547 	atmel_aes_write(dd, AES_AADLENR, req->assoclen);  in atmel_aes_gcm_length()
1548 	atmel_aes_write(dd, AES_CLENR, ctx->textlen);  in atmel_aes_gcm_length()
1552 		dd->datalen = 0;  in atmel_aes_gcm_length()
1553 		return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1558 	if (unlikely(req->assoclen + padlen > dd->buflen))  in atmel_aes_gcm_length()
1559 		return atmel_aes_complete(dd, -EINVAL);  in atmel_aes_gcm_length()
1560 	sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);  in atmel_aes_gcm_length()
1563 	dd->data = (u32 *)dd->buf;  in atmel_aes_gcm_length()
1564 	dd->datalen = req->assoclen + padlen;  in atmel_aes_gcm_length()
1565 	return atmel_aes_gcm_data(dd);  in atmel_aes_gcm_length()
1568 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)  in atmel_aes_gcm_data()  argument
1570 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_data()
1571 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_data()
1577 	while (dd->datalen > 0) {  in atmel_aes_gcm_data()
1578 		atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);  in atmel_aes_gcm_data()
1579 		dd->data += 4;  in atmel_aes_gcm_data()
1580 		dd->datalen -= AES_BLOCK_SIZE;  in atmel_aes_gcm_data()
1582 		isr = atmel_aes_read(dd, AES_ISR);  in atmel_aes_gcm_data()
1584 			dd->resume = atmel_aes_gcm_data;  in atmel_aes_gcm_data()
1585 			atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);  in atmel_aes_gcm_data()
1592 		return atmel_aes_gcm_tag_init(dd);  in atmel_aes_gcm_data()
1601 		mr = atmel_aes_read(dd, AES_MR);  in atmel_aes_gcm_data()
1604 		if (dd->caps.has_dualbuff)  in atmel_aes_gcm_data()
1606 		atmel_aes_write(dd, AES_MR, mr);  in atmel_aes_gcm_data()
1608 		return atmel_aes_dma_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1612 	return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,  in atmel_aes_gcm_data()
1616 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag_init()  argument
1618 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag_init()
1619 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_tag_init()
1620 	__be64 *data = dd->buf;  in atmel_aes_gcm_tag_init()
1622 	if (likely(dd->flags & AES_FLAGS_GTAGEN)) {  in atmel_aes_gcm_tag_init()
1623 		if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {  in atmel_aes_gcm_tag_init()
1624 			dd->resume = atmel_aes_gcm_tag_init;  in atmel_aes_gcm_tag_init()
1625 			atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);  in atmel_aes_gcm_tag_init()
1629 		return atmel_aes_gcm_finalize(dd);  in atmel_aes_gcm_tag_init()
1633 	atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);  in atmel_aes_gcm_tag_init()
1638 	return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,  in atmel_aes_gcm_tag_init()
1642 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)  in atmel_aes_gcm_tag()  argument
1644 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_tag()
1651 	flags = dd->flags;  in atmel_aes_gcm_tag()
1652 	dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);  in atmel_aes_gcm_tag()
1653 	dd->flags |= AES_FLAGS_CTR;  in atmel_aes_gcm_tag()
1654 	atmel_aes_write_ctrl(dd, false, ctx->j0);  in atmel_aes_gcm_tag()
1655 	dd->flags = flags;  in atmel_aes_gcm_tag()
1657 	atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);  in atmel_aes_gcm_tag()
1658 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);  in atmel_aes_gcm_tag()
1661 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)  in atmel_aes_gcm_finalize()  argument
1663 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);  in atmel_aes_gcm_finalize()
1664 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_gcm_finalize()
1666 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_gcm_finalize()
1671 	if (likely(dd->flags & AES_FLAGS_GTAGEN))  in atmel_aes_gcm_finalize()
1672 		atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1674 		atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);  in atmel_aes_gcm_finalize()
1686 	return atmel_aes_complete(dd, err);  in atmel_aes_gcm_finalize()
1694 	struct atmel_aes_dev *dd;  in atmel_aes_gcm_crypt()  local
1700 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_gcm_crypt()
1701 	if (!dd)  in atmel_aes_gcm_crypt()
1707 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_gcm_crypt()
1778 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
1780 static int atmel_aes_xts_start(struct atmel_aes_dev *dd)  in atmel_aes_xts_start()  argument
1782 	struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);  in atmel_aes_xts_start()
1783 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_xts_start()
1788 	atmel_aes_set_mode(dd, rctx);  in atmel_aes_xts_start()
1790 	err = atmel_aes_hw_init(dd);  in atmel_aes_xts_start()
1792 		return atmel_aes_complete(dd, err);  in atmel_aes_xts_start()
1795 	flags = dd->flags;  in atmel_aes_xts_start()
1796 	dd->flags &= ~AES_FLAGS_MODE_MASK;  in atmel_aes_xts_start()
1797 	dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);  in atmel_aes_xts_start()
1798 	atmel_aes_write_ctrl_key(dd, false, NULL,  in atmel_aes_xts_start()
1800 	dd->flags = flags;  in atmel_aes_xts_start()
1802 	atmel_aes_write_block(dd, AES_IDATAR(0), req->iv);  in atmel_aes_xts_start()
1803 	return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);  in atmel_aes_xts_start()
1806 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)  in atmel_aes_xts_process_data()  argument
1808 	struct skcipher_request *req = skcipher_request_cast(dd->areq);  in atmel_aes_xts_process_data()
1816 	atmel_aes_read_block(dd, AES_ODATAR(0), tweak);  in atmel_aes_xts_process_data()
1830 	atmel_aes_write_ctrl(dd, use_dma, NULL);  in atmel_aes_xts_process_data()
1831 	atmel_aes_write_block(dd, AES_TWR(0), tweak);  in atmel_aes_xts_process_data()
1832 	atmel_aes_write_block(dd, AES_ALPHAR(0), one);  in atmel_aes_xts_process_data()
1834 		return atmel_aes_dma_start(dd, req->src, req->dst,  in atmel_aes_xts_process_data()
1838 	return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,  in atmel_aes_xts_process_data()
1897 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
1898 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
1900 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
1902 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
1903 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
1906 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)  in atmel_aes_authenc_complete()  argument
1908 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_complete()
1911 	if (err && (dd->flags & AES_FLAGS_OWN_SHA))  in atmel_aes_authenc_complete()
1913 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_complete()
1916 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)  in atmel_aes_authenc_start()  argument
1918 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_start()
1924 	atmel_aes_set_mode(dd, &rctx->base);  in atmel_aes_authenc_start()
1926 	err = atmel_aes_hw_init(dd);  in atmel_aes_authenc_start()
1928 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_start()
1931 					  atmel_aes_authenc_init, dd);  in atmel_aes_authenc_start()
1934 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_init()  argument
1937 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_init()
1941 		dd->is_async = true;  in atmel_aes_authenc_init()
1943 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_init()
1946 	dd->flags |= AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_init()
1952 				      atmel_aes_authenc_transfer, dd);  in atmel_aes_authenc_init()
1955 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_transfer()  argument
1958 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_transfer()
1960 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_transfer()
1966 		dd->is_async = true;  in atmel_aes_authenc_transfer()
1968 		return atmel_aes_complete(dd, err);  in atmel_aes_authenc_transfer()
1987 	atmel_aes_write_ctrl(dd, true, iv);  in atmel_aes_authenc_transfer()
1991 	atmel_aes_write(dd, AES_EMR, emr);  in atmel_aes_authenc_transfer()
1994 	return atmel_aes_dma_start(dd, src, dst, rctx->textlen,  in atmel_aes_authenc_transfer()
1998 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)  in atmel_aes_authenc_digest()  argument
2000 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_digest()
2004 	dd->flags &= ~AES_FLAGS_OWN_SHA;  in atmel_aes_authenc_digest()
2007 				       atmel_aes_authenc_final, dd);  in atmel_aes_authenc_digest()
2010 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,  in atmel_aes_authenc_final()  argument
2013 	struct aead_request *req = aead_request_cast(dd->areq);  in atmel_aes_authenc_final()
2016 	bool enc = atmel_aes_is_encrypt(dd);  in atmel_aes_authenc_final()
2021 		dd->is_async = true;  in atmel_aes_authenc_final()
2036 	return atmel_aes_complete(dd, err);  in atmel_aes_authenc_final()
2130 	struct atmel_aes_dev *dd;  in atmel_aes_authenc_crypt()  local
2149 	dd = atmel_aes_find_dev(ctx);  in atmel_aes_authenc_crypt()
2150 	if (!dd)  in atmel_aes_authenc_crypt()
2153 	return atmel_aes_handle_queue(dd, &req->base);  in atmel_aes_authenc_crypt()
2252 static int atmel_aes_buff_init(struct atmel_aes_dev *dd)  in atmel_aes_buff_init()  argument
2254 	dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);  in atmel_aes_buff_init()
2255 	dd->buflen = ATMEL_AES_BUFFER_SIZE;  in atmel_aes_buff_init()
2256 	dd->buflen &= ~(AES_BLOCK_SIZE - 1);  in atmel_aes_buff_init()
2258 	if (!dd->buf) {  in atmel_aes_buff_init()
2259 		dev_err(dd->dev, "unable to alloc pages.\n");  in atmel_aes_buff_init()
2266 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_buff_cleanup()  argument
2268 	free_page((unsigned long)dd->buf);  in atmel_aes_buff_cleanup()
2271 static int atmel_aes_dma_init(struct atmel_aes_dev *dd)  in atmel_aes_dma_init()  argument
2276 	dd->src.chan = dma_request_chan(dd->dev, "tx");  in atmel_aes_dma_init()
2277 	if (IS_ERR(dd->src.chan)) {  in atmel_aes_dma_init()
2278 		ret = PTR_ERR(dd->src.chan);  in atmel_aes_dma_init()
2282 	dd->dst.chan = dma_request_chan(dd->dev, "rx");  in atmel_aes_dma_init()
2283 	if (IS_ERR(dd->dst.chan)) {  in atmel_aes_dma_init()
2284 		ret = PTR_ERR(dd->dst.chan);  in atmel_aes_dma_init()
2291 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_init()
2293 	dev_err(dd->dev, "no DMA channel available\n");  in atmel_aes_dma_init()
2297 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)  in atmel_aes_dma_cleanup()  argument
2299 	dma_release_channel(dd->dst.chan);  in atmel_aes_dma_cleanup()
2300 	dma_release_channel(dd->src.chan);  in atmel_aes_dma_cleanup()
2305 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_queue_task()  local
2307 	atmel_aes_handle_queue(dd, NULL);  in atmel_aes_queue_task()
2312 	struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;  in atmel_aes_done_task()  local
2314 	dd->is_async = true;  in atmel_aes_done_task()
2315 	(void)dd->resume(dd);  in atmel_aes_done_task()
2336 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)  in atmel_aes_unregister_algs()  argument
2341 	if (dd->caps.has_authenc)  in atmel_aes_unregister_algs()
2346 	if (dd->caps.has_xts)  in atmel_aes_unregister_algs()
2349 	if (dd->caps.has_gcm)  in atmel_aes_unregister_algs()
2352 	if (dd->caps.has_cfb64)  in atmel_aes_unregister_algs()
2367 static int atmel_aes_register_algs(struct atmel_aes_dev *dd)  in atmel_aes_register_algs()  argument
2379 	if (dd->caps.has_cfb64) {  in atmel_aes_register_algs()
2387 	if (dd->caps.has_gcm) {  in atmel_aes_register_algs()
2395 	if (dd->caps.has_xts) {  in atmel_aes_register_algs()
2404 	if (dd->caps.has_authenc) {  in atmel_aes_register_algs()
2437 static void atmel_aes_get_cap(struct atmel_aes_dev *dd)  in atmel_aes_get_cap()  argument
2439 	dd->caps.has_dualbuff = 0;  in atmel_aes_get_cap()
2440 	dd->caps.has_cfb64 = 0;  in atmel_aes_get_cap()
2441 	dd->caps.has_gcm = 0;  in atmel_aes_get_cap()
2442 	dd->caps.has_xts = 0;  in atmel_aes_get_cap()
2443 	dd->caps.has_authenc = 0;  in atmel_aes_get_cap()
2444 	dd->caps.max_burst_size = 1;  in atmel_aes_get_cap()
2447 	switch (dd->hw_version & 0xff0) {  in atmel_aes_get_cap()
2449 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2450 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2451 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2452 		dd->caps.has_xts = 1;  in atmel_aes_get_cap()
2453 		dd->caps.has_authenc = 1;  in atmel_aes_get_cap()
2454 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2457 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2458 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2459 		dd->caps.has_gcm = 1;  in atmel_aes_get_cap()
2460 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2463 		dd->caps.has_dualbuff = 1;  in atmel_aes_get_cap()
2464 		dd->caps.has_cfb64 = 1;  in atmel_aes_get_cap()
2465 		dd->caps.max_burst_size = 4;  in atmel_aes_get_cap()
2470 		dev_warn(dd->dev,  in atmel_aes_get_cap()