88 #define A7_PDMA_IN_CMD_FLUSH_STAT		BIT(1)
150 	unsigned char short_descr : 1;
151 	unsigned char pad1        : 1;
152 	unsigned char eop         : 1;
153 	unsigned char intr        : 1;
155 	unsigned char pad2        : 1;
176 	unsigned char pad1        : 1;
177 	unsigned char pad2        : 1;
178 	unsigned char eop         : 1;
189 #define ARTPEC6_CRYPTO_HASH_SHA1	1
193 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB	1
213  *  |  0   |  1   |  2   |~~| 11 EOP|  12  |  ....
439 				   1,  in artpec6_crypto_copy_bounce_buffers()
451 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_busy()  local
452 	int fifo_count = ac->pending_count;  in artpec6_crypto_busy()
459 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_submit()  local
462 	spin_lock_bh(&ac->queue_lock);  in artpec6_crypto_submit()
465 		list_add_tail(&req->list, &ac->pending);  in artpec6_crypto_submit()
469 		list_add_tail(&req->list, &ac->queue);  in artpec6_crypto_submit()
474 	spin_unlock_bh(&ac->queue_lock);  in artpec6_crypto_submit()
481 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_start_dma()  local
482 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_start_dma()
483 	void __iomem *base = ac->base;  in artpec6_crypto_start_dma()
490 	ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |  in artpec6_crypto_start_dma()
493 	statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |  in artpec6_crypto_start_dma()
496 	outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |  in artpec6_crypto_start_dma()
512 	ac->pending_count++;  in artpec6_crypto_start_dma()
529 	return should_fail(&artpec6_crypto_fail_dma_array_full, 1);  in fault_inject_dma_descr()
570  * @len: The length of the data, must be between 1 to 7 bytes
588 	} else if (len > 7 || len < 1) {  in artpec6_crypto_setup_out_descr_short()
594 	d->ctrl.short_descr = 1;  in artpec6_crypto_setup_out_descr_short()
663 	dma->stat[dma->in_cnt - 1] = 0;  in artpec6_crypto_dma_map_descs()
792 	base = bbuf + 1;  in artpec6_crypto_alloc_bounce()
853 			chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);  in artpec6_crypto_setup_sg_descrs_in()
905 			sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,  in artpec6_crypto_setup_sg_descrs_out()
964 	d = &dma->out[dma->out_cnt-1];  in artpec6_crypto_terminate_out_descrs()
965 	d->ctrl.eop = 1;  in artpec6_crypto_terminate_out_descrs()
987 	d = &dma->in[dma->in_cnt-1];  in artpec6_crypto_terminate_in_descrs()
988 	d->ctrl.intr = 1;  in artpec6_crypto_terminate_in_descrs()
1022 	target -= 1;  in create_hash_pad()
1023 	diff = dgstlen & (mod - 1);  in create_hash_pad()
1026 	memset(dst + 1, 0, pad_bytes);  in create_hash_pad()
1030 		memset(dst + 1 + pad_bytes, 0, 8);  in create_hash_pad()
1031 		memcpy(dst + 1 + pad_bytes + 8, &bits, 8);  in create_hash_pad()
1033 		memcpy(dst + 1 + pad_bytes, &bits, 8);  in create_hash_pad()
1036 	return pad_bytes + size_bytes + 1;  in create_hash_pad()
1045 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_common_init()  local
1051 	common->dma = kmem_cache_alloc(ac->dma_cache, flags);  in artpec6_crypto_common_init()
1074 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_common_destroy()  local
1078 	kmem_cache_free(ac->dma_cache, common->dma);  in artpec6_crypto_common_destroy()
1145 		req_ctx->decrypt = 1;  in artpec6_crypto_decrypt()
1317 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_hash()  local
1318 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_hash()
1386 	error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_hash()
1668 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_crypto()  local
1669 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_crypto()
1773 	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_crypto()
1803 							     ac->pad_buffer,  in artpec6_crypto_prepare_crypto()
1809 							    ac->pad_buffer, pad,  in artpec6_crypto_prepare_crypto()
1836 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_aead()  local
1837 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_aead()
1899 	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_aead()
1939 							     ac->zero_buffer,  in artpec6_crypto_prepare_aead()
1956 							     ac->zero_buffer,  in artpec6_crypto_prepare_aead()
1990 							    ac->pad_buffer,  in artpec6_crypto_prepare_aead()
2021 					ac->pad_buffer,  in artpec6_crypto_prepare_aead()
2041 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac,  in artpec6_crypto_process_queue()  argument
2046 	while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {  in artpec6_crypto_process_queue()
2047 		req = list_first_entry(&ac->queue,  in artpec6_crypto_process_queue()
2050 		list_move_tail(&req->list, &ac->pending);  in artpec6_crypto_process_queue()
2063 	if (ac->pending_count)  in artpec6_crypto_process_queue()
2064 		mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));  in artpec6_crypto_process_queue()
2066 		del_timer(&ac->timer);  in artpec6_crypto_process_queue()
2071 	struct artpec6_crypto *ac = from_timer(ac, t, timer);  in artpec6_crypto_timeout()  local
2075 	tasklet_schedule(&ac->task);  in artpec6_crypto_timeout()
2080 	struct artpec6_crypto *ac = (struct artpec6_crypto *)data;  in artpec6_crypto_task()  local
2089 	if (list_empty(&ac->pending)) {  in artpec6_crypto_task()
2094 	spin_lock(&ac->queue_lock);  in artpec6_crypto_task()
2096 	list_for_each_entry_safe(req, n, &ac->pending, list) {  in artpec6_crypto_task()
2101 		stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1);  in artpec6_crypto_task()
2107 		stat = req->dma->stat[req->dma->in_cnt-1];  in artpec6_crypto_task()
2118 		if (should_fail(&artpec6_crypto_fail_status_read, 1))  in artpec6_crypto_task()
2126 		ac->pending_count--;  in artpec6_crypto_task()
2129 	artpec6_crypto_process_queue(ac, &complete_in_progress);  in artpec6_crypto_task()
2131 	spin_unlock(&ac->queue_lock);  in artpec6_crypto_task()
2204 			print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,  in artpec6_crypto_complete_aead()
2206 			print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,  in artpec6_crypto_complete_aead()
2260 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_init_hash()  local
2261 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_init_hash()
2375 	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);  in artpec6_crypto_hmac_sha256_init()
2382 	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);  in artpec6_crypto_hmac_sha256_digest()
2437 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_hash_export()  local
2438 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_hash_export()
2465 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_hash_import()  local
2466 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_hash_import()
2487 static int init_crypto_hw(struct artpec6_crypto *ac)  in init_crypto_hw()  argument
2489 	enum artpec6_crypto_variant variant = ac->variant;  in init_crypto_hw()
2490 	void __iomem *base = ac->base;  in init_crypto_hw()
2547 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)  in artpec6_crypto_disable_hw()  argument
2549 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_disable_hw()
2550 	void __iomem *base = ac->base;  in artpec6_crypto_disable_hw()
2568 	struct artpec6_crypto *ac = dev_id;  in artpec6_crypto_irq()  local
2569 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_irq()
2570 	void __iomem *base = ac->base;  in artpec6_crypto_irq()
2611 		tasklet_schedule(&ac->task);  in artpec6_crypto_irq()
2620 	/* SHA-1 */
2727 			.cra_blocksize = 1,
2771 			.cra_blocksize = 1,
2803 			.cra_blocksize = 1,
2853 	struct artpec6_crypto *ac;  in artpec6_crypto_probe()  local
2876 	ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),  in artpec6_crypto_probe()
2878 	if (!ac)  in artpec6_crypto_probe()
2881 	platform_set_drvdata(pdev, ac);  in artpec6_crypto_probe()
2882 	ac->variant = variant;  in artpec6_crypto_probe()
2884 	spin_lock_init(&ac->queue_lock);  in artpec6_crypto_probe()
2885 	INIT_LIST_HEAD(&ac->queue);  in artpec6_crypto_probe()
2886 	INIT_LIST_HEAD(&ac->pending);  in artpec6_crypto_probe()
2887 	timer_setup(&ac->timer, artpec6_crypto_timeout, 0);  in artpec6_crypto_probe()
2889 	ac->base = base;  in artpec6_crypto_probe()
2891 	ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",  in artpec6_crypto_probe()
2896 	if (!ac->dma_cache)  in artpec6_crypto_probe()
2903 	tasklet_init(&ac->task, artpec6_crypto_task,  in artpec6_crypto_probe()
2904 		     (unsigned long)ac);  in artpec6_crypto_probe()
2906 	ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,  in artpec6_crypto_probe()
2908 	if (!ac->pad_buffer)  in artpec6_crypto_probe()
2910 	ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);  in artpec6_crypto_probe()
2912 	ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,  in artpec6_crypto_probe()
2914 	if (!ac->zero_buffer)  in artpec6_crypto_probe()
2916 	ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);  in artpec6_crypto_probe()
2918 	err = init_crypto_hw(ac);  in artpec6_crypto_probe()
2923 			       "artpec6-crypto", ac);  in artpec6_crypto_probe()
2954 	artpec6_crypto_disable_hw(ac);  in artpec6_crypto_probe()
2956 	kmem_cache_destroy(ac->dma_cache);  in artpec6_crypto_probe()
2962 	struct artpec6_crypto *ac = platform_get_drvdata(pdev);  in artpec6_crypto_remove()  local
2969 	tasklet_disable(&ac->task);  in artpec6_crypto_remove()
2970 	devm_free_irq(&pdev->dev, irq, ac);  in artpec6_crypto_remove()
2971 	tasklet_kill(&ac->task);  in artpec6_crypto_remove()
2972 	del_timer_sync(&ac->timer);  in artpec6_crypto_remove()
2974 	artpec6_crypto_disable_hw(ac);  in artpec6_crypto_remove()
2976 	kmem_cache_destroy(ac->dma_cache);  in artpec6_crypto_remove()