Lines Matching full:engine
81 struct spacc_engine *engine; member
95 struct spacc_engine *engine; member
138 struct spacc_engine *engine; member
146 struct spacc_engine *engine; member
186 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) in spacc_fifo_cmd_full() argument
188 u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); in spacc_fifo_cmd_full()
204 return is_cipher_ctx ? ctx->engine->cipher_ctx_base + in spacc_ctx_page_addr()
205 (indx * ctx->engine->cipher_pg_sz) : in spacc_ctx_page_addr()
206 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); in spacc_ctx_page_addr()
240 unsigned indx = ctx->engine->next_ctx++; in spacc_load_ctx()
246 ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; in spacc_load_ctx()
251 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
256 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
273 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, in spacc_sg_to_ddt() argument
287 dev_err(engine->dev, "Invalid numbers of SG.\n"); in spacc_sg_to_ddt()
290 mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
295 ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); in spacc_sg_to_ddt()
306 dma_unmap_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
314 struct spacc_engine *engine = req->engine; in spacc_aead_make_ddts() local
327 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_make_ddts()
337 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_make_ddts()
344 src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); in spacc_aead_make_ddts()
348 dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); in spacc_aead_make_ddts()
356 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
361 dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, in spacc_aead_make_ddts()
365 dma_unmap_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
370 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
403 dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); in spacc_aead_make_ddts()
405 dma_pool_free(engine->req_pool, src_ddt, req->src_addr); in spacc_aead_make_ddts()
418 struct spacc_engine *engine = aead_ctx->generic.engine; in spacc_aead_free_ddts() local
423 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_free_ddts()
428 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); in spacc_aead_free_ddts()
431 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_free_ddts()
434 dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); in spacc_aead_free_ddts()
436 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); in spacc_aead_free_ddts()
438 dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); in spacc_aead_free_ddts()
439 dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); in spacc_aead_free_ddts()
449 dev_err(req->engine->dev, "Invalid numbers of SG.\n"); in spacc_free_ddt()
453 dma_unmap_sg(req->engine->dev, payload, nents, dir); in spacc_free_ddt()
454 dma_pool_free(req->engine->req_pool, ddt, ddt_addr); in spacc_free_ddt()
564 struct spacc_engine *engine = ctx->generic.engine; in spacc_aead_submit() local
573 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_aead_submit()
574 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_aead_submit()
575 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_aead_submit()
587 writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_aead_submit()
588 writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_aead_submit()
589 writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); in spacc_aead_submit()
590 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_aead_submit()
591 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_aead_submit()
600 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_aead_submit()
602 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_aead_submit()
609 static void spacc_push(struct spacc_engine *engine) in spacc_push() argument
613 while (!list_empty(&engine->pending) && in spacc_push()
614 engine->in_flight + 1 <= engine->fifo_sz) { in spacc_push()
616 ++engine->in_flight; in spacc_push()
617 req = list_first_entry(&engine->pending, struct spacc_req, in spacc_push()
619 list_move_tail(&req->list, &engine->in_progress); in spacc_push()
626 * Setup an AEAD request for processing. This will configure the engine, load
634 struct spacc_engine *engine = to_spacc_aead(alg)->engine; in spacc_aead_setup() local
642 dev_req->engine = engine; in spacc_aead_setup()
653 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_aead_setup()
654 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_aead_setup()
655 engine->in_flight + 1 > engine->fifo_sz) { in spacc_aead_setup()
658 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
661 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
663 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
664 spacc_push(engine); in spacc_aead_setup()
666 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
701 struct spacc_engine *engine = spacc_alg->engine; in spacc_aead_cra_init() local
704 ctx->generic.engine = engine; in spacc_aead_cra_init()
789 * IPSec engine only supports 128 and 256 bit AES keys. If we get a in spacc_aes_setkey()
883 struct spacc_engine *engine = ctx->generic.engine; in spacc_ablk_submit() local
890 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_ablk_submit()
891 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_ablk_submit()
892 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_ablk_submit()
894 writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_ablk_submit()
895 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_ablk_submit()
896 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_ablk_submit()
897 writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_ablk_submit()
903 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_ablk_submit()
905 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_ablk_submit()
939 struct spacc_engine *engine = to_spacc_alg(alg)->engine; in spacc_ablk_setup() local
946 dev_req->engine = engine; in spacc_ablk_setup()
954 * Create the DDT's for the engine. If we share the same source and in spacc_ablk_setup()
958 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, in spacc_ablk_setup()
963 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
968 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
978 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_ablk_setup()
980 * Check if the engine will accept the operation now. If it won't then in spacc_ablk_setup()
984 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_ablk_setup()
985 engine->in_flight + 1 > engine->fifo_sz) { in spacc_ablk_setup()
988 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
991 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
993 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
994 spacc_push(engine); in spacc_ablk_setup()
996 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
1017 struct spacc_engine *engine = spacc_alg->engine; in spacc_ablk_cra_init() local
1020 ctx->generic.engine = engine; in spacc_ablk_cra_init()
1025 dev_warn(engine->dev, "failed to allocate fallback for %s\n", in spacc_ablk_cra_init()
1063 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) in spacc_fifo_stat_empty() argument
1065 return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & in spacc_fifo_stat_empty()
1069 static void spacc_process_done(struct spacc_engine *engine) in spacc_process_done() argument
1074 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_process_done()
1076 while (!spacc_fifo_stat_empty(engine)) { in spacc_process_done()
1077 req = list_first_entry(&engine->in_progress, struct spacc_req, in spacc_process_done()
1079 list_move_tail(&req->list, &engine->completed); in spacc_process_done()
1080 --engine->in_flight; in spacc_process_done()
1083 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); in spacc_process_done()
1084 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & in spacc_process_done()
1098 dev_warn(engine->dev, in spacc_process_done()
1104 dev_warn(engine->dev, in spacc_process_done()
1112 tasklet_schedule(&engine->complete); in spacc_process_done()
1114 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_process_done()
1119 struct spacc_engine *engine = (struct spacc_engine *)dev; in spacc_spacc_irq() local
1120 u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1122 writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1123 spacc_process_done(engine); in spacc_spacc_irq()
1130 struct spacc_engine *engine = from_timer(engine, t, packet_timeout); in spacc_packet_timeout() local
1132 spacc_process_done(engine); in spacc_packet_timeout()
1147 struct spacc_engine *engine = (struct spacc_engine *)data; in spacc_spacc_complete() local
1152 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_spacc_complete()
1154 list_splice_init(&engine->completed, &completed); in spacc_spacc_complete()
1155 spacc_push(engine); in spacc_spacc_complete()
1156 if (engine->in_flight) in spacc_spacc_complete()
1157 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_spacc_complete()
1159 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_spacc_complete()
1170 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_suspend() local
1177 clk_disable(engine->clk); in spacc_suspend()
1184 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_resume() local
1186 return clk_enable(engine->clk); in spacc_resume()
1204 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_show() local
1206 return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); in spacc_stat_irq_thresh_show()
1213 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_store() local
1219 thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); in spacc_stat_irq_thresh_store()
1221 engine->stat_irq_thresh = thresh; in spacc_stat_irq_thresh_store()
1222 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_stat_irq_thresh_store()
1223 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_stat_irq_thresh_store()
1621 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), in spacc_probe() local
1623 if (!engine) in spacc_probe()
1627 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; in spacc_probe()
1628 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; in spacc_probe()
1629 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; in spacc_probe()
1630 engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; in spacc_probe()
1631 engine->algs = ipsec_engine_algs; in spacc_probe()
1632 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); in spacc_probe()
1633 engine->aeads = ipsec_engine_aeads; in spacc_probe()
1634 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); in spacc_probe()
1636 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; in spacc_probe()
1637 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; in spacc_probe()
1638 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; in spacc_probe()
1639 engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; in spacc_probe()
1640 engine->algs = l2_engine_algs; in spacc_probe()
1641 engine->num_algs = ARRAY_SIZE(l2_engine_algs); in spacc_probe()
1646 engine->name = dev_name(&pdev->dev); in spacc_probe()
1648 engine->regs = devm_platform_ioremap_resource(pdev, 0); in spacc_probe()
1649 if (IS_ERR(engine->regs)) in spacc_probe()
1650 return PTR_ERR(engine->regs); in spacc_probe()
1654 dev_err(&pdev->dev, "no memory/irq resource for engine\n"); in spacc_probe()
1659 engine->name, engine)) { in spacc_probe()
1660 dev_err(engine->dev, "failed to request IRQ\n"); in spacc_probe()
1664 engine->dev = &pdev->dev; in spacc_probe()
1665 engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; in spacc_probe()
1666 engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; in spacc_probe()
1668 engine->req_pool = dmam_pool_create(engine->name, engine->dev, in spacc_probe()
1670 if (!engine->req_pool) in spacc_probe()
1673 spin_lock_init(&engine->hw_lock); in spacc_probe()
1675 engine->clk = clk_get(&pdev->dev, "ref"); in spacc_probe()
1676 if (IS_ERR(engine->clk)) { in spacc_probe()
1678 return PTR_ERR(engine->clk); in spacc_probe()
1681 if (clk_prepare_enable(engine->clk)) { in spacc_probe()
1697 engine->stat_irq_thresh = (engine->fifo_sz / 2); in spacc_probe()
1704 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_probe()
1705 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_probe()
1707 engine->regs + SPA_IRQ_EN_REG_OFFSET); in spacc_probe()
1709 timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); in spacc_probe()
1711 INIT_LIST_HEAD(&engine->pending); in spacc_probe()
1712 INIT_LIST_HEAD(&engine->completed); in spacc_probe()
1713 INIT_LIST_HEAD(&engine->in_progress); in spacc_probe()
1714 engine->in_flight = 0; in spacc_probe()
1715 tasklet_init(&engine->complete, spacc_spacc_complete, in spacc_probe()
1716 (unsigned long)engine); in spacc_probe()
1718 platform_set_drvdata(pdev, engine); in spacc_probe()
1721 INIT_LIST_HEAD(&engine->registered_algs); in spacc_probe()
1722 for (i = 0; i < engine->num_algs; ++i) { in spacc_probe()
1723 engine->algs[i].engine = engine; in spacc_probe()
1724 err = crypto_register_alg(&engine->algs[i].alg); in spacc_probe()
1726 list_add_tail(&engine->algs[i].entry, in spacc_probe()
1727 &engine->registered_algs); in spacc_probe()
1731 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1732 engine->algs[i].alg.cra_name); in spacc_probe()
1734 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1735 engine->algs[i].alg.cra_name); in spacc_probe()
1738 INIT_LIST_HEAD(&engine->registered_aeads); in spacc_probe()
1739 for (i = 0; i < engine->num_aeads; ++i) { in spacc_probe()
1740 engine->aeads[i].engine = engine; in spacc_probe()
1741 err = crypto_register_aead(&engine->aeads[i].alg); in spacc_probe()
1743 list_add_tail(&engine->aeads[i].entry, in spacc_probe()
1744 &engine->registered_aeads); in spacc_probe()
1748 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1749 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1751 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1752 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1758 del_timer_sync(&engine->packet_timeout); in spacc_probe()
1761 clk_disable_unprepare(engine->clk); in spacc_probe()
1763 clk_put(engine->clk); in spacc_probe()
1772 struct spacc_engine *engine = platform_get_drvdata(pdev); in spacc_remove() local
1774 del_timer_sync(&engine->packet_timeout); in spacc_remove()
1777 list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { in spacc_remove()
1782 list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { in spacc_remove()
1787 clk_disable_unprepare(engine->clk); in spacc_remove()
1788 clk_put(engine->clk); in spacc_remove()