Lines Matching +full:sun8i +full:- +full:a83t +full:- +full:crypto
1 // SPDX-License-Identifier: GPL-2.0
3 * sun8i-ss-cipher.c - hardware cryptographic offloader for
4 * Allwinner A80/A83T SoC
6 * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com>
15 #include <linux/crypto.h>
16 #include <linux/dma-mapping.h>
19 #include <crypto/scatterwalk.h>
20 #include <crypto/internal/skcipher.h>
21 #include "sun8i-ss.h"
28 struct scatterlist *in_sg = areq->src; in sun8i_ss_need_fallback()
29 struct scatterlist *out_sg = areq->dst; in sun8i_ss_need_fallback()
33 if (areq->cryptlen == 0 || areq->cryptlen % 16) { in sun8i_ss_need_fallback()
34 algt->stat_fb_len++; in sun8i_ss_need_fallback()
38 if (sg_nents_for_len(areq->src, areq->cryptlen) > 8 || in sun8i_ss_need_fallback()
39 sg_nents_for_len(areq->dst, areq->cryptlen) > 8) { in sun8i_ss_need_fallback()
40 algt->stat_fb_sgnum++; in sun8i_ss_need_fallback()
44 len = areq->cryptlen; in sun8i_ss_need_fallback()
45 sg = areq->src; in sun8i_ss_need_fallback()
47 todo = min(len, sg->length); in sun8i_ss_need_fallback()
49 algt->stat_fb_sglen++; in sun8i_ss_need_fallback()
52 if (!IS_ALIGNED(sg->offset, 16)) { in sun8i_ss_need_fallback()
53 algt->stat_fb_align++; in sun8i_ss_need_fallback()
56 len -= todo; in sun8i_ss_need_fallback()
59 len = areq->cryptlen; in sun8i_ss_need_fallback()
60 sg = areq->dst; in sun8i_ss_need_fallback()
62 todo = min(len, sg->length); in sun8i_ss_need_fallback()
64 algt->stat_fb_sglen++; in sun8i_ss_need_fallback()
67 if (!IS_ALIGNED(sg->offset, 16)) { in sun8i_ss_need_fallback()
68 algt->stat_fb_align++; in sun8i_ss_need_fallback()
71 len -= todo; in sun8i_ss_need_fallback()
76 in_sg = areq->src; in sun8i_ss_need_fallback()
77 out_sg = areq->dst; in sun8i_ss_need_fallback()
79 if (in_sg->length != out_sg->length) in sun8i_ss_need_fallback()
101 algt->stat_fb++; in sun8i_ss_cipher_fallback()
103 skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); in sun8i_ss_cipher_fallback()
104 skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, in sun8i_ss_cipher_fallback()
105 areq->base.complete, areq->base.data); in sun8i_ss_cipher_fallback()
106 skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, in sun8i_ss_cipher_fallback()
107 areq->cryptlen, areq->iv); in sun8i_ss_cipher_fallback()
108 if (rctx->op_dir & SS_DECRYPTION) in sun8i_ss_cipher_fallback()
109 err = crypto_skcipher_decrypt(&rctx->fallback_req); in sun8i_ss_cipher_fallback()
111 err = crypto_skcipher_encrypt(&rctx->fallback_req); in sun8i_ss_cipher_fallback()
119 struct sun8i_ss_dev *ss = op->ss; in sun8i_ss_setup_ivs()
121 struct scatterlist *sg = areq->src; in sun8i_ss_setup_ivs()
123 unsigned int len = areq->cryptlen; in sun8i_ss_setup_ivs()
125 struct sun8i_ss_flow *sf = &ss->flows[rctx->flow]; in sun8i_ss_setup_ivs()
130 rctx->ivlen = ivsize; in sun8i_ss_setup_ivs()
131 if (rctx->op_dir & SS_DECRYPTION) { in sun8i_ss_setup_ivs()
132 offset = areq->cryptlen - ivsize; in sun8i_ss_setup_ivs()
133 scatterwalk_map_and_copy(sf->biv, areq->src, offset, in sun8i_ss_setup_ivs()
137 /* we need to copy all IVs from source in case DMA is bi-directionnal */ in sun8i_ss_setup_ivs()
144 memcpy(sf->iv[0], areq->iv, ivsize); in sun8i_ss_setup_ivs()
145 a = dma_map_single(ss->dev, sf->iv[i], ivsize, DMA_TO_DEVICE); in sun8i_ss_setup_ivs()
146 if (dma_mapping_error(ss->dev, a)) { in sun8i_ss_setup_ivs()
147 memzero_explicit(sf->iv[i], ivsize); in sun8i_ss_setup_ivs()
148 dev_err(ss->dev, "Cannot DMA MAP IV\n"); in sun8i_ss_setup_ivs()
149 err = -EFAULT; in sun8i_ss_setup_ivs()
152 rctx->p_iv[i] = a; in sun8i_ss_setup_ivs()
154 if (rctx->op_dir & SS_ENCRYPTION) in sun8i_ss_setup_ivs()
157 len -= todo; in sun8i_ss_setup_ivs()
160 offset = sg->length - ivsize; in sun8i_ss_setup_ivs()
161 scatterwalk_map_and_copy(sf->iv[i], sg, offset, ivsize, 0); in sun8i_ss_setup_ivs()
163 rctx->niv = i; in sun8i_ss_setup_ivs()
169 i--; in sun8i_ss_setup_ivs()
171 dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE); in sun8i_ss_setup_ivs()
172 memzero_explicit(sf->iv[i], ivsize); in sun8i_ss_setup_ivs()
173 i--; in sun8i_ss_setup_ivs()
182 struct sun8i_ss_dev *ss = op->ss; in sun8i_ss_cipher()
186 struct sun8i_ss_flow *sf = &ss->flows[rctx->flow]; in sun8i_ss_cipher()
192 int nsgs = sg_nents_for_len(areq->src, areq->cryptlen); in sun8i_ss_cipher()
193 int nsgd = sg_nents_for_len(areq->dst, areq->cryptlen); in sun8i_ss_cipher()
198 dev_dbg(ss->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, in sun8i_ss_cipher()
199 crypto_tfm_alg_name(areq->base.tfm), in sun8i_ss_cipher()
200 areq->cryptlen, in sun8i_ss_cipher()
201 rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), in sun8i_ss_cipher()
202 op->keylen); in sun8i_ss_cipher()
205 algt->stat_req++; in sun8i_ss_cipher()
208 rctx->op_mode = ss->variant->op_mode[algt->ss_blockmode]; in sun8i_ss_cipher()
209 rctx->method = ss->variant->alg_cipher[algt->ss_algo_id]; in sun8i_ss_cipher()
210 rctx->keylen = op->keylen; in sun8i_ss_cipher()
212 rctx->p_key = dma_map_single(ss->dev, op->key, op->keylen, DMA_TO_DEVICE); in sun8i_ss_cipher()
213 if (dma_mapping_error(ss->dev, rctx->p_key)) { in sun8i_ss_cipher()
214 dev_err(ss->dev, "Cannot DMA MAP KEY\n"); in sun8i_ss_cipher()
215 err = -EFAULT; in sun8i_ss_cipher()
220 if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { in sun8i_ss_cipher()
225 if (areq->src == areq->dst) { in sun8i_ss_cipher()
226 nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL); in sun8i_ss_cipher()
228 dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); in sun8i_ss_cipher()
229 err = -EINVAL; in sun8i_ss_cipher()
234 nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); in sun8i_ss_cipher()
236 dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); in sun8i_ss_cipher()
237 err = -EINVAL; in sun8i_ss_cipher()
240 nr_sgd = dma_map_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); in sun8i_ss_cipher()
242 dev_err(ss->dev, "Invalid sg number %d\n", nr_sgd); in sun8i_ss_cipher()
243 err = -EINVAL; in sun8i_ss_cipher()
248 len = areq->cryptlen; in sun8i_ss_cipher()
250 sg = areq->src; in sun8i_ss_cipher()
254 rctx->t_src[i].addr = sg_dma_address(sg); in sun8i_ss_cipher()
256 rctx->t_src[i].len = todo / 4; in sun8i_ss_cipher()
257 dev_dbg(ss->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__, in sun8i_ss_cipher()
258 areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo); in sun8i_ss_cipher()
259 len -= todo; in sun8i_ss_cipher()
265 dev_err(ss->dev, "remaining len %d\n", len); in sun8i_ss_cipher()
266 err = -EINVAL; in sun8i_ss_cipher()
270 len = areq->cryptlen; in sun8i_ss_cipher()
272 sg = areq->dst; in sun8i_ss_cipher()
276 rctx->t_dst[i].addr = sg_dma_address(sg); in sun8i_ss_cipher()
278 rctx->t_dst[i].len = todo / 4; in sun8i_ss_cipher()
279 dev_dbg(ss->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__, in sun8i_ss_cipher()
280 areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo); in sun8i_ss_cipher()
281 len -= todo; in sun8i_ss_cipher()
287 dev_err(ss->dev, "remaining len %d\n", len); in sun8i_ss_cipher()
288 err = -EINVAL; in sun8i_ss_cipher()
292 err = sun8i_ss_run_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); in sun8i_ss_cipher()
295 if (areq->src == areq->dst) { in sun8i_ss_cipher()
296 dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL); in sun8i_ss_cipher()
298 dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE); in sun8i_ss_cipher()
299 dma_unmap_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE); in sun8i_ss_cipher()
303 if (areq->iv && ivsize > 0) { in sun8i_ss_cipher()
304 for (i = 0; i < rctx->niv; i++) { in sun8i_ss_cipher()
305 dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE); in sun8i_ss_cipher()
306 memzero_explicit(sf->iv[i], ivsize); in sun8i_ss_cipher()
309 offset = areq->cryptlen - ivsize; in sun8i_ss_cipher()
310 if (rctx->op_dir & SS_DECRYPTION) { in sun8i_ss_cipher()
311 memcpy(areq->iv, sf->biv, ivsize); in sun8i_ss_cipher()
312 memzero_explicit(sf->biv, ivsize); in sun8i_ss_cipher()
314 scatterwalk_map_and_copy(areq->iv, areq->dst, offset, in sun8i_ss_cipher()
320 dma_unmap_single(ss->dev, rctx->p_key, op->keylen, DMA_TO_DEVICE); in sun8i_ss_cipher()
349 rctx->op_dir = SS_DECRYPTION; in sun8i_ss_skdecrypt()
354 e = sun8i_ss_get_engine_number(op->ss); in sun8i_ss_skdecrypt()
355 engine = op->ss->flows[e].engine; in sun8i_ss_skdecrypt()
356 rctx->flow = e; in sun8i_ss_skdecrypt()
370 rctx->op_dir = SS_ENCRYPTION; in sun8i_ss_skencrypt()
375 e = sun8i_ss_get_engine_number(op->ss); in sun8i_ss_skencrypt()
376 engine = op->ss->flows[e].engine; in sun8i_ss_skencrypt()
377 rctx->flow = e; in sun8i_ss_skencrypt()
394 op->ss = algt->ss; in sun8i_ss_cipher_init()
396 op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); in sun8i_ss_cipher_init()
397 if (IS_ERR(op->fallback_tfm)) { in sun8i_ss_cipher_init()
398 dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n", in sun8i_ss_cipher_init()
399 name, PTR_ERR(op->fallback_tfm)); in sun8i_ss_cipher_init()
400 return PTR_ERR(op->fallback_tfm); in sun8i_ss_cipher_init()
403 sktfm->reqsize = sizeof(struct sun8i_cipher_req_ctx) + in sun8i_ss_cipher_init()
404 crypto_skcipher_reqsize(op->fallback_tfm); in sun8i_ss_cipher_init()
407 memcpy(algt->fbname, in sun8i_ss_cipher_init()
408 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), in sun8i_ss_cipher_init()
411 op->enginectx.op.do_one_request = sun8i_ss_handle_cipher_request; in sun8i_ss_cipher_init()
412 op->enginectx.op.prepare_request = NULL; in sun8i_ss_cipher_init()
413 op->enginectx.op.unprepare_request = NULL; in sun8i_ss_cipher_init()
415 err = pm_runtime_resume_and_get(op->ss->dev); in sun8i_ss_cipher_init()
417 dev_err(op->ss->dev, "pm error %d\n", err); in sun8i_ss_cipher_init()
423 crypto_free_skcipher(op->fallback_tfm); in sun8i_ss_cipher_init()
431 kfree_sensitive(op->key); in sun8i_ss_cipher_exit()
432 crypto_free_skcipher(op->fallback_tfm); in sun8i_ss_cipher_exit()
433 pm_runtime_put_sync(op->ss->dev); in sun8i_ss_cipher_exit()
440 struct sun8i_ss_dev *ss = op->ss; in sun8i_ss_aes_setkey()
450 dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen); in sun8i_ss_aes_setkey()
451 return -EINVAL; in sun8i_ss_aes_setkey()
453 kfree_sensitive(op->key); in sun8i_ss_aes_setkey()
454 op->keylen = keylen; in sun8i_ss_aes_setkey()
455 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); in sun8i_ss_aes_setkey()
456 if (!op->key) in sun8i_ss_aes_setkey()
457 return -ENOMEM; in sun8i_ss_aes_setkey()
459 crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); in sun8i_ss_aes_setkey()
460 crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); in sun8i_ss_aes_setkey()
462 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); in sun8i_ss_aes_setkey()
469 struct sun8i_ss_dev *ss = op->ss; in sun8i_ss_des3_setkey()
472 dev_dbg(ss->dev, "Invalid keylen %u\n", keylen); in sun8i_ss_des3_setkey()
473 return -EINVAL; in sun8i_ss_des3_setkey()
476 kfree_sensitive(op->key); in sun8i_ss_des3_setkey()
477 op->keylen = keylen; in sun8i_ss_des3_setkey()
478 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); in sun8i_ss_des3_setkey()
479 if (!op->key) in sun8i_ss_des3_setkey()
480 return -ENOMEM; in sun8i_ss_des3_setkey()
482 crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); in sun8i_ss_des3_setkey()
483 crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); in sun8i_ss_des3_setkey()
485 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); in sun8i_ss_des3_setkey()