1 /*
2 * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC
3 *
4 * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com>
5 *
6 * This file add support for AES cipher with 128,192,256 bits
7 * keysize in CBC and ECB mode.
8 * Add support also for DES and 3DES in CBC and ECB mode.
9 *
10 * You could find the datasheet in Documentation/arm/sunxi/README
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 */
17 #include "sun4i-ss.h"
18
sun4i_ss_opti_poll(struct skcipher_request * areq)19 static int sun4i_ss_opti_poll(struct skcipher_request *areq)
20 {
21 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
22 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
23 struct sun4i_ss_ctx *ss = op->ss;
24 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
25 struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
26 u32 mode = ctx->mode;
27 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
28 u32 rx_cnt = SS_RX_DEFAULT;
29 u32 tx_cnt = 0;
30 u32 spaces;
31 u32 v;
32 int err = 0;
33 unsigned int i;
34 unsigned int ileft = areq->cryptlen;
35 unsigned int oleft = areq->cryptlen;
36 unsigned int todo;
37 struct sg_mapping_iter mi, mo;
38 unsigned int oi, oo; /* offset for in and out */
39 unsigned long flags;
40
41 if (!areq->cryptlen)
42 return 0;
43
44 if (!areq->iv) {
45 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
46 return -EINVAL;
47 }
48
49 if (!areq->src || !areq->dst) {
50 dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");
51 return -EINVAL;
52 }
53
54 spin_lock_irqsave(&ss->slock, flags);
55
56 for (i = 0; i < op->keylen; i += 4)
57 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
58
59 if (areq->iv) {
60 for (i = 0; i < 4 && i < ivsize / 4; i++) {
61 v = *(u32 *)(areq->iv + i * 4);
62 writel(v, ss->base + SS_IV0 + i * 4);
63 }
64 }
65 writel(mode, ss->base + SS_CTL);
66
67 sg_miter_start(&mi, areq->src, sg_nents(areq->src),
68 SG_MITER_FROM_SG | SG_MITER_ATOMIC);
69 sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),
70 SG_MITER_TO_SG | SG_MITER_ATOMIC);
71 sg_miter_next(&mi);
72 sg_miter_next(&mo);
73 if (!mi.addr || !mo.addr) {
74 dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");
75 err = -EINVAL;
76 goto release_ss;
77 }
78
79 ileft = areq->cryptlen / 4;
80 oleft = areq->cryptlen / 4;
81 oi = 0;
82 oo = 0;
83 do {
84 todo = min3(rx_cnt, ileft, (mi.length - oi) / 4);
85 if (todo) {
86 ileft -= todo;
87 writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo);
88 oi += todo * 4;
89 }
90 if (oi == mi.length) {
91 sg_miter_next(&mi);
92 oi = 0;
93 }
94
95 spaces = readl(ss->base + SS_FCSR);
96 rx_cnt = SS_RXFIFO_SPACES(spaces);
97 tx_cnt = SS_TXFIFO_SPACES(spaces);
98
99 todo = min3(tx_cnt, oleft, (mo.length - oo) / 4);
100 if (todo) {
101 oleft -= todo;
102 readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
103 oo += todo * 4;
104 }
105 if (oo == mo.length) {
106 sg_miter_next(&mo);
107 oo = 0;
108 }
109 } while (oleft);
110
111 if (areq->iv) {
112 for (i = 0; i < 4 && i < ivsize / 4; i++) {
113 v = readl(ss->base + SS_IV0 + i * 4);
114 *(u32 *)(areq->iv + i * 4) = v;
115 }
116 }
117
118 release_ss:
119 sg_miter_stop(&mi);
120 sg_miter_stop(&mo);
121 writel(0, ss->base + SS_CTL);
122 spin_unlock_irqrestore(&ss->slock, flags);
123 return err;
124 }
125
126 /* Generic function that support SG with size not multiple of 4 */
sun4i_ss_cipher_poll(struct skcipher_request * areq)127 static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
128 {
129 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
130 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
131 struct sun4i_ss_ctx *ss = op->ss;
132 int no_chunk = 1;
133 struct scatterlist *in_sg = areq->src;
134 struct scatterlist *out_sg = areq->dst;
135 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
136 struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
137 u32 mode = ctx->mode;
138 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
139 u32 rx_cnt = SS_RX_DEFAULT;
140 u32 tx_cnt = 0;
141 u32 v;
142 u32 spaces;
143 int err = 0;
144 unsigned int i;
145 unsigned int ileft = areq->cryptlen;
146 unsigned int oleft = areq->cryptlen;
147 unsigned int todo;
148 struct sg_mapping_iter mi, mo;
149 unsigned int oi, oo; /* offset for in and out */
150 char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
151 char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
152 unsigned int ob = 0; /* offset in buf */
153 unsigned int obo = 0; /* offset in bufo*/
154 unsigned int obl = 0; /* length of data in bufo */
155 unsigned long flags;
156
157 if (!areq->cryptlen)
158 return 0;
159
160 if (!areq->iv) {
161 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
162 return -EINVAL;
163 }
164
165 if (!areq->src || !areq->dst) {
166 dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");
167 return -EINVAL;
168 }
169
170 /*
171 * if we have only SGs with size multiple of 4,
172 * we can use the SS optimized function
173 */
174 while (in_sg && no_chunk == 1) {
175 if (in_sg->length % 4)
176 no_chunk = 0;
177 in_sg = sg_next(in_sg);
178 }
179 while (out_sg && no_chunk == 1) {
180 if (out_sg->length % 4)
181 no_chunk = 0;
182 out_sg = sg_next(out_sg);
183 }
184
185 if (no_chunk == 1)
186 return sun4i_ss_opti_poll(areq);
187
188 spin_lock_irqsave(&ss->slock, flags);
189
190 for (i = 0; i < op->keylen; i += 4)
191 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
192
193 if (areq->iv) {
194 for (i = 0; i < 4 && i < ivsize / 4; i++) {
195 v = *(u32 *)(areq->iv + i * 4);
196 writel(v, ss->base + SS_IV0 + i * 4);
197 }
198 }
199 writel(mode, ss->base + SS_CTL);
200
201 sg_miter_start(&mi, areq->src, sg_nents(areq->src),
202 SG_MITER_FROM_SG | SG_MITER_ATOMIC);
203 sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),
204 SG_MITER_TO_SG | SG_MITER_ATOMIC);
205 sg_miter_next(&mi);
206 sg_miter_next(&mo);
207 if (!mi.addr || !mo.addr) {
208 dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");
209 err = -EINVAL;
210 goto release_ss;
211 }
212 ileft = areq->cryptlen;
213 oleft = areq->cryptlen;
214 oi = 0;
215 oo = 0;
216
217 while (oleft) {
218 if (ileft) {
219 /*
220 * todo is the number of consecutive 4byte word that we
221 * can read from current SG
222 */
223 todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4);
224 if (todo && !ob) {
225 writesl(ss->base + SS_RXFIFO, mi.addr + oi,
226 todo);
227 ileft -= todo * 4;
228 oi += todo * 4;
229 } else {
230 /*
231 * not enough consecutive bytes, so we need to
232 * linearize in buf. todo is in bytes
233 * After that copy, if we have a multiple of 4
234 * we need to be able to write all buf in one
235 * pass, so it is why we min() with rx_cnt
236 */
237 todo = min3(rx_cnt * 4 - ob, ileft,
238 mi.length - oi);
239 memcpy(buf + ob, mi.addr + oi, todo);
240 ileft -= todo;
241 oi += todo;
242 ob += todo;
243 if (!(ob % 4)) {
244 writesl(ss->base + SS_RXFIFO, buf,
245 ob / 4);
246 ob = 0;
247 }
248 }
249 if (oi == mi.length) {
250 sg_miter_next(&mi);
251 oi = 0;
252 }
253 }
254
255 spaces = readl(ss->base + SS_FCSR);
256 rx_cnt = SS_RXFIFO_SPACES(spaces);
257 tx_cnt = SS_TXFIFO_SPACES(spaces);
258 dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n",
259 mode,
260 oi, mi.length, ileft, areq->cryptlen, rx_cnt,
261 oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob);
262
263 if (!tx_cnt)
264 continue;
265 /* todo in 4bytes word */
266 todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4);
267 if (todo) {
268 readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
269 oleft -= todo * 4;
270 oo += todo * 4;
271 if (oo == mo.length) {
272 sg_miter_next(&mo);
273 oo = 0;
274 }
275 } else {
276 /*
277 * read obl bytes in bufo, we read at maximum for
278 * emptying the device
279 */
280 readsl(ss->base + SS_TXFIFO, bufo, tx_cnt);
281 obl = tx_cnt * 4;
282 obo = 0;
283 do {
284 /*
285 * how many bytes we can copy ?
286 * no more than remaining SG size
287 * no more than remaining buffer
288 * no need to test against oleft
289 */
290 todo = min(mo.length - oo, obl - obo);
291 memcpy(mo.addr + oo, bufo + obo, todo);
292 oleft -= todo;
293 obo += todo;
294 oo += todo;
295 if (oo == mo.length) {
296 sg_miter_next(&mo);
297 oo = 0;
298 }
299 } while (obo < obl);
300 /* bufo must be fully used here */
301 }
302 }
303 if (areq->iv) {
304 for (i = 0; i < 4 && i < ivsize / 4; i++) {
305 v = readl(ss->base + SS_IV0 + i * 4);
306 *(u32 *)(areq->iv + i * 4) = v;
307 }
308 }
309
310 release_ss:
311 sg_miter_stop(&mi);
312 sg_miter_stop(&mo);
313 writel(0, ss->base + SS_CTL);
314 spin_unlock_irqrestore(&ss->slock, flags);
315
316 return err;
317 }
318
319 /* CBC AES */
sun4i_ss_cbc_aes_encrypt(struct skcipher_request * areq)320 int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq)
321 {
322 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
323 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
324 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
325
326 rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
327 op->keymode;
328 return sun4i_ss_cipher_poll(areq);
329 }
330
sun4i_ss_cbc_aes_decrypt(struct skcipher_request * areq)331 int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq)
332 {
333 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
334 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
335 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
336
337 rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
338 op->keymode;
339 return sun4i_ss_cipher_poll(areq);
340 }
341
342 /* ECB AES */
sun4i_ss_ecb_aes_encrypt(struct skcipher_request * areq)343 int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq)
344 {
345 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
346 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
347 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
348
349 rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
350 op->keymode;
351 return sun4i_ss_cipher_poll(areq);
352 }
353
sun4i_ss_ecb_aes_decrypt(struct skcipher_request * areq)354 int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq)
355 {
356 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
357 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
358 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
359
360 rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
361 op->keymode;
362 return sun4i_ss_cipher_poll(areq);
363 }
364
365 /* CBC DES */
sun4i_ss_cbc_des_encrypt(struct skcipher_request * areq)366 int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq)
367 {
368 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
369 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
370 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
371
372 rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
373 op->keymode;
374 return sun4i_ss_cipher_poll(areq);
375 }
376
sun4i_ss_cbc_des_decrypt(struct skcipher_request * areq)377 int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq)
378 {
379 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
380 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
381 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
382
383 rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
384 op->keymode;
385 return sun4i_ss_cipher_poll(areq);
386 }
387
388 /* ECB DES */
sun4i_ss_ecb_des_encrypt(struct skcipher_request * areq)389 int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq)
390 {
391 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
392 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
393 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
394
395 rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
396 op->keymode;
397 return sun4i_ss_cipher_poll(areq);
398 }
399
sun4i_ss_ecb_des_decrypt(struct skcipher_request * areq)400 int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq)
401 {
402 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
403 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
404 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
405
406 rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
407 op->keymode;
408 return sun4i_ss_cipher_poll(areq);
409 }
410
411 /* CBC 3DES */
sun4i_ss_cbc_des3_encrypt(struct skcipher_request * areq)412 int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq)
413 {
414 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
415 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
416 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
417
418 rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
419 op->keymode;
420 return sun4i_ss_cipher_poll(areq);
421 }
422
sun4i_ss_cbc_des3_decrypt(struct skcipher_request * areq)423 int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq)
424 {
425 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
426 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
427 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
428
429 rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
430 op->keymode;
431 return sun4i_ss_cipher_poll(areq);
432 }
433
434 /* ECB 3DES */
sun4i_ss_ecb_des3_encrypt(struct skcipher_request * areq)435 int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq)
436 {
437 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
438 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
439 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
440
441 rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
442 op->keymode;
443 return sun4i_ss_cipher_poll(areq);
444 }
445
sun4i_ss_ecb_des3_decrypt(struct skcipher_request * areq)446 int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq)
447 {
448 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
449 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
450 struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
451
452 rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
453 op->keymode;
454 return sun4i_ss_cipher_poll(areq);
455 }
456
sun4i_ss_cipher_init(struct crypto_tfm * tfm)457 int sun4i_ss_cipher_init(struct crypto_tfm *tfm)
458 {
459 struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
460 struct sun4i_ss_alg_template *algt;
461
462 memset(op, 0, sizeof(struct sun4i_tfm_ctx));
463
464 algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template,
465 alg.crypto.base);
466 op->ss = algt->ss;
467
468 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
469 sizeof(struct sun4i_cipher_req_ctx));
470
471 return 0;
472 }
473
474 /* check and set the AES key, prepare the mode to be used */
sun4i_ss_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)475 int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
476 unsigned int keylen)
477 {
478 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
479 struct sun4i_ss_ctx *ss = op->ss;
480
481 switch (keylen) {
482 case 128 / 8:
483 op->keymode = SS_AES_128BITS;
484 break;
485 case 192 / 8:
486 op->keymode = SS_AES_192BITS;
487 break;
488 case 256 / 8:
489 op->keymode = SS_AES_256BITS;
490 break;
491 default:
492 dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen);
493 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
494 return -EINVAL;
495 }
496 op->keylen = keylen;
497 memcpy(op->key, key, keylen);
498 return 0;
499 }
500
501 /* check and set the DES key, prepare the mode to be used */
sun4i_ss_des_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)502 int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
503 unsigned int keylen)
504 {
505 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
506 struct sun4i_ss_ctx *ss = op->ss;
507 u32 flags;
508 u32 tmp[DES_EXPKEY_WORDS];
509 int ret;
510
511 if (unlikely(keylen != DES_KEY_SIZE)) {
512 dev_err(ss->dev, "Invalid keylen %u\n", keylen);
513 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
514 return -EINVAL;
515 }
516
517 flags = crypto_skcipher_get_flags(tfm);
518
519 ret = des_ekey(tmp, key);
520 if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
521 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
522 dev_dbg(ss->dev, "Weak key %u\n", keylen);
523 return -EINVAL;
524 }
525
526 op->keylen = keylen;
527 memcpy(op->key, key, keylen);
528 return 0;
529 }
530
531 /* check and set the 3DES key, prepare the mode to be used */
sun4i_ss_des3_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)532 int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
533 unsigned int keylen)
534 {
535 struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
536 struct sun4i_ss_ctx *ss = op->ss;
537
538 if (unlikely(keylen != 3 * DES_KEY_SIZE)) {
539 dev_err(ss->dev, "Invalid keylen %u\n", keylen);
540 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
541 return -EINVAL;
542 }
543 op->keylen = keylen;
544 memcpy(op->key, key, keylen);
545 return 0;
546 }
547
