1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Software async crypto daemon.
4 *
5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
6 *
7 * Added AEAD support to cryptd.
8 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
9 * Adrian Hoban <adrian.hoban@intel.com>
10 * Gabriele Paoloni <gabriele.paoloni@intel.com>
11 * Aidan O'Mahony (aidan.o.mahony@intel.com)
12 * Copyright (c) 2010, Intel Corporation.
13 */
14
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/internal/skcipher.h>
18 #include <crypto/cryptd.h>
19 #include <linux/refcount.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/sched.h>
27 #include <linux/slab.h>
28 #include <linux/workqueue.h>
29
30 static unsigned int cryptd_max_cpu_qlen = 1000;
31 module_param(cryptd_max_cpu_qlen, uint, 0);
32 MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
33
34 static struct workqueue_struct *cryptd_wq;
35
36 struct cryptd_cpu_queue {
37 struct crypto_queue queue;
38 struct work_struct work;
39 };
40
41 struct cryptd_queue {
42 /*
43 * Protected by disabling BH to allow enqueueing from softinterrupt and
44 * dequeuing from kworker (cryptd_queue_worker()).
45 */
46 struct cryptd_cpu_queue __percpu *cpu_queue;
47 };
48
49 struct cryptd_instance_ctx {
50 struct crypto_spawn spawn;
51 struct cryptd_queue *queue;
52 };
53
54 struct skcipherd_instance_ctx {
55 struct crypto_skcipher_spawn spawn;
56 struct cryptd_queue *queue;
57 };
58
59 struct hashd_instance_ctx {
60 struct crypto_shash_spawn spawn;
61 struct cryptd_queue *queue;
62 };
63
64 struct aead_instance_ctx {
65 struct crypto_aead_spawn aead_spawn;
66 struct cryptd_queue *queue;
67 };
68
69 struct cryptd_skcipher_ctx {
70 refcount_t refcnt;
71 struct crypto_sync_skcipher *child;
72 };
73
74 struct cryptd_skcipher_request_ctx {
75 crypto_completion_t complete;
76 };
77
78 struct cryptd_hash_ctx {
79 refcount_t refcnt;
80 struct crypto_shash *child;
81 };
82
83 struct cryptd_hash_request_ctx {
84 crypto_completion_t complete;
85 struct shash_desc desc;
86 };
87
88 struct cryptd_aead_ctx {
89 refcount_t refcnt;
90 struct crypto_aead *child;
91 };
92
93 struct cryptd_aead_request_ctx {
94 crypto_completion_t complete;
95 };
96
97 static void cryptd_queue_worker(struct work_struct *work);
98
cryptd_init_queue(struct cryptd_queue * queue,unsigned int max_cpu_qlen)99 static int cryptd_init_queue(struct cryptd_queue *queue,
100 unsigned int max_cpu_qlen)
101 {
102 int cpu;
103 struct cryptd_cpu_queue *cpu_queue;
104
105 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
106 if (!queue->cpu_queue)
107 return -ENOMEM;
108 for_each_possible_cpu(cpu) {
109 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
110 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
111 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
112 }
113 pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen);
114 return 0;
115 }
116
cryptd_fini_queue(struct cryptd_queue * queue)117 static void cryptd_fini_queue(struct cryptd_queue *queue)
118 {
119 int cpu;
120 struct cryptd_cpu_queue *cpu_queue;
121
122 for_each_possible_cpu(cpu) {
123 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
124 BUG_ON(cpu_queue->queue.qlen);
125 }
126 free_percpu(queue->cpu_queue);
127 }
128
cryptd_enqueue_request(struct cryptd_queue * queue,struct crypto_async_request * request)129 static int cryptd_enqueue_request(struct cryptd_queue *queue,
130 struct crypto_async_request *request)
131 {
132 int err;
133 struct cryptd_cpu_queue *cpu_queue;
134 refcount_t *refcnt;
135
136 local_bh_disable();
137 cpu_queue = this_cpu_ptr(queue->cpu_queue);
138 err = crypto_enqueue_request(&cpu_queue->queue, request);
139
140 refcnt = crypto_tfm_ctx(request->tfm);
141
142 if (err == -ENOSPC)
143 goto out;
144
145 queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work);
146
147 if (!refcount_read(refcnt))
148 goto out;
149
150 refcount_inc(refcnt);
151
152 out:
153 local_bh_enable();
154
155 return err;
156 }
157
158 /* Called in workqueue context, do one real cryption work (via
159 * req->complete) and reschedule itself if there are more work to
160 * do. */
cryptd_queue_worker(struct work_struct * work)161 static void cryptd_queue_worker(struct work_struct *work)
162 {
163 struct cryptd_cpu_queue *cpu_queue;
164 struct crypto_async_request *req, *backlog;
165
166 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
167 /*
168 * Only handle one request at a time to avoid hogging crypto workqueue.
169 */
170 local_bh_disable();
171 backlog = crypto_get_backlog(&cpu_queue->queue);
172 req = crypto_dequeue_request(&cpu_queue->queue);
173 local_bh_enable();
174
175 if (!req)
176 return;
177
178 if (backlog)
179 backlog->complete(backlog, -EINPROGRESS);
180 req->complete(req, 0);
181
182 if (cpu_queue->queue.qlen)
183 queue_work(cryptd_wq, &cpu_queue->work);
184 }
185
cryptd_get_queue(struct crypto_tfm * tfm)186 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
187 {
188 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
189 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
190 return ictx->queue;
191 }
192
cryptd_type_and_mask(struct crypto_attr_type * algt,u32 * type,u32 * mask)193 static void cryptd_type_and_mask(struct crypto_attr_type *algt,
194 u32 *type, u32 *mask)
195 {
196 /*
197 * cryptd is allowed to wrap internal algorithms, but in that case the
198 * resulting cryptd instance will be marked as internal as well.
199 */
200 *type = algt->type & CRYPTO_ALG_INTERNAL;
201 *mask = algt->mask & CRYPTO_ALG_INTERNAL;
202
203 /* No point in cryptd wrapping an algorithm that's already async. */
204 *mask |= CRYPTO_ALG_ASYNC;
205
206 *mask |= crypto_algt_inherited_mask(algt);
207 }
208
cryptd_init_instance(struct crypto_instance * inst,struct crypto_alg * alg)209 static int cryptd_init_instance(struct crypto_instance *inst,
210 struct crypto_alg *alg)
211 {
212 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
213 "cryptd(%s)",
214 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
215 return -ENAMETOOLONG;
216
217 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
218
219 inst->alg.cra_priority = alg->cra_priority + 50;
220 inst->alg.cra_blocksize = alg->cra_blocksize;
221 inst->alg.cra_alignmask = alg->cra_alignmask;
222
223 return 0;
224 }
225
cryptd_skcipher_setkey(struct crypto_skcipher * parent,const u8 * key,unsigned int keylen)226 static int cryptd_skcipher_setkey(struct crypto_skcipher *parent,
227 const u8 *key, unsigned int keylen)
228 {
229 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent);
230 struct crypto_sync_skcipher *child = ctx->child;
231
232 crypto_sync_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
233 crypto_sync_skcipher_set_flags(child,
234 crypto_skcipher_get_flags(parent) &
235 CRYPTO_TFM_REQ_MASK);
236 return crypto_sync_skcipher_setkey(child, key, keylen);
237 }
238
cryptd_skcipher_complete(struct skcipher_request * req,int err)239 static void cryptd_skcipher_complete(struct skcipher_request *req, int err)
240 {
241 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
242 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
243 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
244 int refcnt = refcount_read(&ctx->refcnt);
245
246 local_bh_disable();
247 rctx->complete(&req->base, err);
248 local_bh_enable();
249
250 if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt))
251 crypto_free_skcipher(tfm);
252 }
253
cryptd_skcipher_encrypt(struct crypto_async_request * base,int err)254 static void cryptd_skcipher_encrypt(struct crypto_async_request *base,
255 int err)
256 {
257 struct skcipher_request *req = skcipher_request_cast(base);
258 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
259 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
260 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
261 struct crypto_sync_skcipher *child = ctx->child;
262 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
263
264 if (unlikely(err == -EINPROGRESS))
265 goto out;
266
267 skcipher_request_set_sync_tfm(subreq, child);
268 skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
269 NULL, NULL);
270 skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
271 req->iv);
272
273 err = crypto_skcipher_encrypt(subreq);
274 skcipher_request_zero(subreq);
275
276 req->base.complete = rctx->complete;
277
278 out:
279 cryptd_skcipher_complete(req, err);
280 }
281
cryptd_skcipher_decrypt(struct crypto_async_request * base,int err)282 static void cryptd_skcipher_decrypt(struct crypto_async_request *base,
283 int err)
284 {
285 struct skcipher_request *req = skcipher_request_cast(base);
286 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
287 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
288 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
289 struct crypto_sync_skcipher *child = ctx->child;
290 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
291
292 if (unlikely(err == -EINPROGRESS))
293 goto out;
294
295 skcipher_request_set_sync_tfm(subreq, child);
296 skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
297 NULL, NULL);
298 skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
299 req->iv);
300
301 err = crypto_skcipher_decrypt(subreq);
302 skcipher_request_zero(subreq);
303
304 req->base.complete = rctx->complete;
305
306 out:
307 cryptd_skcipher_complete(req, err);
308 }
309
cryptd_skcipher_enqueue(struct skcipher_request * req,crypto_completion_t compl)310 static int cryptd_skcipher_enqueue(struct skcipher_request *req,
311 crypto_completion_t compl)
312 {
313 struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
314 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
315 struct cryptd_queue *queue;
316
317 queue = cryptd_get_queue(crypto_skcipher_tfm(tfm));
318 rctx->complete = req->base.complete;
319 req->base.complete = compl;
320
321 return cryptd_enqueue_request(queue, &req->base);
322 }
323
cryptd_skcipher_encrypt_enqueue(struct skcipher_request * req)324 static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req)
325 {
326 return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt);
327 }
328
cryptd_skcipher_decrypt_enqueue(struct skcipher_request * req)329 static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req)
330 {
331 return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt);
332 }
333
cryptd_skcipher_init_tfm(struct crypto_skcipher * tfm)334 static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm)
335 {
336 struct skcipher_instance *inst = skcipher_alg_instance(tfm);
337 struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst);
338 struct crypto_skcipher_spawn *spawn = &ictx->spawn;
339 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
340 struct crypto_skcipher *cipher;
341
342 cipher = crypto_spawn_skcipher(spawn);
343 if (IS_ERR(cipher))
344 return PTR_ERR(cipher);
345
346 ctx->child = (struct crypto_sync_skcipher *)cipher;
347 crypto_skcipher_set_reqsize(
348 tfm, sizeof(struct cryptd_skcipher_request_ctx));
349 return 0;
350 }
351
cryptd_skcipher_exit_tfm(struct crypto_skcipher * tfm)352 static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm)
353 {
354 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
355
356 crypto_free_sync_skcipher(ctx->child);
357 }
358
cryptd_skcipher_free(struct skcipher_instance * inst)359 static void cryptd_skcipher_free(struct skcipher_instance *inst)
360 {
361 struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst);
362
363 crypto_drop_skcipher(&ctx->spawn);
364 kfree(inst);
365 }
366
cryptd_create_skcipher(struct crypto_template * tmpl,struct rtattr ** tb,struct crypto_attr_type * algt,struct cryptd_queue * queue)367 static int cryptd_create_skcipher(struct crypto_template *tmpl,
368 struct rtattr **tb,
369 struct crypto_attr_type *algt,
370 struct cryptd_queue *queue)
371 {
372 struct skcipherd_instance_ctx *ctx;
373 struct skcipher_instance *inst;
374 struct skcipher_alg *alg;
375 u32 type;
376 u32 mask;
377 int err;
378
379 cryptd_type_and_mask(algt, &type, &mask);
380
381 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
382 if (!inst)
383 return -ENOMEM;
384
385 ctx = skcipher_instance_ctx(inst);
386 ctx->queue = queue;
387
388 err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst),
389 crypto_attr_alg_name(tb[1]), type, mask);
390 if (err)
391 goto err_free_inst;
392
393 alg = crypto_spawn_skcipher_alg(&ctx->spawn);
394 err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base);
395 if (err)
396 goto err_free_inst;
397
398 inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC |
399 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
400 inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
401 inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
402 inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
403 inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
404
405 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx);
406
407 inst->alg.init = cryptd_skcipher_init_tfm;
408 inst->alg.exit = cryptd_skcipher_exit_tfm;
409
410 inst->alg.setkey = cryptd_skcipher_setkey;
411 inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue;
412 inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue;
413
414 inst->free = cryptd_skcipher_free;
415
416 err = skcipher_register_instance(tmpl, inst);
417 if (err) {
418 err_free_inst:
419 cryptd_skcipher_free(inst);
420 }
421 return err;
422 }
423
cryptd_hash_init_tfm(struct crypto_tfm * tfm)424 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
425 {
426 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
427 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
428 struct crypto_shash_spawn *spawn = &ictx->spawn;
429 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
430 struct crypto_shash *hash;
431
432 hash = crypto_spawn_shash(spawn);
433 if (IS_ERR(hash))
434 return PTR_ERR(hash);
435
436 ctx->child = hash;
437 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
438 sizeof(struct cryptd_hash_request_ctx) +
439 crypto_shash_descsize(hash));
440 return 0;
441 }
442
cryptd_hash_exit_tfm(struct crypto_tfm * tfm)443 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
444 {
445 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
446
447 crypto_free_shash(ctx->child);
448 }
449
cryptd_hash_setkey(struct crypto_ahash * parent,const u8 * key,unsigned int keylen)450 static int cryptd_hash_setkey(struct crypto_ahash *parent,
451 const u8 *key, unsigned int keylen)
452 {
453 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
454 struct crypto_shash *child = ctx->child;
455
456 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
457 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
458 CRYPTO_TFM_REQ_MASK);
459 return crypto_shash_setkey(child, key, keylen);
460 }
461
cryptd_hash_enqueue(struct ahash_request * req,crypto_completion_t compl)462 static int cryptd_hash_enqueue(struct ahash_request *req,
463 crypto_completion_t compl)
464 {
465 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
466 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
467 struct cryptd_queue *queue =
468 cryptd_get_queue(crypto_ahash_tfm(tfm));
469
470 rctx->complete = req->base.complete;
471 req->base.complete = compl;
472
473 return cryptd_enqueue_request(queue, &req->base);
474 }
475
cryptd_hash_complete(struct ahash_request * req,int err)476 static void cryptd_hash_complete(struct ahash_request *req, int err)
477 {
478 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
479 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
480 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
481 int refcnt = refcount_read(&ctx->refcnt);
482
483 local_bh_disable();
484 rctx->complete(&req->base, err);
485 local_bh_enable();
486
487 if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt))
488 crypto_free_ahash(tfm);
489 }
490
cryptd_hash_init(struct crypto_async_request * req_async,int err)491 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
492 {
493 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
494 struct crypto_shash *child = ctx->child;
495 struct ahash_request *req = ahash_request_cast(req_async);
496 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
497 struct shash_desc *desc = &rctx->desc;
498
499 if (unlikely(err == -EINPROGRESS))
500 goto out;
501
502 desc->tfm = child;
503
504 err = crypto_shash_init(desc);
505
506 req->base.complete = rctx->complete;
507
508 out:
509 cryptd_hash_complete(req, err);
510 }
511
cryptd_hash_init_enqueue(struct ahash_request * req)512 static int cryptd_hash_init_enqueue(struct ahash_request *req)
513 {
514 return cryptd_hash_enqueue(req, cryptd_hash_init);
515 }
516
cryptd_hash_update(struct crypto_async_request * req_async,int err)517 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
518 {
519 struct ahash_request *req = ahash_request_cast(req_async);
520 struct cryptd_hash_request_ctx *rctx;
521
522 rctx = ahash_request_ctx(req);
523
524 if (unlikely(err == -EINPROGRESS))
525 goto out;
526
527 err = shash_ahash_update(req, &rctx->desc);
528
529 req->base.complete = rctx->complete;
530
531 out:
532 cryptd_hash_complete(req, err);
533 }
534
cryptd_hash_update_enqueue(struct ahash_request * req)535 static int cryptd_hash_update_enqueue(struct ahash_request *req)
536 {
537 return cryptd_hash_enqueue(req, cryptd_hash_update);
538 }
539
cryptd_hash_final(struct crypto_async_request * req_async,int err)540 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
541 {
542 struct ahash_request *req = ahash_request_cast(req_async);
543 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
544
545 if (unlikely(err == -EINPROGRESS))
546 goto out;
547
548 err = crypto_shash_final(&rctx->desc, req->result);
549
550 req->base.complete = rctx->complete;
551
552 out:
553 cryptd_hash_complete(req, err);
554 }
555
cryptd_hash_final_enqueue(struct ahash_request * req)556 static int cryptd_hash_final_enqueue(struct ahash_request *req)
557 {
558 return cryptd_hash_enqueue(req, cryptd_hash_final);
559 }
560
cryptd_hash_finup(struct crypto_async_request * req_async,int err)561 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
562 {
563 struct ahash_request *req = ahash_request_cast(req_async);
564 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
565
566 if (unlikely(err == -EINPROGRESS))
567 goto out;
568
569 err = shash_ahash_finup(req, &rctx->desc);
570
571 req->base.complete = rctx->complete;
572
573 out:
574 cryptd_hash_complete(req, err);
575 }
576
cryptd_hash_finup_enqueue(struct ahash_request * req)577 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
578 {
579 return cryptd_hash_enqueue(req, cryptd_hash_finup);
580 }
581
cryptd_hash_digest(struct crypto_async_request * req_async,int err)582 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
583 {
584 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
585 struct crypto_shash *child = ctx->child;
586 struct ahash_request *req = ahash_request_cast(req_async);
587 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
588 struct shash_desc *desc = &rctx->desc;
589
590 if (unlikely(err == -EINPROGRESS))
591 goto out;
592
593 desc->tfm = child;
594
595 err = shash_ahash_digest(req, desc);
596
597 req->base.complete = rctx->complete;
598
599 out:
600 cryptd_hash_complete(req, err);
601 }
602
cryptd_hash_digest_enqueue(struct ahash_request * req)603 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
604 {
605 return cryptd_hash_enqueue(req, cryptd_hash_digest);
606 }
607
cryptd_hash_export(struct ahash_request * req,void * out)608 static int cryptd_hash_export(struct ahash_request *req, void *out)
609 {
610 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
611
612 return crypto_shash_export(&rctx->desc, out);
613 }
614
cryptd_hash_import(struct ahash_request * req,const void * in)615 static int cryptd_hash_import(struct ahash_request *req, const void *in)
616 {
617 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
618 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
619 struct shash_desc *desc = cryptd_shash_desc(req);
620
621 desc->tfm = ctx->child;
622
623 return crypto_shash_import(desc, in);
624 }
625
cryptd_hash_free(struct ahash_instance * inst)626 static void cryptd_hash_free(struct ahash_instance *inst)
627 {
628 struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst);
629
630 crypto_drop_shash(&ctx->spawn);
631 kfree(inst);
632 }
633
cryptd_create_hash(struct crypto_template * tmpl,struct rtattr ** tb,struct crypto_attr_type * algt,struct cryptd_queue * queue)634 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
635 struct crypto_attr_type *algt,
636 struct cryptd_queue *queue)
637 {
638 struct hashd_instance_ctx *ctx;
639 struct ahash_instance *inst;
640 struct shash_alg *alg;
641 u32 type;
642 u32 mask;
643 int err;
644
645 cryptd_type_and_mask(algt, &type, &mask);
646
647 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
648 if (!inst)
649 return -ENOMEM;
650
651 ctx = ahash_instance_ctx(inst);
652 ctx->queue = queue;
653
654 err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst),
655 crypto_attr_alg_name(tb[1]), type, mask);
656 if (err)
657 goto err_free_inst;
658 alg = crypto_spawn_shash_alg(&ctx->spawn);
659
660 err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base);
661 if (err)
662 goto err_free_inst;
663
664 inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC |
665 (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL|
666 CRYPTO_ALG_OPTIONAL_KEY));
667 inst->alg.halg.digestsize = alg->digestsize;
668 inst->alg.halg.statesize = alg->statesize;
669 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
670
671 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
672 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
673
674 inst->alg.init = cryptd_hash_init_enqueue;
675 inst->alg.update = cryptd_hash_update_enqueue;
676 inst->alg.final = cryptd_hash_final_enqueue;
677 inst->alg.finup = cryptd_hash_finup_enqueue;
678 inst->alg.export = cryptd_hash_export;
679 inst->alg.import = cryptd_hash_import;
680 if (crypto_shash_alg_has_setkey(alg))
681 inst->alg.setkey = cryptd_hash_setkey;
682 inst->alg.digest = cryptd_hash_digest_enqueue;
683
684 inst->free = cryptd_hash_free;
685
686 err = ahash_register_instance(tmpl, inst);
687 if (err) {
688 err_free_inst:
689 cryptd_hash_free(inst);
690 }
691 return err;
692 }
693
cryptd_aead_setkey(struct crypto_aead * parent,const u8 * key,unsigned int keylen)694 static int cryptd_aead_setkey(struct crypto_aead *parent,
695 const u8 *key, unsigned int keylen)
696 {
697 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
698 struct crypto_aead *child = ctx->child;
699
700 return crypto_aead_setkey(child, key, keylen);
701 }
702
cryptd_aead_setauthsize(struct crypto_aead * parent,unsigned int authsize)703 static int cryptd_aead_setauthsize(struct crypto_aead *parent,
704 unsigned int authsize)
705 {
706 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
707 struct crypto_aead *child = ctx->child;
708
709 return crypto_aead_setauthsize(child, authsize);
710 }
711
cryptd_aead_crypt(struct aead_request * req,struct crypto_aead * child,int err,int (* crypt)(struct aead_request * req))712 static void cryptd_aead_crypt(struct aead_request *req,
713 struct crypto_aead *child,
714 int err,
715 int (*crypt)(struct aead_request *req))
716 {
717 struct cryptd_aead_request_ctx *rctx;
718 struct cryptd_aead_ctx *ctx;
719 crypto_completion_t compl;
720 struct crypto_aead *tfm;
721 int refcnt;
722
723 rctx = aead_request_ctx(req);
724 compl = rctx->complete;
725
726 tfm = crypto_aead_reqtfm(req);
727
728 if (unlikely(err == -EINPROGRESS))
729 goto out;
730 aead_request_set_tfm(req, child);
731 err = crypt( req );
732
733 out:
734 ctx = crypto_aead_ctx(tfm);
735 refcnt = refcount_read(&ctx->refcnt);
736
737 local_bh_disable();
738 compl(&req->base, err);
739 local_bh_enable();
740
741 if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt))
742 crypto_free_aead(tfm);
743 }
744
cryptd_aead_encrypt(struct crypto_async_request * areq,int err)745 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
746 {
747 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
748 struct crypto_aead *child = ctx->child;
749 struct aead_request *req;
750
751 req = container_of(areq, struct aead_request, base);
752 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
753 }
754
cryptd_aead_decrypt(struct crypto_async_request * areq,int err)755 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
756 {
757 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
758 struct crypto_aead *child = ctx->child;
759 struct aead_request *req;
760
761 req = container_of(areq, struct aead_request, base);
762 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
763 }
764
cryptd_aead_enqueue(struct aead_request * req,crypto_completion_t compl)765 static int cryptd_aead_enqueue(struct aead_request *req,
766 crypto_completion_t compl)
767 {
768 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
769 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
770 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
771
772 rctx->complete = req->base.complete;
773 req->base.complete = compl;
774 return cryptd_enqueue_request(queue, &req->base);
775 }
776
cryptd_aead_encrypt_enqueue(struct aead_request * req)777 static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
778 {
779 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
780 }
781
cryptd_aead_decrypt_enqueue(struct aead_request * req)782 static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
783 {
784 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
785 }
786
cryptd_aead_init_tfm(struct crypto_aead * tfm)787 static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
788 {
789 struct aead_instance *inst = aead_alg_instance(tfm);
790 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
791 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
792 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
793 struct crypto_aead *cipher;
794
795 cipher = crypto_spawn_aead(spawn);
796 if (IS_ERR(cipher))
797 return PTR_ERR(cipher);
798
799 ctx->child = cipher;
800 crypto_aead_set_reqsize(
801 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
802 crypto_aead_reqsize(cipher)));
803 return 0;
804 }
805
cryptd_aead_exit_tfm(struct crypto_aead * tfm)806 static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
807 {
808 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
809 crypto_free_aead(ctx->child);
810 }
811
cryptd_aead_free(struct aead_instance * inst)812 static void cryptd_aead_free(struct aead_instance *inst)
813 {
814 struct aead_instance_ctx *ctx = aead_instance_ctx(inst);
815
816 crypto_drop_aead(&ctx->aead_spawn);
817 kfree(inst);
818 }
819
cryptd_create_aead(struct crypto_template * tmpl,struct rtattr ** tb,struct crypto_attr_type * algt,struct cryptd_queue * queue)820 static int cryptd_create_aead(struct crypto_template *tmpl,
821 struct rtattr **tb,
822 struct crypto_attr_type *algt,
823 struct cryptd_queue *queue)
824 {
825 struct aead_instance_ctx *ctx;
826 struct aead_instance *inst;
827 struct aead_alg *alg;
828 u32 type;
829 u32 mask;
830 int err;
831
832 cryptd_type_and_mask(algt, &type, &mask);
833
834 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
835 if (!inst)
836 return -ENOMEM;
837
838 ctx = aead_instance_ctx(inst);
839 ctx->queue = queue;
840
841 err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst),
842 crypto_attr_alg_name(tb[1]), type, mask);
843 if (err)
844 goto err_free_inst;
845
846 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
847 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
848 if (err)
849 goto err_free_inst;
850
851 inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC |
852 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
853 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
854
855 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
856 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
857
858 inst->alg.init = cryptd_aead_init_tfm;
859 inst->alg.exit = cryptd_aead_exit_tfm;
860 inst->alg.setkey = cryptd_aead_setkey;
861 inst->alg.setauthsize = cryptd_aead_setauthsize;
862 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
863 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
864
865 inst->free = cryptd_aead_free;
866
867 err = aead_register_instance(tmpl, inst);
868 if (err) {
869 err_free_inst:
870 cryptd_aead_free(inst);
871 }
872 return err;
873 }
874
875 static struct cryptd_queue queue;
876
cryptd_create(struct crypto_template * tmpl,struct rtattr ** tb)877 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
878 {
879 struct crypto_attr_type *algt;
880
881 algt = crypto_get_attr_type(tb);
882 if (IS_ERR(algt))
883 return PTR_ERR(algt);
884
885 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
886 case CRYPTO_ALG_TYPE_SKCIPHER:
887 return cryptd_create_skcipher(tmpl, tb, algt, &queue);
888 case CRYPTO_ALG_TYPE_HASH:
889 return cryptd_create_hash(tmpl, tb, algt, &queue);
890 case CRYPTO_ALG_TYPE_AEAD:
891 return cryptd_create_aead(tmpl, tb, algt, &queue);
892 }
893
894 return -EINVAL;
895 }
896
897 static struct crypto_template cryptd_tmpl = {
898 .name = "cryptd",
899 .create = cryptd_create,
900 .module = THIS_MODULE,
901 };
902
cryptd_alloc_skcipher(const char * alg_name,u32 type,u32 mask)903 struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name,
904 u32 type, u32 mask)
905 {
906 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
907 struct cryptd_skcipher_ctx *ctx;
908 struct crypto_skcipher *tfm;
909
910 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
911 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
912 return ERR_PTR(-EINVAL);
913
914 tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask);
915 if (IS_ERR(tfm))
916 return ERR_CAST(tfm);
917
918 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
919 crypto_free_skcipher(tfm);
920 return ERR_PTR(-EINVAL);
921 }
922
923 ctx = crypto_skcipher_ctx(tfm);
924 refcount_set(&ctx->refcnt, 1);
925
926 return container_of(tfm, struct cryptd_skcipher, base);
927 }
928 EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher);
929
cryptd_skcipher_child(struct cryptd_skcipher * tfm)930 struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm)
931 {
932 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
933
934 return &ctx->child->base;
935 }
936 EXPORT_SYMBOL_GPL(cryptd_skcipher_child);
937
cryptd_skcipher_queued(struct cryptd_skcipher * tfm)938 bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm)
939 {
940 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
941
942 return refcount_read(&ctx->refcnt) - 1;
943 }
944 EXPORT_SYMBOL_GPL(cryptd_skcipher_queued);
945
cryptd_free_skcipher(struct cryptd_skcipher * tfm)946 void cryptd_free_skcipher(struct cryptd_skcipher *tfm)
947 {
948 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
949
950 if (refcount_dec_and_test(&ctx->refcnt))
951 crypto_free_skcipher(&tfm->base);
952 }
953 EXPORT_SYMBOL_GPL(cryptd_free_skcipher);
954
cryptd_alloc_ahash(const char * alg_name,u32 type,u32 mask)955 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
956 u32 type, u32 mask)
957 {
958 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
959 struct cryptd_hash_ctx *ctx;
960 struct crypto_ahash *tfm;
961
962 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
963 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
964 return ERR_PTR(-EINVAL);
965 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
966 if (IS_ERR(tfm))
967 return ERR_CAST(tfm);
968 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
969 crypto_free_ahash(tfm);
970 return ERR_PTR(-EINVAL);
971 }
972
973 ctx = crypto_ahash_ctx(tfm);
974 refcount_set(&ctx->refcnt, 1);
975
976 return __cryptd_ahash_cast(tfm);
977 }
978 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
979
cryptd_ahash_child(struct cryptd_ahash * tfm)980 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
981 {
982 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
983
984 return ctx->child;
985 }
986 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
987
cryptd_shash_desc(struct ahash_request * req)988 struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
989 {
990 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
991 return &rctx->desc;
992 }
993 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
994
cryptd_ahash_queued(struct cryptd_ahash * tfm)995 bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
996 {
997 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
998
999 return refcount_read(&ctx->refcnt) - 1;
1000 }
1001 EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1002
cryptd_free_ahash(struct cryptd_ahash * tfm)1003 void cryptd_free_ahash(struct cryptd_ahash *tfm)
1004 {
1005 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1006
1007 if (refcount_dec_and_test(&ctx->refcnt))
1008 crypto_free_ahash(&tfm->base);
1009 }
1010 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1011
cryptd_alloc_aead(const char * alg_name,u32 type,u32 mask)1012 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1013 u32 type, u32 mask)
1014 {
1015 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1016 struct cryptd_aead_ctx *ctx;
1017 struct crypto_aead *tfm;
1018
1019 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1020 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1021 return ERR_PTR(-EINVAL);
1022 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1023 if (IS_ERR(tfm))
1024 return ERR_CAST(tfm);
1025 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1026 crypto_free_aead(tfm);
1027 return ERR_PTR(-EINVAL);
1028 }
1029
1030 ctx = crypto_aead_ctx(tfm);
1031 refcount_set(&ctx->refcnt, 1);
1032
1033 return __cryptd_aead_cast(tfm);
1034 }
1035 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1036
cryptd_aead_child(struct cryptd_aead * tfm)1037 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1038 {
1039 struct cryptd_aead_ctx *ctx;
1040 ctx = crypto_aead_ctx(&tfm->base);
1041 return ctx->child;
1042 }
1043 EXPORT_SYMBOL_GPL(cryptd_aead_child);
1044
cryptd_aead_queued(struct cryptd_aead * tfm)1045 bool cryptd_aead_queued(struct cryptd_aead *tfm)
1046 {
1047 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1048
1049 return refcount_read(&ctx->refcnt) - 1;
1050 }
1051 EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1052
cryptd_free_aead(struct cryptd_aead * tfm)1053 void cryptd_free_aead(struct cryptd_aead *tfm)
1054 {
1055 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1056
1057 if (refcount_dec_and_test(&ctx->refcnt))
1058 crypto_free_aead(&tfm->base);
1059 }
1060 EXPORT_SYMBOL_GPL(cryptd_free_aead);
1061
cryptd_init(void)1062 static int __init cryptd_init(void)
1063 {
1064 int err;
1065
1066 cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
1067 1);
1068 if (!cryptd_wq)
1069 return -ENOMEM;
1070
1071 err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen);
1072 if (err)
1073 goto err_destroy_wq;
1074
1075 err = crypto_register_template(&cryptd_tmpl);
1076 if (err)
1077 goto err_fini_queue;
1078
1079 return 0;
1080
1081 err_fini_queue:
1082 cryptd_fini_queue(&queue);
1083 err_destroy_wq:
1084 destroy_workqueue(cryptd_wq);
1085 return err;
1086 }
1087
cryptd_exit(void)1088 static void __exit cryptd_exit(void)
1089 {
1090 destroy_workqueue(cryptd_wq);
1091 cryptd_fini_queue(&queue);
1092 crypto_unregister_template(&cryptd_tmpl);
1093 }
1094
1095 subsys_initcall(cryptd_init);
1096 module_exit(cryptd_exit);
1097
1098 MODULE_LICENSE("GPL");
1099 MODULE_DESCRIPTION("Software async crypto daemon");
1100 MODULE_ALIAS_CRYPTO("cryptd");
1101
