blob: 7921251cdb13dccf54841da79b7c9ab7d34fae25 [file] [log] [blame]
Herbert Xu124b53d2007-04-16 20:49:20 +10001/*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
Adrian Hoban298c9262010-09-20 16:05:12 +08006 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
Herbert Xu124b53d2007-04-16 20:49:20 +100013 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20#include <crypto/algapi.h>
Herbert Xu18e33e62008-07-10 16:01:22 +080021#include <crypto/internal/hash.h>
Adrian Hoban298c9262010-09-20 16:05:12 +080022#include <crypto/internal/aead.h>
Huang Ying1cac2cb2009-01-18 16:19:46 +110023#include <crypto/cryptd.h>
Huang Ying254eff72009-02-19 14:42:19 +080024#include <crypto/crypto_wq.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100025#include <linux/err.h>
26#include <linux/init.h>
27#include <linux/kernel.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100028#include <linux/list.h>
29#include <linux/module.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100030#include <linux/scatterlist.h>
31#include <linux/sched.h>
32#include <linux/slab.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100033
Huang Ying254eff72009-02-19 14:42:19 +080034#define CRYPTD_MAX_CPU_QLEN 100
Herbert Xu124b53d2007-04-16 20:49:20 +100035
Huang Ying254eff72009-02-19 14:42:19 +080036struct cryptd_cpu_queue {
Herbert Xu124b53d2007-04-16 20:49:20 +100037 struct crypto_queue queue;
Huang Ying254eff72009-02-19 14:42:19 +080038 struct work_struct work;
39};
40
41struct cryptd_queue {
Tejun Heoa29d8b82010-02-02 14:39:15 +090042 struct cryptd_cpu_queue __percpu *cpu_queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100043};
44
45struct cryptd_instance_ctx {
46 struct crypto_spawn spawn;
Huang Ying254eff72009-02-19 14:42:19 +080047 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100048};
49
Herbert Xu46309d82009-07-12 21:38:59 +080050struct hashd_instance_ctx {
51 struct crypto_shash_spawn spawn;
52 struct cryptd_queue *queue;
53};
54
Adrian Hoban298c9262010-09-20 16:05:12 +080055struct aead_instance_ctx {
56 struct crypto_aead_spawn aead_spawn;
57 struct cryptd_queue *queue;
58};
59
Herbert Xu124b53d2007-04-16 20:49:20 +100060struct cryptd_blkcipher_ctx {
61 struct crypto_blkcipher *child;
62};
63
64struct cryptd_blkcipher_request_ctx {
65 crypto_completion_t complete;
66};
67
Loc Hob8a28252008-05-14 21:23:00 +080068struct cryptd_hash_ctx {
Herbert Xu46309d82009-07-12 21:38:59 +080069 struct crypto_shash *child;
Loc Hob8a28252008-05-14 21:23:00 +080070};
71
72struct cryptd_hash_request_ctx {
73 crypto_completion_t complete;
Herbert Xu46309d82009-07-12 21:38:59 +080074 struct shash_desc desc;
Loc Hob8a28252008-05-14 21:23:00 +080075};
Herbert Xu124b53d2007-04-16 20:49:20 +100076
Adrian Hoban298c9262010-09-20 16:05:12 +080077struct cryptd_aead_ctx {
78 struct crypto_aead *child;
79};
80
81struct cryptd_aead_request_ctx {
82 crypto_completion_t complete;
83};
84
Huang Ying254eff72009-02-19 14:42:19 +080085static void cryptd_queue_worker(struct work_struct *work);
86
87static int cryptd_init_queue(struct cryptd_queue *queue,
88 unsigned int max_cpu_qlen)
89{
90 int cpu;
91 struct cryptd_cpu_queue *cpu_queue;
92
93 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
94 if (!queue->cpu_queue)
95 return -ENOMEM;
96 for_each_possible_cpu(cpu) {
97 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
98 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
99 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
100 }
101 return 0;
102}
103
104static void cryptd_fini_queue(struct cryptd_queue *queue)
105{
106 int cpu;
107 struct cryptd_cpu_queue *cpu_queue;
108
109 for_each_possible_cpu(cpu) {
110 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
111 BUG_ON(cpu_queue->queue.qlen);
112 }
113 free_percpu(queue->cpu_queue);
114}
115
116static int cryptd_enqueue_request(struct cryptd_queue *queue,
117 struct crypto_async_request *request)
118{
119 int cpu, err;
120 struct cryptd_cpu_queue *cpu_queue;
121
122 cpu = get_cpu();
Christoph Lameter0b44f482009-10-03 19:48:23 +0900123 cpu_queue = this_cpu_ptr(queue->cpu_queue);
Huang Ying254eff72009-02-19 14:42:19 +0800124 err = crypto_enqueue_request(&cpu_queue->queue, request);
125 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
126 put_cpu();
127
128 return err;
129}
130
131/* Called in workqueue context, do one real cryption work (via
132 * req->complete) and reschedule itself if there are more work to
133 * do. */
134static void cryptd_queue_worker(struct work_struct *work)
135{
136 struct cryptd_cpu_queue *cpu_queue;
137 struct crypto_async_request *req, *backlog;
138
139 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300140 /*
141 * Only handle one request at a time to avoid hogging crypto workqueue.
142 * preempt_disable/enable is used to prevent being preempted by
143 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
144 * cryptd_enqueue_request() being accessed from software interrupts.
145 */
146 local_bh_disable();
Huang Ying254eff72009-02-19 14:42:19 +0800147 preempt_disable();
148 backlog = crypto_get_backlog(&cpu_queue->queue);
149 req = crypto_dequeue_request(&cpu_queue->queue);
150 preempt_enable();
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300151 local_bh_enable();
Huang Ying254eff72009-02-19 14:42:19 +0800152
153 if (!req)
154 return;
155
156 if (backlog)
157 backlog->complete(backlog, -EINPROGRESS);
158 req->complete(req, 0);
159
160 if (cpu_queue->queue.qlen)
161 queue_work(kcrypto_wq, &cpu_queue->work);
162}
163
164static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
Herbert Xu124b53d2007-04-16 20:49:20 +1000165{
166 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
167 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
Huang Ying254eff72009-02-19 14:42:19 +0800168 return ictx->queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000169}
170
Stephan Mueller466a7b92015-03-30 21:57:06 +0200171static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
172 u32 *mask)
173{
174 struct crypto_attr_type *algt;
175
176 algt = crypto_get_attr_type(tb);
177 if (IS_ERR(algt))
178 return;
Herbert Xuf6da3202015-07-09 07:17:19 +0800179
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800180 *type |= algt->type & CRYPTO_ALG_INTERNAL;
181 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200182}
183
Herbert Xu124b53d2007-04-16 20:49:20 +1000184static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
185 const u8 *key, unsigned int keylen)
186{
187 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
188 struct crypto_blkcipher *child = ctx->child;
189 int err;
190
191 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
192 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
193 CRYPTO_TFM_REQ_MASK);
194 err = crypto_blkcipher_setkey(child, key, keylen);
195 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
196 CRYPTO_TFM_RES_MASK);
197 return err;
198}
199
200static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
201 struct crypto_blkcipher *child,
202 int err,
203 int (*crypt)(struct blkcipher_desc *desc,
204 struct scatterlist *dst,
205 struct scatterlist *src,
206 unsigned int len))
207{
208 struct cryptd_blkcipher_request_ctx *rctx;
209 struct blkcipher_desc desc;
210
211 rctx = ablkcipher_request_ctx(req);
212
Herbert Xu93aa7f82008-05-07 21:10:13 +0800213 if (unlikely(err == -EINPROGRESS))
214 goto out;
Herbert Xu124b53d2007-04-16 20:49:20 +1000215
216 desc.tfm = child;
217 desc.info = req->info;
218 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
219
220 err = crypt(&desc, req->dst, req->src, req->nbytes);
221
222 req->base.complete = rctx->complete;
223
Herbert Xu93aa7f82008-05-07 21:10:13 +0800224out:
Herbert Xu124b53d2007-04-16 20:49:20 +1000225 local_bh_disable();
Herbert Xu93aa7f82008-05-07 21:10:13 +0800226 rctx->complete(&req->base, err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000227 local_bh_enable();
228}
229
230static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
231{
232 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
233 struct crypto_blkcipher *child = ctx->child;
234
235 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
236 crypto_blkcipher_crt(child)->encrypt);
237}
238
239static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
240{
241 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
242 struct crypto_blkcipher *child = ctx->child;
243
244 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
245 crypto_blkcipher_crt(child)->decrypt);
246}
247
248static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700249 crypto_completion_t compl)
Herbert Xu124b53d2007-04-16 20:49:20 +1000250{
251 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
252 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800253 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000254
Huang Ying254eff72009-02-19 14:42:19 +0800255 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
Herbert Xu124b53d2007-04-16 20:49:20 +1000256 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700257 req->base.complete = compl;
Herbert Xu124b53d2007-04-16 20:49:20 +1000258
Huang Ying254eff72009-02-19 14:42:19 +0800259 return cryptd_enqueue_request(queue, &req->base);
Herbert Xu124b53d2007-04-16 20:49:20 +1000260}
261
262static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
263{
264 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
265}
266
267static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
268{
269 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
270}
271
272static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
273{
274 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
275 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
276 struct crypto_spawn *spawn = &ictx->spawn;
277 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
278 struct crypto_blkcipher *cipher;
279
280 cipher = crypto_spawn_blkcipher(spawn);
281 if (IS_ERR(cipher))
282 return PTR_ERR(cipher);
283
284 ctx->child = cipher;
285 tfm->crt_ablkcipher.reqsize =
286 sizeof(struct cryptd_blkcipher_request_ctx);
287 return 0;
288}
289
290static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
291{
292 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000293
294 crypto_free_blkcipher(ctx->child);
295}
296
Herbert Xu9b8c4562015-05-21 15:10:57 +0800297static int cryptd_init_instance(struct crypto_instance *inst,
298 struct crypto_alg *alg)
299{
300 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
301 "cryptd(%s)",
302 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
303 return -ENAMETOOLONG;
304
305 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
306
307 inst->alg.cra_priority = alg->cra_priority + 50;
308 inst->alg.cra_blocksize = alg->cra_blocksize;
309 inst->alg.cra_alignmask = alg->cra_alignmask;
310
311 return 0;
312}
313
Herbert Xu0b535ad2009-07-14 19:11:32 +0800314static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
315 unsigned int tail)
Herbert Xu124b53d2007-04-16 20:49:20 +1000316{
Herbert Xu0b535ad2009-07-14 19:11:32 +0800317 char *p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000318 struct crypto_instance *inst;
Herbert Xu124b53d2007-04-16 20:49:20 +1000319 int err;
320
Herbert Xu0b535ad2009-07-14 19:11:32 +0800321 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
322 if (!p)
323 return ERR_PTR(-ENOMEM);
324
325 inst = (void *)(p + head);
Herbert Xu124b53d2007-04-16 20:49:20 +1000326
Herbert Xu9b8c4562015-05-21 15:10:57 +0800327 err = cryptd_init_instance(inst, alg);
328 if (err)
Herbert Xu124b53d2007-04-16 20:49:20 +1000329 goto out_free_inst;
330
Herbert Xu124b53d2007-04-16 20:49:20 +1000331out:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800332 return p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000333
334out_free_inst:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800335 kfree(p);
336 p = ERR_PTR(err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000337 goto out;
338}
339
Herbert Xu9cd899a2009-07-14 18:45:45 +0800340static int cryptd_create_blkcipher(struct crypto_template *tmpl,
341 struct rtattr **tb,
342 struct cryptd_queue *queue)
Herbert Xu124b53d2007-04-16 20:49:20 +1000343{
Herbert Xu46309d82009-07-12 21:38:59 +0800344 struct cryptd_instance_ctx *ctx;
Herbert Xu124b53d2007-04-16 20:49:20 +1000345 struct crypto_instance *inst;
346 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200347 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
348 u32 mask = CRYPTO_ALG_TYPE_MASK;
Herbert Xu46309d82009-07-12 21:38:59 +0800349 int err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000350
Stephan Mueller466a7b92015-03-30 21:57:06 +0200351 cryptd_check_internal(tb, &type, &mask);
352
353 alg = crypto_get_attr_alg(tb, type, mask);
Herbert Xu124b53d2007-04-16 20:49:20 +1000354 if (IS_ERR(alg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800355 return PTR_ERR(alg);
Herbert Xu124b53d2007-04-16 20:49:20 +1000356
Herbert Xu0b535ad2009-07-14 19:11:32 +0800357 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800358 err = PTR_ERR(inst);
Herbert Xu124b53d2007-04-16 20:49:20 +1000359 if (IS_ERR(inst))
360 goto out_put_alg;
361
Herbert Xu46309d82009-07-12 21:38:59 +0800362 ctx = crypto_instance_ctx(inst);
363 ctx->queue = queue;
364
365 err = crypto_init_spawn(&ctx->spawn, alg, inst,
366 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
367 if (err)
368 goto out_free_inst;
369
Stephan Mueller466a7b92015-03-30 21:57:06 +0200370 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
371 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
372 type |= CRYPTO_ALG_INTERNAL;
373 inst->alg.cra_flags = type;
Herbert Xu124b53d2007-04-16 20:49:20 +1000374 inst->alg.cra_type = &crypto_ablkcipher_type;
375
376 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
377 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
378 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
379
Herbert Xu927eead2007-11-27 21:15:31 +0800380 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
381
Herbert Xu124b53d2007-04-16 20:49:20 +1000382 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
383
384 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
385 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
386
387 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
388 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
389 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
390
Herbert Xu9cd899a2009-07-14 18:45:45 +0800391 err = crypto_register_instance(tmpl, inst);
392 if (err) {
393 crypto_drop_spawn(&ctx->spawn);
394out_free_inst:
395 kfree(inst);
396 }
397
Herbert Xu124b53d2007-04-16 20:49:20 +1000398out_put_alg:
399 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800400 return err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000401}
402
Loc Hob8a28252008-05-14 21:23:00 +0800403static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
404{
405 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800406 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
407 struct crypto_shash_spawn *spawn = &ictx->spawn;
Loc Hob8a28252008-05-14 21:23:00 +0800408 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800409 struct crypto_shash *hash;
Loc Hob8a28252008-05-14 21:23:00 +0800410
Herbert Xu46309d82009-07-12 21:38:59 +0800411 hash = crypto_spawn_shash(spawn);
412 if (IS_ERR(hash))
413 return PTR_ERR(hash);
Loc Hob8a28252008-05-14 21:23:00 +0800414
Herbert Xu46309d82009-07-12 21:38:59 +0800415 ctx->child = hash;
Herbert Xu0d6669e2009-07-12 23:06:33 +0800416 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
417 sizeof(struct cryptd_hash_request_ctx) +
418 crypto_shash_descsize(hash));
Loc Hob8a28252008-05-14 21:23:00 +0800419 return 0;
420}
421
422static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
423{
424 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Loc Hob8a28252008-05-14 21:23:00 +0800425
Herbert Xu46309d82009-07-12 21:38:59 +0800426 crypto_free_shash(ctx->child);
Loc Hob8a28252008-05-14 21:23:00 +0800427}
428
429static int cryptd_hash_setkey(struct crypto_ahash *parent,
430 const u8 *key, unsigned int keylen)
431{
432 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
Herbert Xu46309d82009-07-12 21:38:59 +0800433 struct crypto_shash *child = ctx->child;
Loc Hob8a28252008-05-14 21:23:00 +0800434 int err;
435
Herbert Xu46309d82009-07-12 21:38:59 +0800436 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
437 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
438 CRYPTO_TFM_REQ_MASK);
439 err = crypto_shash_setkey(child, key, keylen);
440 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
441 CRYPTO_TFM_RES_MASK);
Loc Hob8a28252008-05-14 21:23:00 +0800442 return err;
443}
444
445static int cryptd_hash_enqueue(struct ahash_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700446 crypto_completion_t compl)
Loc Hob8a28252008-05-14 21:23:00 +0800447{
448 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
449 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800450 struct cryptd_queue *queue =
451 cryptd_get_queue(crypto_ahash_tfm(tfm));
Loc Hob8a28252008-05-14 21:23:00 +0800452
453 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700454 req->base.complete = compl;
Loc Hob8a28252008-05-14 21:23:00 +0800455
Huang Ying254eff72009-02-19 14:42:19 +0800456 return cryptd_enqueue_request(queue, &req->base);
Loc Hob8a28252008-05-14 21:23:00 +0800457}
458
459static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
460{
Herbert Xu46309d82009-07-12 21:38:59 +0800461 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
462 struct crypto_shash *child = ctx->child;
463 struct ahash_request *req = ahash_request_cast(req_async);
464 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
465 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800466
467 if (unlikely(err == -EINPROGRESS))
468 goto out;
469
Herbert Xu46309d82009-07-12 21:38:59 +0800470 desc->tfm = child;
471 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800472
Herbert Xu46309d82009-07-12 21:38:59 +0800473 err = crypto_shash_init(desc);
Loc Hob8a28252008-05-14 21:23:00 +0800474
475 req->base.complete = rctx->complete;
476
477out:
478 local_bh_disable();
479 rctx->complete(&req->base, err);
480 local_bh_enable();
481}
482
483static int cryptd_hash_init_enqueue(struct ahash_request *req)
484{
485 return cryptd_hash_enqueue(req, cryptd_hash_init);
486}
487
488static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
489{
Herbert Xu46309d82009-07-12 21:38:59 +0800490 struct ahash_request *req = ahash_request_cast(req_async);
Loc Hob8a28252008-05-14 21:23:00 +0800491 struct cryptd_hash_request_ctx *rctx;
Loc Hob8a28252008-05-14 21:23:00 +0800492
493 rctx = ahash_request_ctx(req);
494
495 if (unlikely(err == -EINPROGRESS))
496 goto out;
497
Herbert Xu46309d82009-07-12 21:38:59 +0800498 err = shash_ahash_update(req, &rctx->desc);
Loc Hob8a28252008-05-14 21:23:00 +0800499
500 req->base.complete = rctx->complete;
501
502out:
503 local_bh_disable();
504 rctx->complete(&req->base, err);
505 local_bh_enable();
506}
507
508static int cryptd_hash_update_enqueue(struct ahash_request *req)
509{
510 return cryptd_hash_enqueue(req, cryptd_hash_update);
511}
512
513static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
514{
Herbert Xu46309d82009-07-12 21:38:59 +0800515 struct ahash_request *req = ahash_request_cast(req_async);
516 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Loc Hob8a28252008-05-14 21:23:00 +0800517
518 if (unlikely(err == -EINPROGRESS))
519 goto out;
520
Herbert Xu46309d82009-07-12 21:38:59 +0800521 err = crypto_shash_final(&rctx->desc, req->result);
Loc Hob8a28252008-05-14 21:23:00 +0800522
523 req->base.complete = rctx->complete;
524
525out:
526 local_bh_disable();
527 rctx->complete(&req->base, err);
528 local_bh_enable();
529}
530
531static int cryptd_hash_final_enqueue(struct ahash_request *req)
532{
533 return cryptd_hash_enqueue(req, cryptd_hash_final);
534}
535
Herbert Xu6fba00d2009-07-22 11:10:22 +0800536static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
537{
538 struct ahash_request *req = ahash_request_cast(req_async);
539 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
540
541 if (unlikely(err == -EINPROGRESS))
542 goto out;
543
544 err = shash_ahash_finup(req, &rctx->desc);
545
546 req->base.complete = rctx->complete;
547
548out:
549 local_bh_disable();
550 rctx->complete(&req->base, err);
551 local_bh_enable();
552}
553
554static int cryptd_hash_finup_enqueue(struct ahash_request *req)
555{
556 return cryptd_hash_enqueue(req, cryptd_hash_finup);
557}
558
Loc Hob8a28252008-05-14 21:23:00 +0800559static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
560{
Herbert Xu46309d82009-07-12 21:38:59 +0800561 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
562 struct crypto_shash *child = ctx->child;
563 struct ahash_request *req = ahash_request_cast(req_async);
564 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
565 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800566
567 if (unlikely(err == -EINPROGRESS))
568 goto out;
569
Herbert Xu46309d82009-07-12 21:38:59 +0800570 desc->tfm = child;
571 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800572
Herbert Xu46309d82009-07-12 21:38:59 +0800573 err = shash_ahash_digest(req, desc);
Loc Hob8a28252008-05-14 21:23:00 +0800574
575 req->base.complete = rctx->complete;
576
577out:
578 local_bh_disable();
579 rctx->complete(&req->base, err);
580 local_bh_enable();
581}
582
583static int cryptd_hash_digest_enqueue(struct ahash_request *req)
584{
585 return cryptd_hash_enqueue(req, cryptd_hash_digest);
586}
587
Herbert Xu6fba00d2009-07-22 11:10:22 +0800588static int cryptd_hash_export(struct ahash_request *req, void *out)
589{
590 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
591
592 return crypto_shash_export(&rctx->desc, out);
593}
594
595static int cryptd_hash_import(struct ahash_request *req, const void *in)
596{
597 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
598
599 return crypto_shash_import(&rctx->desc, in);
600}
601
Herbert Xu9cd899a2009-07-14 18:45:45 +0800602static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
603 struct cryptd_queue *queue)
Loc Hob8a28252008-05-14 21:23:00 +0800604{
Herbert Xu46309d82009-07-12 21:38:59 +0800605 struct hashd_instance_ctx *ctx;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800606 struct ahash_instance *inst;
Herbert Xu46309d82009-07-12 21:38:59 +0800607 struct shash_alg *salg;
Loc Hob8a28252008-05-14 21:23:00 +0800608 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200609 u32 type = 0;
610 u32 mask = 0;
Herbert Xu46309d82009-07-12 21:38:59 +0800611 int err;
Loc Hob8a28252008-05-14 21:23:00 +0800612
Stephan Mueller466a7b92015-03-30 21:57:06 +0200613 cryptd_check_internal(tb, &type, &mask);
614
615 salg = shash_attr_alg(tb[1], type, mask);
Herbert Xu46309d82009-07-12 21:38:59 +0800616 if (IS_ERR(salg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800617 return PTR_ERR(salg);
Loc Hob8a28252008-05-14 21:23:00 +0800618
Herbert Xu46309d82009-07-12 21:38:59 +0800619 alg = &salg->base;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800620 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
621 sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800622 err = PTR_ERR(inst);
Loc Hob8a28252008-05-14 21:23:00 +0800623 if (IS_ERR(inst))
624 goto out_put_alg;
625
Herbert Xu0b535ad2009-07-14 19:11:32 +0800626 ctx = ahash_instance_ctx(inst);
Herbert Xu46309d82009-07-12 21:38:59 +0800627 ctx->queue = queue;
628
Herbert Xu0b535ad2009-07-14 19:11:32 +0800629 err = crypto_init_shash_spawn(&ctx->spawn, salg,
630 ahash_crypto_instance(inst));
Herbert Xu46309d82009-07-12 21:38:59 +0800631 if (err)
632 goto out_free_inst;
633
Stephan Mueller466a7b92015-03-30 21:57:06 +0200634 type = CRYPTO_ALG_ASYNC;
635 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
636 type |= CRYPTO_ALG_INTERNAL;
637 inst->alg.halg.base.cra_flags = type;
Loc Hob8a28252008-05-14 21:23:00 +0800638
Herbert Xu0b535ad2009-07-14 19:11:32 +0800639 inst->alg.halg.digestsize = salg->digestsize;
Wang, Rui Y1a078342015-11-29 22:45:34 +0800640 inst->alg.halg.statesize = salg->statesize;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800641 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
Loc Hob8a28252008-05-14 21:23:00 +0800642
Herbert Xu0b535ad2009-07-14 19:11:32 +0800643 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
644 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
Loc Hob8a28252008-05-14 21:23:00 +0800645
Herbert Xu0b535ad2009-07-14 19:11:32 +0800646 inst->alg.init = cryptd_hash_init_enqueue;
647 inst->alg.update = cryptd_hash_update_enqueue;
648 inst->alg.final = cryptd_hash_final_enqueue;
Herbert Xu6fba00d2009-07-22 11:10:22 +0800649 inst->alg.finup = cryptd_hash_finup_enqueue;
650 inst->alg.export = cryptd_hash_export;
651 inst->alg.import = cryptd_hash_import;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800652 inst->alg.setkey = cryptd_hash_setkey;
653 inst->alg.digest = cryptd_hash_digest_enqueue;
Loc Hob8a28252008-05-14 21:23:00 +0800654
Herbert Xu0b535ad2009-07-14 19:11:32 +0800655 err = ahash_register_instance(tmpl, inst);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800656 if (err) {
657 crypto_drop_shash(&ctx->spawn);
658out_free_inst:
659 kfree(inst);
660 }
661
Loc Hob8a28252008-05-14 21:23:00 +0800662out_put_alg:
663 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800664 return err;
Loc Hob8a28252008-05-14 21:23:00 +0800665}
666
Herbert Xu92b98762015-05-28 22:08:01 +0800667static int cryptd_aead_setkey(struct crypto_aead *parent,
668 const u8 *key, unsigned int keylen)
669{
670 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
671 struct crypto_aead *child = ctx->child;
672
673 return crypto_aead_setkey(child, key, keylen);
674}
675
676static int cryptd_aead_setauthsize(struct crypto_aead *parent,
677 unsigned int authsize)
678{
679 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
680 struct crypto_aead *child = ctx->child;
681
682 return crypto_aead_setauthsize(child, authsize);
683}
684
Adrian Hoban298c9262010-09-20 16:05:12 +0800685static void cryptd_aead_crypt(struct aead_request *req,
686 struct crypto_aead *child,
687 int err,
688 int (*crypt)(struct aead_request *req))
689{
690 struct cryptd_aead_request_ctx *rctx;
Herbert Xuec9f2002015-07-06 19:11:03 +0800691 crypto_completion_t compl;
692
Adrian Hoban298c9262010-09-20 16:05:12 +0800693 rctx = aead_request_ctx(req);
Herbert Xuec9f2002015-07-06 19:11:03 +0800694 compl = rctx->complete;
Adrian Hoban298c9262010-09-20 16:05:12 +0800695
696 if (unlikely(err == -EINPROGRESS))
697 goto out;
698 aead_request_set_tfm(req, child);
699 err = crypt( req );
Adrian Hoban298c9262010-09-20 16:05:12 +0800700out:
701 local_bh_disable();
Herbert Xuec9f2002015-07-06 19:11:03 +0800702 compl(&req->base, err);
Adrian Hoban298c9262010-09-20 16:05:12 +0800703 local_bh_enable();
704}
705
706static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
707{
708 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
709 struct crypto_aead *child = ctx->child;
710 struct aead_request *req;
711
712 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800713 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800714}
715
716static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
717{
718 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
719 struct crypto_aead *child = ctx->child;
720 struct aead_request *req;
721
722 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800723 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800724}
725
726static int cryptd_aead_enqueue(struct aead_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700727 crypto_completion_t compl)
Adrian Hoban298c9262010-09-20 16:05:12 +0800728{
729 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
730 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
731 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
732
733 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700734 req->base.complete = compl;
Adrian Hoban298c9262010-09-20 16:05:12 +0800735 return cryptd_enqueue_request(queue, &req->base);
736}
737
738static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
739{
740 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
741}
742
743static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
744{
745 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
746}
747
Herbert Xuf614e542015-05-28 22:08:04 +0800748static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800749{
Herbert Xuf614e542015-05-28 22:08:04 +0800750 struct aead_instance *inst = aead_alg_instance(tfm);
751 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800752 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
Herbert Xuf614e542015-05-28 22:08:04 +0800753 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800754 struct crypto_aead *cipher;
755
756 cipher = crypto_spawn_aead(spawn);
757 if (IS_ERR(cipher))
758 return PTR_ERR(cipher);
759
Adrian Hoban298c9262010-09-20 16:05:12 +0800760 ctx->child = cipher;
Herbert Xuec9f2002015-07-06 19:11:03 +0800761 crypto_aead_set_reqsize(
762 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
763 crypto_aead_reqsize(cipher)));
Adrian Hoban298c9262010-09-20 16:05:12 +0800764 return 0;
765}
766
Herbert Xuf614e542015-05-28 22:08:04 +0800767static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800768{
Herbert Xuf614e542015-05-28 22:08:04 +0800769 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800770 crypto_free_aead(ctx->child);
771}
772
773static int cryptd_create_aead(struct crypto_template *tmpl,
774 struct rtattr **tb,
775 struct cryptd_queue *queue)
776{
777 struct aead_instance_ctx *ctx;
Herbert Xuf614e542015-05-28 22:08:04 +0800778 struct aead_instance *inst;
779 struct aead_alg *alg;
Herbert Xu9b8c4562015-05-21 15:10:57 +0800780 const char *name;
781 u32 type = 0;
Herbert Xuec9f2002015-07-06 19:11:03 +0800782 u32 mask = CRYPTO_ALG_ASYNC;
Adrian Hoban298c9262010-09-20 16:05:12 +0800783 int err;
784
Stephan Mueller466a7b92015-03-30 21:57:06 +0200785 cryptd_check_internal(tb, &type, &mask);
786
Herbert Xu9b8c4562015-05-21 15:10:57 +0800787 name = crypto_attr_alg_name(tb[1]);
788 if (IS_ERR(name))
789 return PTR_ERR(name);
Adrian Hoban298c9262010-09-20 16:05:12 +0800790
Herbert Xu9b8c4562015-05-21 15:10:57 +0800791 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
792 if (!inst)
793 return -ENOMEM;
Adrian Hoban298c9262010-09-20 16:05:12 +0800794
Herbert Xuf614e542015-05-28 22:08:04 +0800795 ctx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800796 ctx->queue = queue;
797
Herbert Xuf614e542015-05-28 22:08:04 +0800798 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
Herbert Xu9b8c4562015-05-21 15:10:57 +0800799 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
Adrian Hoban298c9262010-09-20 16:05:12 +0800800 if (err)
801 goto out_free_inst;
802
Herbert Xuf614e542015-05-28 22:08:04 +0800803 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
804 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
Herbert Xu9b8c4562015-05-21 15:10:57 +0800805 if (err)
806 goto out_drop_aead;
807
Herbert Xuf614e542015-05-28 22:08:04 +0800808 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800809 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
Herbert Xuf614e542015-05-28 22:08:04 +0800810 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
Adrian Hoban298c9262010-09-20 16:05:12 +0800811
Herbert Xuf614e542015-05-28 22:08:04 +0800812 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
813 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
814
815 inst->alg.init = cryptd_aead_init_tfm;
816 inst->alg.exit = cryptd_aead_exit_tfm;
817 inst->alg.setkey = cryptd_aead_setkey;
818 inst->alg.setauthsize = cryptd_aead_setauthsize;
819 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
820 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
821
822 err = aead_register_instance(tmpl, inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800823 if (err) {
Herbert Xu9b8c4562015-05-21 15:10:57 +0800824out_drop_aead:
825 crypto_drop_aead(&ctx->aead_spawn);
Adrian Hoban298c9262010-09-20 16:05:12 +0800826out_free_inst:
827 kfree(inst);
828 }
Adrian Hoban298c9262010-09-20 16:05:12 +0800829 return err;
830}
831
Huang Ying254eff72009-02-19 14:42:19 +0800832static struct cryptd_queue queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000833
Herbert Xu9cd899a2009-07-14 18:45:45 +0800834static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
Herbert Xu124b53d2007-04-16 20:49:20 +1000835{
836 struct crypto_attr_type *algt;
837
838 algt = crypto_get_attr_type(tb);
839 if (IS_ERR(algt))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800840 return PTR_ERR(algt);
Herbert Xu124b53d2007-04-16 20:49:20 +1000841
842 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
843 case CRYPTO_ALG_TYPE_BLKCIPHER:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800844 return cryptd_create_blkcipher(tmpl, tb, &queue);
Loc Hob8a28252008-05-14 21:23:00 +0800845 case CRYPTO_ALG_TYPE_DIGEST:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800846 return cryptd_create_hash(tmpl, tb, &queue);
Adrian Hoban298c9262010-09-20 16:05:12 +0800847 case CRYPTO_ALG_TYPE_AEAD:
848 return cryptd_create_aead(tmpl, tb, &queue);
Herbert Xu124b53d2007-04-16 20:49:20 +1000849 }
850
Herbert Xu9cd899a2009-07-14 18:45:45 +0800851 return -EINVAL;
Herbert Xu124b53d2007-04-16 20:49:20 +1000852}
853
854static void cryptd_free(struct crypto_instance *inst)
855{
856 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800857 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800858 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800859
860 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
861 case CRYPTO_ALG_TYPE_AHASH:
862 crypto_drop_shash(&hctx->spawn);
863 kfree(ahash_instance(inst));
864 return;
Adrian Hoban298c9262010-09-20 16:05:12 +0800865 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuf614e542015-05-28 22:08:04 +0800866 crypto_drop_aead(&aead_ctx->aead_spawn);
867 kfree(aead_instance(inst));
Adrian Hoban298c9262010-09-20 16:05:12 +0800868 return;
869 default:
870 crypto_drop_spawn(&ctx->spawn);
871 kfree(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800872 }
Herbert Xu124b53d2007-04-16 20:49:20 +1000873}
874
875static struct crypto_template cryptd_tmpl = {
876 .name = "cryptd",
Herbert Xu9cd899a2009-07-14 18:45:45 +0800877 .create = cryptd_create,
Herbert Xu124b53d2007-04-16 20:49:20 +1000878 .free = cryptd_free,
879 .module = THIS_MODULE,
880};
881
Huang Ying1cac2cb2009-01-18 16:19:46 +1100882struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
883 u32 type, u32 mask)
884{
885 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Huang Ying505fd212009-03-29 15:33:53 +0800886 struct crypto_tfm *tfm;
Huang Ying1cac2cb2009-01-18 16:19:46 +1100887
888 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
889 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
890 return ERR_PTR(-EINVAL);
Alexander Kuleshovc012a792015-11-25 23:48:28 +0600891 type = crypto_skcipher_type(type);
Huang Ying505fd212009-03-29 15:33:53 +0800892 mask &= ~CRYPTO_ALG_TYPE_MASK;
893 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
894 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100895 if (IS_ERR(tfm))
896 return ERR_CAST(tfm);
Huang Ying505fd212009-03-29 15:33:53 +0800897 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
898 crypto_free_tfm(tfm);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100899 return ERR_PTR(-EINVAL);
900 }
901
Huang Ying505fd212009-03-29 15:33:53 +0800902 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
Huang Ying1cac2cb2009-01-18 16:19:46 +1100903}
904EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
905
906struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
907{
908 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
909 return ctx->child;
910}
911EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
912
913void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
914{
915 crypto_free_ablkcipher(&tfm->base);
916}
917EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
918
Huang Yingace13662009-08-06 15:35:20 +1000919struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
920 u32 type, u32 mask)
921{
922 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
923 struct crypto_ahash *tfm;
924
925 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
926 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
927 return ERR_PTR(-EINVAL);
928 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
929 if (IS_ERR(tfm))
930 return ERR_CAST(tfm);
931 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
932 crypto_free_ahash(tfm);
933 return ERR_PTR(-EINVAL);
934 }
935
936 return __cryptd_ahash_cast(tfm);
937}
938EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
939
940struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
941{
942 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
943
944 return ctx->child;
945}
946EXPORT_SYMBOL_GPL(cryptd_ahash_child);
947
Huang Ying0e1227d2009-10-19 11:53:06 +0900948struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
949{
950 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
951 return &rctx->desc;
952}
953EXPORT_SYMBOL_GPL(cryptd_shash_desc);
954
Huang Yingace13662009-08-06 15:35:20 +1000955void cryptd_free_ahash(struct cryptd_ahash *tfm)
956{
957 crypto_free_ahash(&tfm->base);
958}
959EXPORT_SYMBOL_GPL(cryptd_free_ahash);
960
Adrian Hoban298c9262010-09-20 16:05:12 +0800961struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
962 u32 type, u32 mask)
963{
964 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
965 struct crypto_aead *tfm;
966
967 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
968 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
969 return ERR_PTR(-EINVAL);
970 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
971 if (IS_ERR(tfm))
972 return ERR_CAST(tfm);
973 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
974 crypto_free_aead(tfm);
975 return ERR_PTR(-EINVAL);
976 }
977 return __cryptd_aead_cast(tfm);
978}
979EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
980
981struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
982{
983 struct cryptd_aead_ctx *ctx;
984 ctx = crypto_aead_ctx(&tfm->base);
985 return ctx->child;
986}
987EXPORT_SYMBOL_GPL(cryptd_aead_child);
988
989void cryptd_free_aead(struct cryptd_aead *tfm)
990{
991 crypto_free_aead(&tfm->base);
992}
993EXPORT_SYMBOL_GPL(cryptd_free_aead);
994
Herbert Xu124b53d2007-04-16 20:49:20 +1000995static int __init cryptd_init(void)
996{
997 int err;
998
Huang Ying254eff72009-02-19 14:42:19 +0800999 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
Herbert Xu124b53d2007-04-16 20:49:20 +10001000 if (err)
1001 return err;
1002
1003 err = crypto_register_template(&cryptd_tmpl);
1004 if (err)
Huang Ying254eff72009-02-19 14:42:19 +08001005 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001006
1007 return err;
1008}
1009
1010static void __exit cryptd_exit(void)
1011{
Huang Ying254eff72009-02-19 14:42:19 +08001012 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001013 crypto_unregister_template(&cryptd_tmpl);
1014}
1015
Herbert Xub2bac6a2011-08-19 16:11:23 +08001016subsys_initcall(cryptd_init);
Herbert Xu124b53d2007-04-16 20:49:20 +10001017module_exit(cryptd_exit);
1018
1019MODULE_LICENSE("GPL");
1020MODULE_DESCRIPTION("Software async crypto daemon");
Kees Cook4943ba12014-11-24 16:32:38 -08001021MODULE_ALIAS_CRYPTO("cryptd");