blob: 22ba81f76764aff2edd614ad8fad2914df33a8a9 [file] [log] [blame]
Herbert Xu124b53d2007-04-16 20:49:20 +10001/*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
Adrian Hoban298c9262010-09-20 16:05:12 +08006 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
Herbert Xu124b53d2007-04-16 20:49:20 +100013 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20#include <crypto/algapi.h>
Herbert Xu18e33e62008-07-10 16:01:22 +080021#include <crypto/internal/hash.h>
Adrian Hoban298c9262010-09-20 16:05:12 +080022#include <crypto/internal/aead.h>
Huang Ying1cac2cb2009-01-18 16:19:46 +110023#include <crypto/cryptd.h>
Huang Ying254eff72009-02-19 14:42:19 +080024#include <crypto/crypto_wq.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100025#include <linux/err.h>
26#include <linux/init.h>
27#include <linux/kernel.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100028#include <linux/list.h>
29#include <linux/module.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100030#include <linux/scatterlist.h>
31#include <linux/sched.h>
32#include <linux/slab.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100033
Huang Ying254eff72009-02-19 14:42:19 +080034#define CRYPTD_MAX_CPU_QLEN 100
Herbert Xu124b53d2007-04-16 20:49:20 +100035
Huang Ying254eff72009-02-19 14:42:19 +080036struct cryptd_cpu_queue {
Herbert Xu124b53d2007-04-16 20:49:20 +100037 struct crypto_queue queue;
Huang Ying254eff72009-02-19 14:42:19 +080038 struct work_struct work;
39};
40
41struct cryptd_queue {
Tejun Heoa29d8b82010-02-02 14:39:15 +090042 struct cryptd_cpu_queue __percpu *cpu_queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100043};
44
45struct cryptd_instance_ctx {
46 struct crypto_spawn spawn;
Huang Ying254eff72009-02-19 14:42:19 +080047 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100048};
49
Herbert Xu46309d82009-07-12 21:38:59 +080050struct hashd_instance_ctx {
51 struct crypto_shash_spawn spawn;
52 struct cryptd_queue *queue;
53};
54
Adrian Hoban298c9262010-09-20 16:05:12 +080055struct aead_instance_ctx {
56 struct crypto_aead_spawn aead_spawn;
57 struct cryptd_queue *queue;
58};
59
Herbert Xu124b53d2007-04-16 20:49:20 +100060struct cryptd_blkcipher_ctx {
61 struct crypto_blkcipher *child;
62};
63
64struct cryptd_blkcipher_request_ctx {
65 crypto_completion_t complete;
66};
67
Loc Hob8a28252008-05-14 21:23:00 +080068struct cryptd_hash_ctx {
Herbert Xu46309d82009-07-12 21:38:59 +080069 struct crypto_shash *child;
Loc Hob8a28252008-05-14 21:23:00 +080070};
71
72struct cryptd_hash_request_ctx {
73 crypto_completion_t complete;
Herbert Xu46309d82009-07-12 21:38:59 +080074 struct shash_desc desc;
Loc Hob8a28252008-05-14 21:23:00 +080075};
Herbert Xu124b53d2007-04-16 20:49:20 +100076
Adrian Hoban298c9262010-09-20 16:05:12 +080077struct cryptd_aead_ctx {
78 struct crypto_aead *child;
79};
80
81struct cryptd_aead_request_ctx {
82 crypto_completion_t complete;
83};
84
Huang Ying254eff72009-02-19 14:42:19 +080085static void cryptd_queue_worker(struct work_struct *work);
86
87static int cryptd_init_queue(struct cryptd_queue *queue,
88 unsigned int max_cpu_qlen)
89{
90 int cpu;
91 struct cryptd_cpu_queue *cpu_queue;
92
93 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
94 if (!queue->cpu_queue)
95 return -ENOMEM;
96 for_each_possible_cpu(cpu) {
97 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
98 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
99 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
100 }
101 return 0;
102}
103
104static void cryptd_fini_queue(struct cryptd_queue *queue)
105{
106 int cpu;
107 struct cryptd_cpu_queue *cpu_queue;
108
109 for_each_possible_cpu(cpu) {
110 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
111 BUG_ON(cpu_queue->queue.qlen);
112 }
113 free_percpu(queue->cpu_queue);
114}
115
116static int cryptd_enqueue_request(struct cryptd_queue *queue,
117 struct crypto_async_request *request)
118{
119 int cpu, err;
120 struct cryptd_cpu_queue *cpu_queue;
121
122 cpu = get_cpu();
Christoph Lameter0b44f482009-10-03 19:48:23 +0900123 cpu_queue = this_cpu_ptr(queue->cpu_queue);
Huang Ying254eff72009-02-19 14:42:19 +0800124 err = crypto_enqueue_request(&cpu_queue->queue, request);
125 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
126 put_cpu();
127
128 return err;
129}
130
131/* Called in workqueue context, do one real cryption work (via
132 * req->complete) and reschedule itself if there are more work to
133 * do. */
134static void cryptd_queue_worker(struct work_struct *work)
135{
136 struct cryptd_cpu_queue *cpu_queue;
137 struct crypto_async_request *req, *backlog;
138
139 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300140 /*
141 * Only handle one request at a time to avoid hogging crypto workqueue.
142 * preempt_disable/enable is used to prevent being preempted by
143 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
144 * cryptd_enqueue_request() being accessed from software interrupts.
145 */
146 local_bh_disable();
Huang Ying254eff72009-02-19 14:42:19 +0800147 preempt_disable();
148 backlog = crypto_get_backlog(&cpu_queue->queue);
149 req = crypto_dequeue_request(&cpu_queue->queue);
150 preempt_enable();
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300151 local_bh_enable();
Huang Ying254eff72009-02-19 14:42:19 +0800152
153 if (!req)
154 return;
155
156 if (backlog)
157 backlog->complete(backlog, -EINPROGRESS);
158 req->complete(req, 0);
159
160 if (cpu_queue->queue.qlen)
161 queue_work(kcrypto_wq, &cpu_queue->work);
162}
163
164static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
Herbert Xu124b53d2007-04-16 20:49:20 +1000165{
166 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
167 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
Huang Ying254eff72009-02-19 14:42:19 +0800168 return ictx->queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000169}
170
Stephan Mueller466a7b92015-03-30 21:57:06 +0200171static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
172 u32 *mask)
173{
174 struct crypto_attr_type *algt;
175
176 algt = crypto_get_attr_type(tb);
177 if (IS_ERR(algt))
178 return;
179 if ((algt->type & CRYPTO_ALG_INTERNAL))
180 *type |= CRYPTO_ALG_INTERNAL;
181 if ((algt->mask & CRYPTO_ALG_INTERNAL))
182 *mask |= CRYPTO_ALG_INTERNAL;
183}
184
Herbert Xu124b53d2007-04-16 20:49:20 +1000185static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
186 const u8 *key, unsigned int keylen)
187{
188 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
189 struct crypto_blkcipher *child = ctx->child;
190 int err;
191
192 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
193 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
194 CRYPTO_TFM_REQ_MASK);
195 err = crypto_blkcipher_setkey(child, key, keylen);
196 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
197 CRYPTO_TFM_RES_MASK);
198 return err;
199}
200
201static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
202 struct crypto_blkcipher *child,
203 int err,
204 int (*crypt)(struct blkcipher_desc *desc,
205 struct scatterlist *dst,
206 struct scatterlist *src,
207 unsigned int len))
208{
209 struct cryptd_blkcipher_request_ctx *rctx;
210 struct blkcipher_desc desc;
211
212 rctx = ablkcipher_request_ctx(req);
213
Herbert Xu93aa7f82008-05-07 21:10:13 +0800214 if (unlikely(err == -EINPROGRESS))
215 goto out;
Herbert Xu124b53d2007-04-16 20:49:20 +1000216
217 desc.tfm = child;
218 desc.info = req->info;
219 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
220
221 err = crypt(&desc, req->dst, req->src, req->nbytes);
222
223 req->base.complete = rctx->complete;
224
Herbert Xu93aa7f82008-05-07 21:10:13 +0800225out:
Herbert Xu124b53d2007-04-16 20:49:20 +1000226 local_bh_disable();
Herbert Xu93aa7f82008-05-07 21:10:13 +0800227 rctx->complete(&req->base, err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000228 local_bh_enable();
229}
230
231static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
232{
233 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
234 struct crypto_blkcipher *child = ctx->child;
235
236 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
237 crypto_blkcipher_crt(child)->encrypt);
238}
239
240static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
241{
242 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
243 struct crypto_blkcipher *child = ctx->child;
244
245 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
246 crypto_blkcipher_crt(child)->decrypt);
247}
248
249static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700250 crypto_completion_t compl)
Herbert Xu124b53d2007-04-16 20:49:20 +1000251{
252 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
253 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800254 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000255
Huang Ying254eff72009-02-19 14:42:19 +0800256 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
Herbert Xu124b53d2007-04-16 20:49:20 +1000257 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700258 req->base.complete = compl;
Herbert Xu124b53d2007-04-16 20:49:20 +1000259
Huang Ying254eff72009-02-19 14:42:19 +0800260 return cryptd_enqueue_request(queue, &req->base);
Herbert Xu124b53d2007-04-16 20:49:20 +1000261}
262
263static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
264{
265 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
266}
267
268static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
269{
270 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
271}
272
273static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
274{
275 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
276 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
277 struct crypto_spawn *spawn = &ictx->spawn;
278 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
279 struct crypto_blkcipher *cipher;
280
281 cipher = crypto_spawn_blkcipher(spawn);
282 if (IS_ERR(cipher))
283 return PTR_ERR(cipher);
284
285 ctx->child = cipher;
286 tfm->crt_ablkcipher.reqsize =
287 sizeof(struct cryptd_blkcipher_request_ctx);
288 return 0;
289}
290
291static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
292{
293 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000294
295 crypto_free_blkcipher(ctx->child);
296}
297
Herbert Xu9b8c4562015-05-21 15:10:57 +0800298static int cryptd_init_instance(struct crypto_instance *inst,
299 struct crypto_alg *alg)
300{
301 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
302 "cryptd(%s)",
303 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
304 return -ENAMETOOLONG;
305
306 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
307
308 inst->alg.cra_priority = alg->cra_priority + 50;
309 inst->alg.cra_blocksize = alg->cra_blocksize;
310 inst->alg.cra_alignmask = alg->cra_alignmask;
311
312 return 0;
313}
314
Herbert Xu0b535ad2009-07-14 19:11:32 +0800315static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
316 unsigned int tail)
Herbert Xu124b53d2007-04-16 20:49:20 +1000317{
Herbert Xu0b535ad2009-07-14 19:11:32 +0800318 char *p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000319 struct crypto_instance *inst;
Herbert Xu124b53d2007-04-16 20:49:20 +1000320 int err;
321
Herbert Xu0b535ad2009-07-14 19:11:32 +0800322 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
323 if (!p)
324 return ERR_PTR(-ENOMEM);
325
326 inst = (void *)(p + head);
Herbert Xu124b53d2007-04-16 20:49:20 +1000327
Herbert Xu9b8c4562015-05-21 15:10:57 +0800328 err = cryptd_init_instance(inst, alg);
329 if (err)
Herbert Xu124b53d2007-04-16 20:49:20 +1000330 goto out_free_inst;
331
Herbert Xu124b53d2007-04-16 20:49:20 +1000332out:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800333 return p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000334
335out_free_inst:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800336 kfree(p);
337 p = ERR_PTR(err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000338 goto out;
339}
340
Herbert Xu9cd899a2009-07-14 18:45:45 +0800341static int cryptd_create_blkcipher(struct crypto_template *tmpl,
342 struct rtattr **tb,
343 struct cryptd_queue *queue)
Herbert Xu124b53d2007-04-16 20:49:20 +1000344{
Herbert Xu46309d82009-07-12 21:38:59 +0800345 struct cryptd_instance_ctx *ctx;
Herbert Xu124b53d2007-04-16 20:49:20 +1000346 struct crypto_instance *inst;
347 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200348 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
349 u32 mask = CRYPTO_ALG_TYPE_MASK;
Herbert Xu46309d82009-07-12 21:38:59 +0800350 int err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000351
Stephan Mueller466a7b92015-03-30 21:57:06 +0200352 cryptd_check_internal(tb, &type, &mask);
353
354 alg = crypto_get_attr_alg(tb, type, mask);
Herbert Xu124b53d2007-04-16 20:49:20 +1000355 if (IS_ERR(alg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800356 return PTR_ERR(alg);
Herbert Xu124b53d2007-04-16 20:49:20 +1000357
Herbert Xu0b535ad2009-07-14 19:11:32 +0800358 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800359 err = PTR_ERR(inst);
Herbert Xu124b53d2007-04-16 20:49:20 +1000360 if (IS_ERR(inst))
361 goto out_put_alg;
362
Herbert Xu46309d82009-07-12 21:38:59 +0800363 ctx = crypto_instance_ctx(inst);
364 ctx->queue = queue;
365
366 err = crypto_init_spawn(&ctx->spawn, alg, inst,
367 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
368 if (err)
369 goto out_free_inst;
370
Stephan Mueller466a7b92015-03-30 21:57:06 +0200371 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
372 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
373 type |= CRYPTO_ALG_INTERNAL;
374 inst->alg.cra_flags = type;
Herbert Xu124b53d2007-04-16 20:49:20 +1000375 inst->alg.cra_type = &crypto_ablkcipher_type;
376
377 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
378 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
379 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
380
Herbert Xu927eead2007-11-27 21:15:31 +0800381 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
382
Herbert Xu124b53d2007-04-16 20:49:20 +1000383 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
384
385 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
386 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
387
388 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
389 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
390 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
391
Herbert Xu9cd899a2009-07-14 18:45:45 +0800392 err = crypto_register_instance(tmpl, inst);
393 if (err) {
394 crypto_drop_spawn(&ctx->spawn);
395out_free_inst:
396 kfree(inst);
397 }
398
Herbert Xu124b53d2007-04-16 20:49:20 +1000399out_put_alg:
400 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800401 return err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000402}
403
Loc Hob8a28252008-05-14 21:23:00 +0800404static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
405{
406 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800407 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
408 struct crypto_shash_spawn *spawn = &ictx->spawn;
Loc Hob8a28252008-05-14 21:23:00 +0800409 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800410 struct crypto_shash *hash;
Loc Hob8a28252008-05-14 21:23:00 +0800411
Herbert Xu46309d82009-07-12 21:38:59 +0800412 hash = crypto_spawn_shash(spawn);
413 if (IS_ERR(hash))
414 return PTR_ERR(hash);
Loc Hob8a28252008-05-14 21:23:00 +0800415
Herbert Xu46309d82009-07-12 21:38:59 +0800416 ctx->child = hash;
Herbert Xu0d6669e2009-07-12 23:06:33 +0800417 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
418 sizeof(struct cryptd_hash_request_ctx) +
419 crypto_shash_descsize(hash));
Loc Hob8a28252008-05-14 21:23:00 +0800420 return 0;
421}
422
423static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
424{
425 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Loc Hob8a28252008-05-14 21:23:00 +0800426
Herbert Xu46309d82009-07-12 21:38:59 +0800427 crypto_free_shash(ctx->child);
Loc Hob8a28252008-05-14 21:23:00 +0800428}
429
430static int cryptd_hash_setkey(struct crypto_ahash *parent,
431 const u8 *key, unsigned int keylen)
432{
433 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
Herbert Xu46309d82009-07-12 21:38:59 +0800434 struct crypto_shash *child = ctx->child;
Loc Hob8a28252008-05-14 21:23:00 +0800435 int err;
436
Herbert Xu46309d82009-07-12 21:38:59 +0800437 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
438 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
439 CRYPTO_TFM_REQ_MASK);
440 err = crypto_shash_setkey(child, key, keylen);
441 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
442 CRYPTO_TFM_RES_MASK);
Loc Hob8a28252008-05-14 21:23:00 +0800443 return err;
444}
445
446static int cryptd_hash_enqueue(struct ahash_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700447 crypto_completion_t compl)
Loc Hob8a28252008-05-14 21:23:00 +0800448{
449 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
450 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800451 struct cryptd_queue *queue =
452 cryptd_get_queue(crypto_ahash_tfm(tfm));
Loc Hob8a28252008-05-14 21:23:00 +0800453
454 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700455 req->base.complete = compl;
Loc Hob8a28252008-05-14 21:23:00 +0800456
Huang Ying254eff72009-02-19 14:42:19 +0800457 return cryptd_enqueue_request(queue, &req->base);
Loc Hob8a28252008-05-14 21:23:00 +0800458}
459
460static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
461{
Herbert Xu46309d82009-07-12 21:38:59 +0800462 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
463 struct crypto_shash *child = ctx->child;
464 struct ahash_request *req = ahash_request_cast(req_async);
465 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
466 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800467
468 if (unlikely(err == -EINPROGRESS))
469 goto out;
470
Herbert Xu46309d82009-07-12 21:38:59 +0800471 desc->tfm = child;
472 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800473
Herbert Xu46309d82009-07-12 21:38:59 +0800474 err = crypto_shash_init(desc);
Loc Hob8a28252008-05-14 21:23:00 +0800475
476 req->base.complete = rctx->complete;
477
478out:
479 local_bh_disable();
480 rctx->complete(&req->base, err);
481 local_bh_enable();
482}
483
484static int cryptd_hash_init_enqueue(struct ahash_request *req)
485{
486 return cryptd_hash_enqueue(req, cryptd_hash_init);
487}
488
489static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
490{
Herbert Xu46309d82009-07-12 21:38:59 +0800491 struct ahash_request *req = ahash_request_cast(req_async);
Loc Hob8a28252008-05-14 21:23:00 +0800492 struct cryptd_hash_request_ctx *rctx;
Loc Hob8a28252008-05-14 21:23:00 +0800493
494 rctx = ahash_request_ctx(req);
495
496 if (unlikely(err == -EINPROGRESS))
497 goto out;
498
Herbert Xu46309d82009-07-12 21:38:59 +0800499 err = shash_ahash_update(req, &rctx->desc);
Loc Hob8a28252008-05-14 21:23:00 +0800500
501 req->base.complete = rctx->complete;
502
503out:
504 local_bh_disable();
505 rctx->complete(&req->base, err);
506 local_bh_enable();
507}
508
509static int cryptd_hash_update_enqueue(struct ahash_request *req)
510{
511 return cryptd_hash_enqueue(req, cryptd_hash_update);
512}
513
514static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
515{
Herbert Xu46309d82009-07-12 21:38:59 +0800516 struct ahash_request *req = ahash_request_cast(req_async);
517 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Loc Hob8a28252008-05-14 21:23:00 +0800518
519 if (unlikely(err == -EINPROGRESS))
520 goto out;
521
Herbert Xu46309d82009-07-12 21:38:59 +0800522 err = crypto_shash_final(&rctx->desc, req->result);
Loc Hob8a28252008-05-14 21:23:00 +0800523
524 req->base.complete = rctx->complete;
525
526out:
527 local_bh_disable();
528 rctx->complete(&req->base, err);
529 local_bh_enable();
530}
531
532static int cryptd_hash_final_enqueue(struct ahash_request *req)
533{
534 return cryptd_hash_enqueue(req, cryptd_hash_final);
535}
536
Herbert Xu6fba00d2009-07-22 11:10:22 +0800537static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
538{
539 struct ahash_request *req = ahash_request_cast(req_async);
540 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
541
542 if (unlikely(err == -EINPROGRESS))
543 goto out;
544
545 err = shash_ahash_finup(req, &rctx->desc);
546
547 req->base.complete = rctx->complete;
548
549out:
550 local_bh_disable();
551 rctx->complete(&req->base, err);
552 local_bh_enable();
553}
554
555static int cryptd_hash_finup_enqueue(struct ahash_request *req)
556{
557 return cryptd_hash_enqueue(req, cryptd_hash_finup);
558}
559
Loc Hob8a28252008-05-14 21:23:00 +0800560static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
561{
Herbert Xu46309d82009-07-12 21:38:59 +0800562 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
563 struct crypto_shash *child = ctx->child;
564 struct ahash_request *req = ahash_request_cast(req_async);
565 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
566 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800567
568 if (unlikely(err == -EINPROGRESS))
569 goto out;
570
Herbert Xu46309d82009-07-12 21:38:59 +0800571 desc->tfm = child;
572 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800573
Herbert Xu46309d82009-07-12 21:38:59 +0800574 err = shash_ahash_digest(req, desc);
Loc Hob8a28252008-05-14 21:23:00 +0800575
576 req->base.complete = rctx->complete;
577
578out:
579 local_bh_disable();
580 rctx->complete(&req->base, err);
581 local_bh_enable();
582}
583
584static int cryptd_hash_digest_enqueue(struct ahash_request *req)
585{
586 return cryptd_hash_enqueue(req, cryptd_hash_digest);
587}
588
Herbert Xu6fba00d2009-07-22 11:10:22 +0800589static int cryptd_hash_export(struct ahash_request *req, void *out)
590{
591 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
592
593 return crypto_shash_export(&rctx->desc, out);
594}
595
596static int cryptd_hash_import(struct ahash_request *req, const void *in)
597{
598 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
599
600 return crypto_shash_import(&rctx->desc, in);
601}
602
Herbert Xu9cd899a2009-07-14 18:45:45 +0800603static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
604 struct cryptd_queue *queue)
Loc Hob8a28252008-05-14 21:23:00 +0800605{
Herbert Xu46309d82009-07-12 21:38:59 +0800606 struct hashd_instance_ctx *ctx;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800607 struct ahash_instance *inst;
Herbert Xu46309d82009-07-12 21:38:59 +0800608 struct shash_alg *salg;
Loc Hob8a28252008-05-14 21:23:00 +0800609 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200610 u32 type = 0;
611 u32 mask = 0;
Herbert Xu46309d82009-07-12 21:38:59 +0800612 int err;
Loc Hob8a28252008-05-14 21:23:00 +0800613
Stephan Mueller466a7b92015-03-30 21:57:06 +0200614 cryptd_check_internal(tb, &type, &mask);
615
616 salg = shash_attr_alg(tb[1], type, mask);
Herbert Xu46309d82009-07-12 21:38:59 +0800617 if (IS_ERR(salg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800618 return PTR_ERR(salg);
Loc Hob8a28252008-05-14 21:23:00 +0800619
Herbert Xu46309d82009-07-12 21:38:59 +0800620 alg = &salg->base;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800621 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
622 sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800623 err = PTR_ERR(inst);
Loc Hob8a28252008-05-14 21:23:00 +0800624 if (IS_ERR(inst))
625 goto out_put_alg;
626
Herbert Xu0b535ad2009-07-14 19:11:32 +0800627 ctx = ahash_instance_ctx(inst);
Herbert Xu46309d82009-07-12 21:38:59 +0800628 ctx->queue = queue;
629
Herbert Xu0b535ad2009-07-14 19:11:32 +0800630 err = crypto_init_shash_spawn(&ctx->spawn, salg,
631 ahash_crypto_instance(inst));
Herbert Xu46309d82009-07-12 21:38:59 +0800632 if (err)
633 goto out_free_inst;
634
Stephan Mueller466a7b92015-03-30 21:57:06 +0200635 type = CRYPTO_ALG_ASYNC;
636 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
637 type |= CRYPTO_ALG_INTERNAL;
638 inst->alg.halg.base.cra_flags = type;
Loc Hob8a28252008-05-14 21:23:00 +0800639
Herbert Xu0b535ad2009-07-14 19:11:32 +0800640 inst->alg.halg.digestsize = salg->digestsize;
641 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
Loc Hob8a28252008-05-14 21:23:00 +0800642
Herbert Xu0b535ad2009-07-14 19:11:32 +0800643 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
644 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
Loc Hob8a28252008-05-14 21:23:00 +0800645
Herbert Xu0b535ad2009-07-14 19:11:32 +0800646 inst->alg.init = cryptd_hash_init_enqueue;
647 inst->alg.update = cryptd_hash_update_enqueue;
648 inst->alg.final = cryptd_hash_final_enqueue;
Herbert Xu6fba00d2009-07-22 11:10:22 +0800649 inst->alg.finup = cryptd_hash_finup_enqueue;
650 inst->alg.export = cryptd_hash_export;
651 inst->alg.import = cryptd_hash_import;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800652 inst->alg.setkey = cryptd_hash_setkey;
653 inst->alg.digest = cryptd_hash_digest_enqueue;
Loc Hob8a28252008-05-14 21:23:00 +0800654
Herbert Xu0b535ad2009-07-14 19:11:32 +0800655 err = ahash_register_instance(tmpl, inst);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800656 if (err) {
657 crypto_drop_shash(&ctx->spawn);
658out_free_inst:
659 kfree(inst);
660 }
661
Loc Hob8a28252008-05-14 21:23:00 +0800662out_put_alg:
663 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800664 return err;
Loc Hob8a28252008-05-14 21:23:00 +0800665}
666
Herbert Xu92b98762015-05-28 22:08:01 +0800667static int cryptd_aead_setkey(struct crypto_aead *parent,
668 const u8 *key, unsigned int keylen)
669{
670 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
671 struct crypto_aead *child = ctx->child;
672
673 return crypto_aead_setkey(child, key, keylen);
674}
675
676static int cryptd_aead_setauthsize(struct crypto_aead *parent,
677 unsigned int authsize)
678{
679 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
680 struct crypto_aead *child = ctx->child;
681
682 return crypto_aead_setauthsize(child, authsize);
683}
684
Adrian Hoban298c9262010-09-20 16:05:12 +0800685static void cryptd_aead_crypt(struct aead_request *req,
686 struct crypto_aead *child,
687 int err,
688 int (*crypt)(struct aead_request *req))
689{
690 struct cryptd_aead_request_ctx *rctx;
691 rctx = aead_request_ctx(req);
692
693 if (unlikely(err == -EINPROGRESS))
694 goto out;
695 aead_request_set_tfm(req, child);
696 err = crypt( req );
697 req->base.complete = rctx->complete;
698out:
699 local_bh_disable();
700 rctx->complete(&req->base, err);
701 local_bh_enable();
702}
703
704static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
705{
706 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
707 struct crypto_aead *child = ctx->child;
708 struct aead_request *req;
709
710 req = container_of(areq, struct aead_request, base);
711 cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->encrypt);
712}
713
714static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
715{
716 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
717 struct crypto_aead *child = ctx->child;
718 struct aead_request *req;
719
720 req = container_of(areq, struct aead_request, base);
721 cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->decrypt);
722}
723
724static int cryptd_aead_enqueue(struct aead_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700725 crypto_completion_t compl)
Adrian Hoban298c9262010-09-20 16:05:12 +0800726{
727 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
728 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
729 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
730
731 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700732 req->base.complete = compl;
Adrian Hoban298c9262010-09-20 16:05:12 +0800733 return cryptd_enqueue_request(queue, &req->base);
734}
735
736static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
737{
738 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
739}
740
741static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
742{
743 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
744}
745
Herbert Xuf614e542015-05-28 22:08:04 +0800746static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800747{
Herbert Xuf614e542015-05-28 22:08:04 +0800748 struct aead_instance *inst = aead_alg_instance(tfm);
749 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800750 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
Herbert Xuf614e542015-05-28 22:08:04 +0800751 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800752 struct crypto_aead *cipher;
753
754 cipher = crypto_spawn_aead(spawn);
755 if (IS_ERR(cipher))
756 return PTR_ERR(cipher);
757
Adrian Hoban298c9262010-09-20 16:05:12 +0800758 ctx->child = cipher;
Herbert Xuf614e542015-05-28 22:08:04 +0800759 crypto_aead_set_reqsize(tfm, sizeof(struct cryptd_aead_request_ctx));
Adrian Hoban298c9262010-09-20 16:05:12 +0800760 return 0;
761}
762
Herbert Xuf614e542015-05-28 22:08:04 +0800763static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800764{
Herbert Xuf614e542015-05-28 22:08:04 +0800765 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800766 crypto_free_aead(ctx->child);
767}
768
769static int cryptd_create_aead(struct crypto_template *tmpl,
770 struct rtattr **tb,
771 struct cryptd_queue *queue)
772{
773 struct aead_instance_ctx *ctx;
Herbert Xuf614e542015-05-28 22:08:04 +0800774 struct aead_instance *inst;
775 struct aead_alg *alg;
Herbert Xu9b8c4562015-05-21 15:10:57 +0800776 const char *name;
777 u32 type = 0;
778 u32 mask = 0;
Adrian Hoban298c9262010-09-20 16:05:12 +0800779 int err;
780
Stephan Mueller466a7b92015-03-30 21:57:06 +0200781 cryptd_check_internal(tb, &type, &mask);
782
Herbert Xu9b8c4562015-05-21 15:10:57 +0800783 name = crypto_attr_alg_name(tb[1]);
784 if (IS_ERR(name))
785 return PTR_ERR(name);
Adrian Hoban298c9262010-09-20 16:05:12 +0800786
Herbert Xu9b8c4562015-05-21 15:10:57 +0800787 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
788 if (!inst)
789 return -ENOMEM;
Adrian Hoban298c9262010-09-20 16:05:12 +0800790
Herbert Xuf614e542015-05-28 22:08:04 +0800791 ctx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800792 ctx->queue = queue;
793
Herbert Xuf614e542015-05-28 22:08:04 +0800794 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
Herbert Xu9b8c4562015-05-21 15:10:57 +0800795 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
Adrian Hoban298c9262010-09-20 16:05:12 +0800796 if (err)
797 goto out_free_inst;
798
Herbert Xuf614e542015-05-28 22:08:04 +0800799 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
800 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
Herbert Xu9b8c4562015-05-21 15:10:57 +0800801 if (err)
802 goto out_drop_aead;
803
Herbert Xuf614e542015-05-28 22:08:04 +0800804 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
805 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
806 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
Adrian Hoban298c9262010-09-20 16:05:12 +0800807
Herbert Xuf614e542015-05-28 22:08:04 +0800808 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
809 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
810
811 inst->alg.init = cryptd_aead_init_tfm;
812 inst->alg.exit = cryptd_aead_exit_tfm;
813 inst->alg.setkey = cryptd_aead_setkey;
814 inst->alg.setauthsize = cryptd_aead_setauthsize;
815 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
816 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
817
818 err = aead_register_instance(tmpl, inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800819 if (err) {
Herbert Xu9b8c4562015-05-21 15:10:57 +0800820out_drop_aead:
821 crypto_drop_aead(&ctx->aead_spawn);
Adrian Hoban298c9262010-09-20 16:05:12 +0800822out_free_inst:
823 kfree(inst);
824 }
Adrian Hoban298c9262010-09-20 16:05:12 +0800825 return err;
826}
827
Huang Ying254eff72009-02-19 14:42:19 +0800828static struct cryptd_queue queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000829
Herbert Xu9cd899a2009-07-14 18:45:45 +0800830static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
Herbert Xu124b53d2007-04-16 20:49:20 +1000831{
832 struct crypto_attr_type *algt;
833
834 algt = crypto_get_attr_type(tb);
835 if (IS_ERR(algt))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800836 return PTR_ERR(algt);
Herbert Xu124b53d2007-04-16 20:49:20 +1000837
838 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
839 case CRYPTO_ALG_TYPE_BLKCIPHER:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800840 return cryptd_create_blkcipher(tmpl, tb, &queue);
Loc Hob8a28252008-05-14 21:23:00 +0800841 case CRYPTO_ALG_TYPE_DIGEST:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800842 return cryptd_create_hash(tmpl, tb, &queue);
Adrian Hoban298c9262010-09-20 16:05:12 +0800843 case CRYPTO_ALG_TYPE_AEAD:
844 return cryptd_create_aead(tmpl, tb, &queue);
Herbert Xu124b53d2007-04-16 20:49:20 +1000845 }
846
Herbert Xu9cd899a2009-07-14 18:45:45 +0800847 return -EINVAL;
Herbert Xu124b53d2007-04-16 20:49:20 +1000848}
849
850static void cryptd_free(struct crypto_instance *inst)
851{
852 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800853 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800854 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800855
856 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
857 case CRYPTO_ALG_TYPE_AHASH:
858 crypto_drop_shash(&hctx->spawn);
859 kfree(ahash_instance(inst));
860 return;
Adrian Hoban298c9262010-09-20 16:05:12 +0800861 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuf614e542015-05-28 22:08:04 +0800862 crypto_drop_aead(&aead_ctx->aead_spawn);
863 kfree(aead_instance(inst));
Adrian Hoban298c9262010-09-20 16:05:12 +0800864 return;
865 default:
866 crypto_drop_spawn(&ctx->spawn);
867 kfree(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800868 }
Herbert Xu124b53d2007-04-16 20:49:20 +1000869}
870
871static struct crypto_template cryptd_tmpl = {
872 .name = "cryptd",
Herbert Xu9cd899a2009-07-14 18:45:45 +0800873 .create = cryptd_create,
Herbert Xu124b53d2007-04-16 20:49:20 +1000874 .free = cryptd_free,
875 .module = THIS_MODULE,
876};
877
Huang Ying1cac2cb2009-01-18 16:19:46 +1100878struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
879 u32 type, u32 mask)
880{
881 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Huang Ying505fd212009-03-29 15:33:53 +0800882 struct crypto_tfm *tfm;
Huang Ying1cac2cb2009-01-18 16:19:46 +1100883
884 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
885 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
886 return ERR_PTR(-EINVAL);
Huang Ying505fd212009-03-29 15:33:53 +0800887 type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
888 type |= CRYPTO_ALG_TYPE_BLKCIPHER;
889 mask &= ~CRYPTO_ALG_TYPE_MASK;
890 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
891 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100892 if (IS_ERR(tfm))
893 return ERR_CAST(tfm);
Huang Ying505fd212009-03-29 15:33:53 +0800894 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
895 crypto_free_tfm(tfm);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100896 return ERR_PTR(-EINVAL);
897 }
898
Huang Ying505fd212009-03-29 15:33:53 +0800899 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
Huang Ying1cac2cb2009-01-18 16:19:46 +1100900}
901EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
902
903struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
904{
905 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
906 return ctx->child;
907}
908EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
909
910void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
911{
912 crypto_free_ablkcipher(&tfm->base);
913}
914EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
915
Huang Yingace13662009-08-06 15:35:20 +1000916struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
917 u32 type, u32 mask)
918{
919 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
920 struct crypto_ahash *tfm;
921
922 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
923 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
924 return ERR_PTR(-EINVAL);
925 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
926 if (IS_ERR(tfm))
927 return ERR_CAST(tfm);
928 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
929 crypto_free_ahash(tfm);
930 return ERR_PTR(-EINVAL);
931 }
932
933 return __cryptd_ahash_cast(tfm);
934}
935EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
936
937struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
938{
939 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
940
941 return ctx->child;
942}
943EXPORT_SYMBOL_GPL(cryptd_ahash_child);
944
Huang Ying0e1227d2009-10-19 11:53:06 +0900945struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
946{
947 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
948 return &rctx->desc;
949}
950EXPORT_SYMBOL_GPL(cryptd_shash_desc);
951
Huang Yingace13662009-08-06 15:35:20 +1000952void cryptd_free_ahash(struct cryptd_ahash *tfm)
953{
954 crypto_free_ahash(&tfm->base);
955}
956EXPORT_SYMBOL_GPL(cryptd_free_ahash);
957
Adrian Hoban298c9262010-09-20 16:05:12 +0800958struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
959 u32 type, u32 mask)
960{
961 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
962 struct crypto_aead *tfm;
963
964 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
965 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
966 return ERR_PTR(-EINVAL);
967 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
968 if (IS_ERR(tfm))
969 return ERR_CAST(tfm);
970 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
971 crypto_free_aead(tfm);
972 return ERR_PTR(-EINVAL);
973 }
974 return __cryptd_aead_cast(tfm);
975}
976EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
977
978struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
979{
980 struct cryptd_aead_ctx *ctx;
981 ctx = crypto_aead_ctx(&tfm->base);
982 return ctx->child;
983}
984EXPORT_SYMBOL_GPL(cryptd_aead_child);
985
986void cryptd_free_aead(struct cryptd_aead *tfm)
987{
988 crypto_free_aead(&tfm->base);
989}
990EXPORT_SYMBOL_GPL(cryptd_free_aead);
991
Herbert Xu124b53d2007-04-16 20:49:20 +1000992static int __init cryptd_init(void)
993{
994 int err;
995
Huang Ying254eff72009-02-19 14:42:19 +0800996 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
Herbert Xu124b53d2007-04-16 20:49:20 +1000997 if (err)
998 return err;
999
1000 err = crypto_register_template(&cryptd_tmpl);
1001 if (err)
Huang Ying254eff72009-02-19 14:42:19 +08001002 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001003
1004 return err;
1005}
1006
1007static void __exit cryptd_exit(void)
1008{
Huang Ying254eff72009-02-19 14:42:19 +08001009 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001010 crypto_unregister_template(&cryptd_tmpl);
1011}
1012
Herbert Xub2bac6a2011-08-19 16:11:23 +08001013subsys_initcall(cryptd_init);
Herbert Xu124b53d2007-04-16 20:49:20 +10001014module_exit(cryptd_exit);
1015
1016MODULE_LICENSE("GPL");
1017MODULE_DESCRIPTION("Software async crypto daemon");
Kees Cook4943ba12014-11-24 16:32:38 -08001018MODULE_ALIAS_CRYPTO("cryptd");