blob: 9b66ce7e728d9f21dbf3ae4a93be51c7670d2297 [file] [log] [blame]
Herbert Xu124b53d2007-04-16 20:49:20 +10001/*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
Adrian Hoban298c9262010-09-20 16:05:12 +08006 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
Herbert Xu124b53d2007-04-16 20:49:20 +100013 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20#include <crypto/algapi.h>
Herbert Xu18e33e62008-07-10 16:01:22 +080021#include <crypto/internal/hash.h>
Adrian Hoban298c9262010-09-20 16:05:12 +080022#include <crypto/internal/aead.h>
Huang Ying1cac2cb2009-01-18 16:19:46 +110023#include <crypto/cryptd.h>
Huang Ying254eff72009-02-19 14:42:19 +080024#include <crypto/crypto_wq.h>
Herbert Xu81760ea2016-06-21 16:55:13 +080025#include <linux/atomic.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100026#include <linux/err.h>
27#include <linux/init.h>
28#include <linux/kernel.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100029#include <linux/list.h>
30#include <linux/module.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100031#include <linux/scatterlist.h>
32#include <linux/sched.h>
33#include <linux/slab.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100034
Herbert Xu81760ea2016-06-21 16:55:13 +080035#define CRYPTD_MAX_CPU_QLEN 1000
Herbert Xu124b53d2007-04-16 20:49:20 +100036
Huang Ying254eff72009-02-19 14:42:19 +080037struct cryptd_cpu_queue {
Herbert Xu124b53d2007-04-16 20:49:20 +100038 struct crypto_queue queue;
Huang Ying254eff72009-02-19 14:42:19 +080039 struct work_struct work;
40};
41
42struct cryptd_queue {
Tejun Heoa29d8b82010-02-02 14:39:15 +090043 struct cryptd_cpu_queue __percpu *cpu_queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100044};
45
46struct cryptd_instance_ctx {
47 struct crypto_spawn spawn;
Huang Ying254eff72009-02-19 14:42:19 +080048 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100049};
50
Herbert Xu46309d82009-07-12 21:38:59 +080051struct hashd_instance_ctx {
52 struct crypto_shash_spawn spawn;
53 struct cryptd_queue *queue;
54};
55
Adrian Hoban298c9262010-09-20 16:05:12 +080056struct aead_instance_ctx {
57 struct crypto_aead_spawn aead_spawn;
58 struct cryptd_queue *queue;
59};
60
Herbert Xu124b53d2007-04-16 20:49:20 +100061struct cryptd_blkcipher_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080062 atomic_t refcnt;
Herbert Xu124b53d2007-04-16 20:49:20 +100063 struct crypto_blkcipher *child;
64};
65
66struct cryptd_blkcipher_request_ctx {
67 crypto_completion_t complete;
68};
69
Loc Hob8a28252008-05-14 21:23:00 +080070struct cryptd_hash_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080071 atomic_t refcnt;
Herbert Xu46309d82009-07-12 21:38:59 +080072 struct crypto_shash *child;
Loc Hob8a28252008-05-14 21:23:00 +080073};
74
75struct cryptd_hash_request_ctx {
76 crypto_completion_t complete;
Herbert Xu46309d82009-07-12 21:38:59 +080077 struct shash_desc desc;
Loc Hob8a28252008-05-14 21:23:00 +080078};
Herbert Xu124b53d2007-04-16 20:49:20 +100079
Adrian Hoban298c9262010-09-20 16:05:12 +080080struct cryptd_aead_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080081 atomic_t refcnt;
Adrian Hoban298c9262010-09-20 16:05:12 +080082 struct crypto_aead *child;
83};
84
85struct cryptd_aead_request_ctx {
86 crypto_completion_t complete;
87};
88
Huang Ying254eff72009-02-19 14:42:19 +080089static void cryptd_queue_worker(struct work_struct *work);
90
91static int cryptd_init_queue(struct cryptd_queue *queue,
92 unsigned int max_cpu_qlen)
93{
94 int cpu;
95 struct cryptd_cpu_queue *cpu_queue;
96
97 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
98 if (!queue->cpu_queue)
99 return -ENOMEM;
100 for_each_possible_cpu(cpu) {
101 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
102 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
103 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
104 }
105 return 0;
106}
107
108static void cryptd_fini_queue(struct cryptd_queue *queue)
109{
110 int cpu;
111 struct cryptd_cpu_queue *cpu_queue;
112
113 for_each_possible_cpu(cpu) {
114 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
115 BUG_ON(cpu_queue->queue.qlen);
116 }
117 free_percpu(queue->cpu_queue);
118}
119
120static int cryptd_enqueue_request(struct cryptd_queue *queue,
121 struct crypto_async_request *request)
122{
123 int cpu, err;
124 struct cryptd_cpu_queue *cpu_queue;
Herbert Xu81760ea2016-06-21 16:55:13 +0800125 struct crypto_tfm *tfm;
126 atomic_t *refcnt;
127 bool may_backlog;
Huang Ying254eff72009-02-19 14:42:19 +0800128
129 cpu = get_cpu();
Christoph Lameter0b44f482009-10-03 19:48:23 +0900130 cpu_queue = this_cpu_ptr(queue->cpu_queue);
Huang Ying254eff72009-02-19 14:42:19 +0800131 err = crypto_enqueue_request(&cpu_queue->queue, request);
Herbert Xu81760ea2016-06-21 16:55:13 +0800132
133 refcnt = crypto_tfm_ctx(request->tfm);
134 may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
135
136 if (err == -EBUSY && !may_backlog)
137 goto out_put_cpu;
138
Huang Ying254eff72009-02-19 14:42:19 +0800139 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
Herbert Xu81760ea2016-06-21 16:55:13 +0800140
141 if (!atomic_read(refcnt))
142 goto out_put_cpu;
143
144 tfm = request->tfm;
145 atomic_inc(refcnt);
146
147out_put_cpu:
Huang Ying254eff72009-02-19 14:42:19 +0800148 put_cpu();
149
150 return err;
151}
152
153/* Called in workqueue context, do one real cryption work (via
154 * req->complete) and reschedule itself if there are more work to
155 * do. */
156static void cryptd_queue_worker(struct work_struct *work)
157{
158 struct cryptd_cpu_queue *cpu_queue;
159 struct crypto_async_request *req, *backlog;
160
161 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300162 /*
163 * Only handle one request at a time to avoid hogging crypto workqueue.
164 * preempt_disable/enable is used to prevent being preempted by
165 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
166 * cryptd_enqueue_request() being accessed from software interrupts.
167 */
168 local_bh_disable();
Huang Ying254eff72009-02-19 14:42:19 +0800169 preempt_disable();
170 backlog = crypto_get_backlog(&cpu_queue->queue);
171 req = crypto_dequeue_request(&cpu_queue->queue);
172 preempt_enable();
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300173 local_bh_enable();
Huang Ying254eff72009-02-19 14:42:19 +0800174
175 if (!req)
176 return;
177
178 if (backlog)
179 backlog->complete(backlog, -EINPROGRESS);
180 req->complete(req, 0);
181
182 if (cpu_queue->queue.qlen)
183 queue_work(kcrypto_wq, &cpu_queue->work);
184}
185
186static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
Herbert Xu124b53d2007-04-16 20:49:20 +1000187{
188 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
189 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
Huang Ying254eff72009-02-19 14:42:19 +0800190 return ictx->queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000191}
192
Stephan Mueller466a7b92015-03-30 21:57:06 +0200193static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
194 u32 *mask)
195{
196 struct crypto_attr_type *algt;
197
198 algt = crypto_get_attr_type(tb);
199 if (IS_ERR(algt))
200 return;
Herbert Xuf6da3202015-07-09 07:17:19 +0800201
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800202 *type |= algt->type & CRYPTO_ALG_INTERNAL;
203 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200204}
205
Herbert Xu124b53d2007-04-16 20:49:20 +1000206static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
207 const u8 *key, unsigned int keylen)
208{
209 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
210 struct crypto_blkcipher *child = ctx->child;
211 int err;
212
213 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
214 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
215 CRYPTO_TFM_REQ_MASK);
216 err = crypto_blkcipher_setkey(child, key, keylen);
217 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
218 CRYPTO_TFM_RES_MASK);
219 return err;
220}
221
222static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
223 struct crypto_blkcipher *child,
224 int err,
225 int (*crypt)(struct blkcipher_desc *desc,
226 struct scatterlist *dst,
227 struct scatterlist *src,
228 unsigned int len))
229{
230 struct cryptd_blkcipher_request_ctx *rctx;
Herbert Xu81760ea2016-06-21 16:55:13 +0800231 struct cryptd_blkcipher_ctx *ctx;
232 struct crypto_ablkcipher *tfm;
Herbert Xu124b53d2007-04-16 20:49:20 +1000233 struct blkcipher_desc desc;
Herbert Xu81760ea2016-06-21 16:55:13 +0800234 int refcnt;
Herbert Xu124b53d2007-04-16 20:49:20 +1000235
236 rctx = ablkcipher_request_ctx(req);
237
Herbert Xu93aa7f82008-05-07 21:10:13 +0800238 if (unlikely(err == -EINPROGRESS))
239 goto out;
Herbert Xu124b53d2007-04-16 20:49:20 +1000240
241 desc.tfm = child;
242 desc.info = req->info;
243 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
244
245 err = crypt(&desc, req->dst, req->src, req->nbytes);
246
247 req->base.complete = rctx->complete;
248
Herbert Xu93aa7f82008-05-07 21:10:13 +0800249out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800250 tfm = crypto_ablkcipher_reqtfm(req);
251 ctx = crypto_ablkcipher_ctx(tfm);
252 refcnt = atomic_read(&ctx->refcnt);
253
Herbert Xu124b53d2007-04-16 20:49:20 +1000254 local_bh_disable();
Herbert Xu93aa7f82008-05-07 21:10:13 +0800255 rctx->complete(&req->base, err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000256 local_bh_enable();
Herbert Xu81760ea2016-06-21 16:55:13 +0800257
258 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
259 crypto_free_ablkcipher(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000260}
261
262static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
263{
264 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
265 struct crypto_blkcipher *child = ctx->child;
266
267 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
268 crypto_blkcipher_crt(child)->encrypt);
269}
270
271static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
272{
273 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
274 struct crypto_blkcipher *child = ctx->child;
275
276 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
277 crypto_blkcipher_crt(child)->decrypt);
278}
279
280static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700281 crypto_completion_t compl)
Herbert Xu124b53d2007-04-16 20:49:20 +1000282{
283 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
284 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800285 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000286
Huang Ying254eff72009-02-19 14:42:19 +0800287 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
Herbert Xu124b53d2007-04-16 20:49:20 +1000288 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700289 req->base.complete = compl;
Herbert Xu124b53d2007-04-16 20:49:20 +1000290
Huang Ying254eff72009-02-19 14:42:19 +0800291 return cryptd_enqueue_request(queue, &req->base);
Herbert Xu124b53d2007-04-16 20:49:20 +1000292}
293
294static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
295{
296 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
297}
298
299static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
300{
301 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
302}
303
304static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
305{
306 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
307 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
308 struct crypto_spawn *spawn = &ictx->spawn;
309 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
310 struct crypto_blkcipher *cipher;
311
312 cipher = crypto_spawn_blkcipher(spawn);
313 if (IS_ERR(cipher))
314 return PTR_ERR(cipher);
315
316 ctx->child = cipher;
317 tfm->crt_ablkcipher.reqsize =
318 sizeof(struct cryptd_blkcipher_request_ctx);
319 return 0;
320}
321
322static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
323{
324 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000325
326 crypto_free_blkcipher(ctx->child);
327}
328
Herbert Xu9b8c4562015-05-21 15:10:57 +0800329static int cryptd_init_instance(struct crypto_instance *inst,
330 struct crypto_alg *alg)
331{
332 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
333 "cryptd(%s)",
334 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
335 return -ENAMETOOLONG;
336
337 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
338
339 inst->alg.cra_priority = alg->cra_priority + 50;
340 inst->alg.cra_blocksize = alg->cra_blocksize;
341 inst->alg.cra_alignmask = alg->cra_alignmask;
342
343 return 0;
344}
345
Herbert Xu0b535ad2009-07-14 19:11:32 +0800346static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
347 unsigned int tail)
Herbert Xu124b53d2007-04-16 20:49:20 +1000348{
Herbert Xu0b535ad2009-07-14 19:11:32 +0800349 char *p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000350 struct crypto_instance *inst;
Herbert Xu124b53d2007-04-16 20:49:20 +1000351 int err;
352
Herbert Xu0b535ad2009-07-14 19:11:32 +0800353 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
354 if (!p)
355 return ERR_PTR(-ENOMEM);
356
357 inst = (void *)(p + head);
Herbert Xu124b53d2007-04-16 20:49:20 +1000358
Herbert Xu9b8c4562015-05-21 15:10:57 +0800359 err = cryptd_init_instance(inst, alg);
360 if (err)
Herbert Xu124b53d2007-04-16 20:49:20 +1000361 goto out_free_inst;
362
Herbert Xu124b53d2007-04-16 20:49:20 +1000363out:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800364 return p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000365
366out_free_inst:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800367 kfree(p);
368 p = ERR_PTR(err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000369 goto out;
370}
371
Herbert Xu9cd899a2009-07-14 18:45:45 +0800372static int cryptd_create_blkcipher(struct crypto_template *tmpl,
373 struct rtattr **tb,
374 struct cryptd_queue *queue)
Herbert Xu124b53d2007-04-16 20:49:20 +1000375{
Herbert Xu46309d82009-07-12 21:38:59 +0800376 struct cryptd_instance_ctx *ctx;
Herbert Xu124b53d2007-04-16 20:49:20 +1000377 struct crypto_instance *inst;
378 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200379 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
380 u32 mask = CRYPTO_ALG_TYPE_MASK;
Herbert Xu46309d82009-07-12 21:38:59 +0800381 int err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000382
Stephan Mueller466a7b92015-03-30 21:57:06 +0200383 cryptd_check_internal(tb, &type, &mask);
384
385 alg = crypto_get_attr_alg(tb, type, mask);
Herbert Xu124b53d2007-04-16 20:49:20 +1000386 if (IS_ERR(alg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800387 return PTR_ERR(alg);
Herbert Xu124b53d2007-04-16 20:49:20 +1000388
Herbert Xu0b535ad2009-07-14 19:11:32 +0800389 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800390 err = PTR_ERR(inst);
Herbert Xu124b53d2007-04-16 20:49:20 +1000391 if (IS_ERR(inst))
392 goto out_put_alg;
393
Herbert Xu46309d82009-07-12 21:38:59 +0800394 ctx = crypto_instance_ctx(inst);
395 ctx->queue = queue;
396
397 err = crypto_init_spawn(&ctx->spawn, alg, inst,
398 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
399 if (err)
400 goto out_free_inst;
401
Stephan Mueller466a7b92015-03-30 21:57:06 +0200402 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
403 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
404 type |= CRYPTO_ALG_INTERNAL;
405 inst->alg.cra_flags = type;
Herbert Xu124b53d2007-04-16 20:49:20 +1000406 inst->alg.cra_type = &crypto_ablkcipher_type;
407
408 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
409 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
410 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
411
Herbert Xu927eead2007-11-27 21:15:31 +0800412 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
413
Herbert Xu124b53d2007-04-16 20:49:20 +1000414 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
415
416 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
417 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
418
419 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
420 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
421 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
422
Herbert Xu9cd899a2009-07-14 18:45:45 +0800423 err = crypto_register_instance(tmpl, inst);
424 if (err) {
425 crypto_drop_spawn(&ctx->spawn);
426out_free_inst:
427 kfree(inst);
428 }
429
Herbert Xu124b53d2007-04-16 20:49:20 +1000430out_put_alg:
431 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800432 return err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000433}
434
Loc Hob8a28252008-05-14 21:23:00 +0800435static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
436{
437 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800438 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
439 struct crypto_shash_spawn *spawn = &ictx->spawn;
Loc Hob8a28252008-05-14 21:23:00 +0800440 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800441 struct crypto_shash *hash;
Loc Hob8a28252008-05-14 21:23:00 +0800442
Herbert Xu46309d82009-07-12 21:38:59 +0800443 hash = crypto_spawn_shash(spawn);
444 if (IS_ERR(hash))
445 return PTR_ERR(hash);
Loc Hob8a28252008-05-14 21:23:00 +0800446
Herbert Xu46309d82009-07-12 21:38:59 +0800447 ctx->child = hash;
Herbert Xu0d6669e2009-07-12 23:06:33 +0800448 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
449 sizeof(struct cryptd_hash_request_ctx) +
450 crypto_shash_descsize(hash));
Loc Hob8a28252008-05-14 21:23:00 +0800451 return 0;
452}
453
454static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
455{
456 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Loc Hob8a28252008-05-14 21:23:00 +0800457
Herbert Xu46309d82009-07-12 21:38:59 +0800458 crypto_free_shash(ctx->child);
Loc Hob8a28252008-05-14 21:23:00 +0800459}
460
461static int cryptd_hash_setkey(struct crypto_ahash *parent,
462 const u8 *key, unsigned int keylen)
463{
464 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
Herbert Xu46309d82009-07-12 21:38:59 +0800465 struct crypto_shash *child = ctx->child;
Loc Hob8a28252008-05-14 21:23:00 +0800466 int err;
467
Herbert Xu46309d82009-07-12 21:38:59 +0800468 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
469 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
470 CRYPTO_TFM_REQ_MASK);
471 err = crypto_shash_setkey(child, key, keylen);
472 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
473 CRYPTO_TFM_RES_MASK);
Loc Hob8a28252008-05-14 21:23:00 +0800474 return err;
475}
476
477static int cryptd_hash_enqueue(struct ahash_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700478 crypto_completion_t compl)
Loc Hob8a28252008-05-14 21:23:00 +0800479{
480 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
481 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800482 struct cryptd_queue *queue =
483 cryptd_get_queue(crypto_ahash_tfm(tfm));
Loc Hob8a28252008-05-14 21:23:00 +0800484
485 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700486 req->base.complete = compl;
Loc Hob8a28252008-05-14 21:23:00 +0800487
Huang Ying254eff72009-02-19 14:42:19 +0800488 return cryptd_enqueue_request(queue, &req->base);
Loc Hob8a28252008-05-14 21:23:00 +0800489}
490
Herbert Xu81760ea2016-06-21 16:55:13 +0800491static void cryptd_hash_complete(struct ahash_request *req, int err)
492{
493 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
494 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
495 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
496 int refcnt = atomic_read(&ctx->refcnt);
497
498 local_bh_disable();
499 rctx->complete(&req->base, err);
500 local_bh_enable();
501
502 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
503 crypto_free_ahash(tfm);
504}
505
Loc Hob8a28252008-05-14 21:23:00 +0800506static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
507{
Herbert Xu46309d82009-07-12 21:38:59 +0800508 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
509 struct crypto_shash *child = ctx->child;
510 struct ahash_request *req = ahash_request_cast(req_async);
511 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
512 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800513
514 if (unlikely(err == -EINPROGRESS))
515 goto out;
516
Herbert Xu46309d82009-07-12 21:38:59 +0800517 desc->tfm = child;
518 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800519
Herbert Xu46309d82009-07-12 21:38:59 +0800520 err = crypto_shash_init(desc);
Loc Hob8a28252008-05-14 21:23:00 +0800521
522 req->base.complete = rctx->complete;
523
524out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800525 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800526}
527
528static int cryptd_hash_init_enqueue(struct ahash_request *req)
529{
530 return cryptd_hash_enqueue(req, cryptd_hash_init);
531}
532
533static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
534{
Herbert Xu46309d82009-07-12 21:38:59 +0800535 struct ahash_request *req = ahash_request_cast(req_async);
Loc Hob8a28252008-05-14 21:23:00 +0800536 struct cryptd_hash_request_ctx *rctx;
Loc Hob8a28252008-05-14 21:23:00 +0800537
538 rctx = ahash_request_ctx(req);
539
540 if (unlikely(err == -EINPROGRESS))
541 goto out;
542
Herbert Xu46309d82009-07-12 21:38:59 +0800543 err = shash_ahash_update(req, &rctx->desc);
Loc Hob8a28252008-05-14 21:23:00 +0800544
545 req->base.complete = rctx->complete;
546
547out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800548 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800549}
550
551static int cryptd_hash_update_enqueue(struct ahash_request *req)
552{
553 return cryptd_hash_enqueue(req, cryptd_hash_update);
554}
555
556static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
557{
Herbert Xu46309d82009-07-12 21:38:59 +0800558 struct ahash_request *req = ahash_request_cast(req_async);
559 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Loc Hob8a28252008-05-14 21:23:00 +0800560
561 if (unlikely(err == -EINPROGRESS))
562 goto out;
563
Herbert Xu46309d82009-07-12 21:38:59 +0800564 err = crypto_shash_final(&rctx->desc, req->result);
Loc Hob8a28252008-05-14 21:23:00 +0800565
566 req->base.complete = rctx->complete;
567
568out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800569 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800570}
571
572static int cryptd_hash_final_enqueue(struct ahash_request *req)
573{
574 return cryptd_hash_enqueue(req, cryptd_hash_final);
575}
576
Herbert Xu6fba00d2009-07-22 11:10:22 +0800577static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
578{
579 struct ahash_request *req = ahash_request_cast(req_async);
580 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
581
582 if (unlikely(err == -EINPROGRESS))
583 goto out;
584
585 err = shash_ahash_finup(req, &rctx->desc);
586
587 req->base.complete = rctx->complete;
588
589out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800590 cryptd_hash_complete(req, err);
Herbert Xu6fba00d2009-07-22 11:10:22 +0800591}
592
593static int cryptd_hash_finup_enqueue(struct ahash_request *req)
594{
595 return cryptd_hash_enqueue(req, cryptd_hash_finup);
596}
597
Loc Hob8a28252008-05-14 21:23:00 +0800598static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
599{
Herbert Xu46309d82009-07-12 21:38:59 +0800600 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
601 struct crypto_shash *child = ctx->child;
602 struct ahash_request *req = ahash_request_cast(req_async);
603 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
604 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800605
606 if (unlikely(err == -EINPROGRESS))
607 goto out;
608
Herbert Xu46309d82009-07-12 21:38:59 +0800609 desc->tfm = child;
610 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800611
Herbert Xu46309d82009-07-12 21:38:59 +0800612 err = shash_ahash_digest(req, desc);
Loc Hob8a28252008-05-14 21:23:00 +0800613
614 req->base.complete = rctx->complete;
615
616out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800617 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800618}
619
620static int cryptd_hash_digest_enqueue(struct ahash_request *req)
621{
622 return cryptd_hash_enqueue(req, cryptd_hash_digest);
623}
624
Herbert Xu6fba00d2009-07-22 11:10:22 +0800625static int cryptd_hash_export(struct ahash_request *req, void *out)
626{
627 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
628
629 return crypto_shash_export(&rctx->desc, out);
630}
631
632static int cryptd_hash_import(struct ahash_request *req, const void *in)
633{
Ard Biesheuvel0bd22232016-09-01 14:25:43 +0100634 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
635 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
636 struct shash_desc *desc = cryptd_shash_desc(req);
Herbert Xu6fba00d2009-07-22 11:10:22 +0800637
Ard Biesheuvel0bd22232016-09-01 14:25:43 +0100638 desc->tfm = ctx->child;
639 desc->flags = req->base.flags;
640
641 return crypto_shash_import(desc, in);
Herbert Xu6fba00d2009-07-22 11:10:22 +0800642}
643
Herbert Xu9cd899a2009-07-14 18:45:45 +0800644static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
645 struct cryptd_queue *queue)
Loc Hob8a28252008-05-14 21:23:00 +0800646{
Herbert Xu46309d82009-07-12 21:38:59 +0800647 struct hashd_instance_ctx *ctx;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800648 struct ahash_instance *inst;
Herbert Xu46309d82009-07-12 21:38:59 +0800649 struct shash_alg *salg;
Loc Hob8a28252008-05-14 21:23:00 +0800650 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200651 u32 type = 0;
652 u32 mask = 0;
Herbert Xu46309d82009-07-12 21:38:59 +0800653 int err;
Loc Hob8a28252008-05-14 21:23:00 +0800654
Stephan Mueller466a7b92015-03-30 21:57:06 +0200655 cryptd_check_internal(tb, &type, &mask);
656
657 salg = shash_attr_alg(tb[1], type, mask);
Herbert Xu46309d82009-07-12 21:38:59 +0800658 if (IS_ERR(salg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800659 return PTR_ERR(salg);
Loc Hob8a28252008-05-14 21:23:00 +0800660
Herbert Xu46309d82009-07-12 21:38:59 +0800661 alg = &salg->base;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800662 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
663 sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800664 err = PTR_ERR(inst);
Loc Hob8a28252008-05-14 21:23:00 +0800665 if (IS_ERR(inst))
666 goto out_put_alg;
667
Herbert Xu0b535ad2009-07-14 19:11:32 +0800668 ctx = ahash_instance_ctx(inst);
Herbert Xu46309d82009-07-12 21:38:59 +0800669 ctx->queue = queue;
670
Herbert Xu0b535ad2009-07-14 19:11:32 +0800671 err = crypto_init_shash_spawn(&ctx->spawn, salg,
672 ahash_crypto_instance(inst));
Herbert Xu46309d82009-07-12 21:38:59 +0800673 if (err)
674 goto out_free_inst;
675
Eric Biggersb392a532018-01-03 11:16:26 -0800676 inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
677 (alg->cra_flags & (CRYPTO_ALG_INTERNAL |
678 CRYPTO_ALG_OPTIONAL_KEY));
Loc Hob8a28252008-05-14 21:23:00 +0800679
Herbert Xu0b535ad2009-07-14 19:11:32 +0800680 inst->alg.halg.digestsize = salg->digestsize;
Wang, Rui Y1a078342015-11-29 22:45:34 +0800681 inst->alg.halg.statesize = salg->statesize;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800682 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
Loc Hob8a28252008-05-14 21:23:00 +0800683
Herbert Xu0b535ad2009-07-14 19:11:32 +0800684 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
685 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
Loc Hob8a28252008-05-14 21:23:00 +0800686
Herbert Xu0b535ad2009-07-14 19:11:32 +0800687 inst->alg.init = cryptd_hash_init_enqueue;
688 inst->alg.update = cryptd_hash_update_enqueue;
689 inst->alg.final = cryptd_hash_final_enqueue;
Herbert Xu6fba00d2009-07-22 11:10:22 +0800690 inst->alg.finup = cryptd_hash_finup_enqueue;
691 inst->alg.export = cryptd_hash_export;
692 inst->alg.import = cryptd_hash_import;
Eric Biggersc1ebf9f2018-01-03 11:16:23 -0800693 if (crypto_shash_alg_has_setkey(salg))
694 inst->alg.setkey = cryptd_hash_setkey;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800695 inst->alg.digest = cryptd_hash_digest_enqueue;
Loc Hob8a28252008-05-14 21:23:00 +0800696
Herbert Xu0b535ad2009-07-14 19:11:32 +0800697 err = ahash_register_instance(tmpl, inst);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800698 if (err) {
699 crypto_drop_shash(&ctx->spawn);
700out_free_inst:
701 kfree(inst);
702 }
703
Loc Hob8a28252008-05-14 21:23:00 +0800704out_put_alg:
705 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800706 return err;
Loc Hob8a28252008-05-14 21:23:00 +0800707}
708
Herbert Xu92b98762015-05-28 22:08:01 +0800709static int cryptd_aead_setkey(struct crypto_aead *parent,
710 const u8 *key, unsigned int keylen)
711{
712 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
713 struct crypto_aead *child = ctx->child;
714
715 return crypto_aead_setkey(child, key, keylen);
716}
717
718static int cryptd_aead_setauthsize(struct crypto_aead *parent,
719 unsigned int authsize)
720{
721 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
722 struct crypto_aead *child = ctx->child;
723
724 return crypto_aead_setauthsize(child, authsize);
725}
726
Adrian Hoban298c9262010-09-20 16:05:12 +0800727static void cryptd_aead_crypt(struct aead_request *req,
728 struct crypto_aead *child,
729 int err,
730 int (*crypt)(struct aead_request *req))
731{
732 struct cryptd_aead_request_ctx *rctx;
Herbert Xu81760ea2016-06-21 16:55:13 +0800733 struct cryptd_aead_ctx *ctx;
Herbert Xuec9f2002015-07-06 19:11:03 +0800734 crypto_completion_t compl;
Herbert Xu81760ea2016-06-21 16:55:13 +0800735 struct crypto_aead *tfm;
736 int refcnt;
Herbert Xuec9f2002015-07-06 19:11:03 +0800737
Adrian Hoban298c9262010-09-20 16:05:12 +0800738 rctx = aead_request_ctx(req);
Herbert Xuec9f2002015-07-06 19:11:03 +0800739 compl = rctx->complete;
Adrian Hoban298c9262010-09-20 16:05:12 +0800740
Herbert Xu31bd44e2016-08-25 16:49:51 +0800741 tfm = crypto_aead_reqtfm(req);
742
Adrian Hoban298c9262010-09-20 16:05:12 +0800743 if (unlikely(err == -EINPROGRESS))
744 goto out;
745 aead_request_set_tfm(req, child);
746 err = crypt( req );
Herbert Xu81760ea2016-06-21 16:55:13 +0800747
Adrian Hoban298c9262010-09-20 16:05:12 +0800748out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800749 ctx = crypto_aead_ctx(tfm);
750 refcnt = atomic_read(&ctx->refcnt);
751
Adrian Hoban298c9262010-09-20 16:05:12 +0800752 local_bh_disable();
Herbert Xuec9f2002015-07-06 19:11:03 +0800753 compl(&req->base, err);
Adrian Hoban298c9262010-09-20 16:05:12 +0800754 local_bh_enable();
Herbert Xu81760ea2016-06-21 16:55:13 +0800755
756 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
757 crypto_free_aead(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800758}
759
760static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
761{
762 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
763 struct crypto_aead *child = ctx->child;
764 struct aead_request *req;
765
766 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800767 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800768}
769
770static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
771{
772 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
773 struct crypto_aead *child = ctx->child;
774 struct aead_request *req;
775
776 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800777 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800778}
779
780static int cryptd_aead_enqueue(struct aead_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700781 crypto_completion_t compl)
Adrian Hoban298c9262010-09-20 16:05:12 +0800782{
783 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
784 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
785 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
786
787 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700788 req->base.complete = compl;
Adrian Hoban298c9262010-09-20 16:05:12 +0800789 return cryptd_enqueue_request(queue, &req->base);
790}
791
792static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
793{
794 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
795}
796
797static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
798{
799 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
800}
801
Herbert Xuf614e542015-05-28 22:08:04 +0800802static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800803{
Herbert Xuf614e542015-05-28 22:08:04 +0800804 struct aead_instance *inst = aead_alg_instance(tfm);
805 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800806 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
Herbert Xuf614e542015-05-28 22:08:04 +0800807 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800808 struct crypto_aead *cipher;
809
810 cipher = crypto_spawn_aead(spawn);
811 if (IS_ERR(cipher))
812 return PTR_ERR(cipher);
813
Adrian Hoban298c9262010-09-20 16:05:12 +0800814 ctx->child = cipher;
Herbert Xuec9f2002015-07-06 19:11:03 +0800815 crypto_aead_set_reqsize(
816 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
817 crypto_aead_reqsize(cipher)));
Adrian Hoban298c9262010-09-20 16:05:12 +0800818 return 0;
819}
820
Herbert Xuf614e542015-05-28 22:08:04 +0800821static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800822{
Herbert Xuf614e542015-05-28 22:08:04 +0800823 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800824 crypto_free_aead(ctx->child);
825}
826
827static int cryptd_create_aead(struct crypto_template *tmpl,
828 struct rtattr **tb,
829 struct cryptd_queue *queue)
830{
831 struct aead_instance_ctx *ctx;
Herbert Xuf614e542015-05-28 22:08:04 +0800832 struct aead_instance *inst;
833 struct aead_alg *alg;
Herbert Xu9b8c4562015-05-21 15:10:57 +0800834 const char *name;
835 u32 type = 0;
Herbert Xuec9f2002015-07-06 19:11:03 +0800836 u32 mask = CRYPTO_ALG_ASYNC;
Adrian Hoban298c9262010-09-20 16:05:12 +0800837 int err;
838
Stephan Mueller466a7b92015-03-30 21:57:06 +0200839 cryptd_check_internal(tb, &type, &mask);
840
Herbert Xu9b8c4562015-05-21 15:10:57 +0800841 name = crypto_attr_alg_name(tb[1]);
842 if (IS_ERR(name))
843 return PTR_ERR(name);
Adrian Hoban298c9262010-09-20 16:05:12 +0800844
Herbert Xu9b8c4562015-05-21 15:10:57 +0800845 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
846 if (!inst)
847 return -ENOMEM;
Adrian Hoban298c9262010-09-20 16:05:12 +0800848
Herbert Xuf614e542015-05-28 22:08:04 +0800849 ctx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800850 ctx->queue = queue;
851
Herbert Xuf614e542015-05-28 22:08:04 +0800852 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
Herbert Xu9b8c4562015-05-21 15:10:57 +0800853 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
Adrian Hoban298c9262010-09-20 16:05:12 +0800854 if (err)
855 goto out_free_inst;
856
Herbert Xuf614e542015-05-28 22:08:04 +0800857 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
858 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
Herbert Xu9b8c4562015-05-21 15:10:57 +0800859 if (err)
860 goto out_drop_aead;
861
Herbert Xuf614e542015-05-28 22:08:04 +0800862 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800863 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
Herbert Xuf614e542015-05-28 22:08:04 +0800864 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
Adrian Hoban298c9262010-09-20 16:05:12 +0800865
Herbert Xuf614e542015-05-28 22:08:04 +0800866 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
867 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
868
869 inst->alg.init = cryptd_aead_init_tfm;
870 inst->alg.exit = cryptd_aead_exit_tfm;
871 inst->alg.setkey = cryptd_aead_setkey;
872 inst->alg.setauthsize = cryptd_aead_setauthsize;
873 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
874 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
875
876 err = aead_register_instance(tmpl, inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800877 if (err) {
Herbert Xu9b8c4562015-05-21 15:10:57 +0800878out_drop_aead:
879 crypto_drop_aead(&ctx->aead_spawn);
Adrian Hoban298c9262010-09-20 16:05:12 +0800880out_free_inst:
881 kfree(inst);
882 }
Adrian Hoban298c9262010-09-20 16:05:12 +0800883 return err;
884}
885
Huang Ying254eff72009-02-19 14:42:19 +0800886static struct cryptd_queue queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000887
Herbert Xu9cd899a2009-07-14 18:45:45 +0800888static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
Herbert Xu124b53d2007-04-16 20:49:20 +1000889{
890 struct crypto_attr_type *algt;
891
892 algt = crypto_get_attr_type(tb);
893 if (IS_ERR(algt))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800894 return PTR_ERR(algt);
Herbert Xu124b53d2007-04-16 20:49:20 +1000895
896 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
897 case CRYPTO_ALG_TYPE_BLKCIPHER:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800898 return cryptd_create_blkcipher(tmpl, tb, &queue);
Loc Hob8a28252008-05-14 21:23:00 +0800899 case CRYPTO_ALG_TYPE_DIGEST:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800900 return cryptd_create_hash(tmpl, tb, &queue);
Adrian Hoban298c9262010-09-20 16:05:12 +0800901 case CRYPTO_ALG_TYPE_AEAD:
902 return cryptd_create_aead(tmpl, tb, &queue);
Herbert Xu124b53d2007-04-16 20:49:20 +1000903 }
904
Herbert Xu9cd899a2009-07-14 18:45:45 +0800905 return -EINVAL;
Herbert Xu124b53d2007-04-16 20:49:20 +1000906}
907
908static void cryptd_free(struct crypto_instance *inst)
909{
910 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800911 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800912 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800913
914 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
915 case CRYPTO_ALG_TYPE_AHASH:
916 crypto_drop_shash(&hctx->spawn);
917 kfree(ahash_instance(inst));
918 return;
Adrian Hoban298c9262010-09-20 16:05:12 +0800919 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuf614e542015-05-28 22:08:04 +0800920 crypto_drop_aead(&aead_ctx->aead_spawn);
921 kfree(aead_instance(inst));
Adrian Hoban298c9262010-09-20 16:05:12 +0800922 return;
923 default:
924 crypto_drop_spawn(&ctx->spawn);
925 kfree(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800926 }
Herbert Xu124b53d2007-04-16 20:49:20 +1000927}
928
929static struct crypto_template cryptd_tmpl = {
930 .name = "cryptd",
Herbert Xu9cd899a2009-07-14 18:45:45 +0800931 .create = cryptd_create,
Herbert Xu124b53d2007-04-16 20:49:20 +1000932 .free = cryptd_free,
933 .module = THIS_MODULE,
934};
935
Huang Ying1cac2cb2009-01-18 16:19:46 +1100936struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
937 u32 type, u32 mask)
938{
939 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +0800940 struct cryptd_blkcipher_ctx *ctx;
Huang Ying505fd212009-03-29 15:33:53 +0800941 struct crypto_tfm *tfm;
Huang Ying1cac2cb2009-01-18 16:19:46 +1100942
943 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
944 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
945 return ERR_PTR(-EINVAL);
Alexander Kuleshovc012a792015-11-25 23:48:28 +0600946 type = crypto_skcipher_type(type);
Huang Ying505fd212009-03-29 15:33:53 +0800947 mask &= ~CRYPTO_ALG_TYPE_MASK;
948 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
949 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100950 if (IS_ERR(tfm))
951 return ERR_CAST(tfm);
Huang Ying505fd212009-03-29 15:33:53 +0800952 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
953 crypto_free_tfm(tfm);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100954 return ERR_PTR(-EINVAL);
955 }
956
Herbert Xu81760ea2016-06-21 16:55:13 +0800957 ctx = crypto_tfm_ctx(tfm);
958 atomic_set(&ctx->refcnt, 1);
959
Huang Ying505fd212009-03-29 15:33:53 +0800960 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
Huang Ying1cac2cb2009-01-18 16:19:46 +1100961}
962EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
963
964struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
965{
966 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
967 return ctx->child;
968}
969EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
970
Herbert Xu81760ea2016-06-21 16:55:13 +0800971bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
972{
973 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
974
975 return atomic_read(&ctx->refcnt) - 1;
976}
977EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
978
Huang Ying1cac2cb2009-01-18 16:19:46 +1100979void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
980{
Herbert Xu81760ea2016-06-21 16:55:13 +0800981 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
982
983 if (atomic_dec_and_test(&ctx->refcnt))
984 crypto_free_ablkcipher(&tfm->base);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100985}
986EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
987
Huang Yingace13662009-08-06 15:35:20 +1000988struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
989 u32 type, u32 mask)
990{
991 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +0800992 struct cryptd_hash_ctx *ctx;
Huang Yingace13662009-08-06 15:35:20 +1000993 struct crypto_ahash *tfm;
994
995 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
996 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
997 return ERR_PTR(-EINVAL);
998 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
999 if (IS_ERR(tfm))
1000 return ERR_CAST(tfm);
1001 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1002 crypto_free_ahash(tfm);
1003 return ERR_PTR(-EINVAL);
1004 }
1005
Herbert Xu81760ea2016-06-21 16:55:13 +08001006 ctx = crypto_ahash_ctx(tfm);
1007 atomic_set(&ctx->refcnt, 1);
1008
Huang Yingace13662009-08-06 15:35:20 +10001009 return __cryptd_ahash_cast(tfm);
1010}
1011EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1012
1013struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1014{
1015 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1016
1017 return ctx->child;
1018}
1019EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1020
Huang Ying0e1227d2009-10-19 11:53:06 +09001021struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1022{
1023 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1024 return &rctx->desc;
1025}
1026EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1027
Herbert Xu81760ea2016-06-21 16:55:13 +08001028bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1029{
1030 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1031
1032 return atomic_read(&ctx->refcnt) - 1;
1033}
1034EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1035
Huang Yingace13662009-08-06 15:35:20 +10001036void cryptd_free_ahash(struct cryptd_ahash *tfm)
1037{
Herbert Xu81760ea2016-06-21 16:55:13 +08001038 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1039
1040 if (atomic_dec_and_test(&ctx->refcnt))
1041 crypto_free_ahash(&tfm->base);
Huang Yingace13662009-08-06 15:35:20 +10001042}
1043EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1044
Adrian Hoban298c9262010-09-20 16:05:12 +08001045struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1046 u32 type, u32 mask)
1047{
1048 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +08001049 struct cryptd_aead_ctx *ctx;
Adrian Hoban298c9262010-09-20 16:05:12 +08001050 struct crypto_aead *tfm;
1051
1052 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1053 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1054 return ERR_PTR(-EINVAL);
1055 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1056 if (IS_ERR(tfm))
1057 return ERR_CAST(tfm);
1058 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1059 crypto_free_aead(tfm);
1060 return ERR_PTR(-EINVAL);
1061 }
Herbert Xu81760ea2016-06-21 16:55:13 +08001062
1063 ctx = crypto_aead_ctx(tfm);
1064 atomic_set(&ctx->refcnt, 1);
1065
Adrian Hoban298c9262010-09-20 16:05:12 +08001066 return __cryptd_aead_cast(tfm);
1067}
1068EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1069
1070struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1071{
1072 struct cryptd_aead_ctx *ctx;
1073 ctx = crypto_aead_ctx(&tfm->base);
1074 return ctx->child;
1075}
1076EXPORT_SYMBOL_GPL(cryptd_aead_child);
1077
Herbert Xu81760ea2016-06-21 16:55:13 +08001078bool cryptd_aead_queued(struct cryptd_aead *tfm)
1079{
1080 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1081
1082 return atomic_read(&ctx->refcnt) - 1;
1083}
1084EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1085
Adrian Hoban298c9262010-09-20 16:05:12 +08001086void cryptd_free_aead(struct cryptd_aead *tfm)
1087{
Herbert Xu81760ea2016-06-21 16:55:13 +08001088 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1089
1090 if (atomic_dec_and_test(&ctx->refcnt))
1091 crypto_free_aead(&tfm->base);
Adrian Hoban298c9262010-09-20 16:05:12 +08001092}
1093EXPORT_SYMBOL_GPL(cryptd_free_aead);
1094
Herbert Xu124b53d2007-04-16 20:49:20 +10001095static int __init cryptd_init(void)
1096{
1097 int err;
1098
Huang Ying254eff72009-02-19 14:42:19 +08001099 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
Herbert Xu124b53d2007-04-16 20:49:20 +10001100 if (err)
1101 return err;
1102
1103 err = crypto_register_template(&cryptd_tmpl);
1104 if (err)
Huang Ying254eff72009-02-19 14:42:19 +08001105 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001106
1107 return err;
1108}
1109
1110static void __exit cryptd_exit(void)
1111{
Huang Ying254eff72009-02-19 14:42:19 +08001112 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001113 crypto_unregister_template(&cryptd_tmpl);
1114}
1115
Herbert Xub2bac6a2011-08-19 16:11:23 +08001116subsys_initcall(cryptd_init);
Herbert Xu124b53d2007-04-16 20:49:20 +10001117module_exit(cryptd_exit);
1118
1119MODULE_LICENSE("GPL");
1120MODULE_DESCRIPTION("Software async crypto daemon");
Kees Cook4943ba12014-11-24 16:32:38 -08001121MODULE_ALIAS_CRYPTO("cryptd");