blob: cf8037a87b2d124259df01378df3ac1b43973336 [file] [log] [blame]
Herbert Xu124b53d2007-04-16 20:49:20 +10001/*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
Adrian Hoban298c9262010-09-20 16:05:12 +08006 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
Herbert Xu124b53d2007-04-16 20:49:20 +100013 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20#include <crypto/algapi.h>
Herbert Xu18e33e62008-07-10 16:01:22 +080021#include <crypto/internal/hash.h>
Adrian Hoban298c9262010-09-20 16:05:12 +080022#include <crypto/internal/aead.h>
Huang Ying1cac2cb2009-01-18 16:19:46 +110023#include <crypto/cryptd.h>
Huang Ying254eff72009-02-19 14:42:19 +080024#include <crypto/crypto_wq.h>
Herbert Xu81760ea2016-06-21 16:55:13 +080025#include <linux/atomic.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100026#include <linux/err.h>
27#include <linux/init.h>
28#include <linux/kernel.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100029#include <linux/list.h>
30#include <linux/module.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100031#include <linux/scatterlist.h>
32#include <linux/sched.h>
33#include <linux/slab.h>
Herbert Xu124b53d2007-04-16 20:49:20 +100034
Herbert Xu81760ea2016-06-21 16:55:13 +080035#define CRYPTD_MAX_CPU_QLEN 1000
Herbert Xu124b53d2007-04-16 20:49:20 +100036
Huang Ying254eff72009-02-19 14:42:19 +080037struct cryptd_cpu_queue {
Herbert Xu124b53d2007-04-16 20:49:20 +100038 struct crypto_queue queue;
Huang Ying254eff72009-02-19 14:42:19 +080039 struct work_struct work;
40};
41
42struct cryptd_queue {
Tejun Heoa29d8b82010-02-02 14:39:15 +090043 struct cryptd_cpu_queue __percpu *cpu_queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100044};
45
46struct cryptd_instance_ctx {
47 struct crypto_spawn spawn;
Huang Ying254eff72009-02-19 14:42:19 +080048 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +100049};
50
Herbert Xu46309d82009-07-12 21:38:59 +080051struct hashd_instance_ctx {
52 struct crypto_shash_spawn spawn;
53 struct cryptd_queue *queue;
54};
55
Adrian Hoban298c9262010-09-20 16:05:12 +080056struct aead_instance_ctx {
57 struct crypto_aead_spawn aead_spawn;
58 struct cryptd_queue *queue;
59};
60
Herbert Xu124b53d2007-04-16 20:49:20 +100061struct cryptd_blkcipher_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080062 atomic_t refcnt;
Herbert Xu124b53d2007-04-16 20:49:20 +100063 struct crypto_blkcipher *child;
64};
65
66struct cryptd_blkcipher_request_ctx {
67 crypto_completion_t complete;
68};
69
Loc Hob8a28252008-05-14 21:23:00 +080070struct cryptd_hash_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080071 atomic_t refcnt;
Herbert Xu46309d82009-07-12 21:38:59 +080072 struct crypto_shash *child;
Loc Hob8a28252008-05-14 21:23:00 +080073};
74
75struct cryptd_hash_request_ctx {
76 crypto_completion_t complete;
Herbert Xu46309d82009-07-12 21:38:59 +080077 struct shash_desc desc;
Loc Hob8a28252008-05-14 21:23:00 +080078};
Herbert Xu124b53d2007-04-16 20:49:20 +100079
Adrian Hoban298c9262010-09-20 16:05:12 +080080struct cryptd_aead_ctx {
Herbert Xu81760ea2016-06-21 16:55:13 +080081 atomic_t refcnt;
Adrian Hoban298c9262010-09-20 16:05:12 +080082 struct crypto_aead *child;
83};
84
85struct cryptd_aead_request_ctx {
86 crypto_completion_t complete;
87};
88
Huang Ying254eff72009-02-19 14:42:19 +080089static void cryptd_queue_worker(struct work_struct *work);
90
91static int cryptd_init_queue(struct cryptd_queue *queue,
92 unsigned int max_cpu_qlen)
93{
94 int cpu;
95 struct cryptd_cpu_queue *cpu_queue;
96
97 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
98 if (!queue->cpu_queue)
99 return -ENOMEM;
100 for_each_possible_cpu(cpu) {
101 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
102 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
103 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
104 }
105 return 0;
106}
107
108static void cryptd_fini_queue(struct cryptd_queue *queue)
109{
110 int cpu;
111 struct cryptd_cpu_queue *cpu_queue;
112
113 for_each_possible_cpu(cpu) {
114 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
115 BUG_ON(cpu_queue->queue.qlen);
116 }
117 free_percpu(queue->cpu_queue);
118}
119
120static int cryptd_enqueue_request(struct cryptd_queue *queue,
121 struct crypto_async_request *request)
122{
123 int cpu, err;
124 struct cryptd_cpu_queue *cpu_queue;
Herbert Xu81760ea2016-06-21 16:55:13 +0800125 struct crypto_tfm *tfm;
126 atomic_t *refcnt;
127 bool may_backlog;
Huang Ying254eff72009-02-19 14:42:19 +0800128
129 cpu = get_cpu();
Christoph Lameter0b44f482009-10-03 19:48:23 +0900130 cpu_queue = this_cpu_ptr(queue->cpu_queue);
Huang Ying254eff72009-02-19 14:42:19 +0800131 err = crypto_enqueue_request(&cpu_queue->queue, request);
Herbert Xu81760ea2016-06-21 16:55:13 +0800132
133 refcnt = crypto_tfm_ctx(request->tfm);
134 may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
135
136 if (err == -EBUSY && !may_backlog)
137 goto out_put_cpu;
138
Huang Ying254eff72009-02-19 14:42:19 +0800139 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
Herbert Xu81760ea2016-06-21 16:55:13 +0800140
141 if (!atomic_read(refcnt))
142 goto out_put_cpu;
143
144 tfm = request->tfm;
145 atomic_inc(refcnt);
146
147out_put_cpu:
Huang Ying254eff72009-02-19 14:42:19 +0800148 put_cpu();
149
150 return err;
151}
152
153/* Called in workqueue context, do one real cryption work (via
154 * req->complete) and reschedule itself if there are more work to
155 * do. */
156static void cryptd_queue_worker(struct work_struct *work)
157{
158 struct cryptd_cpu_queue *cpu_queue;
159 struct crypto_async_request *req, *backlog;
160
161 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300162 /*
163 * Only handle one request at a time to avoid hogging crypto workqueue.
164 * preempt_disable/enable is used to prevent being preempted by
165 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
166 * cryptd_enqueue_request() being accessed from software interrupts.
167 */
168 local_bh_disable();
Huang Ying254eff72009-02-19 14:42:19 +0800169 preempt_disable();
170 backlog = crypto_get_backlog(&cpu_queue->queue);
171 req = crypto_dequeue_request(&cpu_queue->queue);
172 preempt_enable();
Jussi Kivilinna9efade12012-10-21 20:42:28 +0300173 local_bh_enable();
Huang Ying254eff72009-02-19 14:42:19 +0800174
175 if (!req)
176 return;
177
178 if (backlog)
179 backlog->complete(backlog, -EINPROGRESS);
180 req->complete(req, 0);
181
182 if (cpu_queue->queue.qlen)
183 queue_work(kcrypto_wq, &cpu_queue->work);
184}
185
186static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
Herbert Xu124b53d2007-04-16 20:49:20 +1000187{
188 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
189 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
Huang Ying254eff72009-02-19 14:42:19 +0800190 return ictx->queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000191}
192
Stephan Mueller466a7b92015-03-30 21:57:06 +0200193static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
194 u32 *mask)
195{
196 struct crypto_attr_type *algt;
197
198 algt = crypto_get_attr_type(tb);
199 if (IS_ERR(algt))
200 return;
Herbert Xuf6da3202015-07-09 07:17:19 +0800201
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800202 *type |= algt->type & CRYPTO_ALG_INTERNAL;
203 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200204}
205
Herbert Xu124b53d2007-04-16 20:49:20 +1000206static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
207 const u8 *key, unsigned int keylen)
208{
209 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
210 struct crypto_blkcipher *child = ctx->child;
211 int err;
212
213 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
214 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
215 CRYPTO_TFM_REQ_MASK);
216 err = crypto_blkcipher_setkey(child, key, keylen);
217 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
218 CRYPTO_TFM_RES_MASK);
219 return err;
220}
221
222static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
223 struct crypto_blkcipher *child,
224 int err,
225 int (*crypt)(struct blkcipher_desc *desc,
226 struct scatterlist *dst,
227 struct scatterlist *src,
228 unsigned int len))
229{
230 struct cryptd_blkcipher_request_ctx *rctx;
Herbert Xu81760ea2016-06-21 16:55:13 +0800231 struct cryptd_blkcipher_ctx *ctx;
232 struct crypto_ablkcipher *tfm;
Herbert Xu124b53d2007-04-16 20:49:20 +1000233 struct blkcipher_desc desc;
Herbert Xu81760ea2016-06-21 16:55:13 +0800234 int refcnt;
Herbert Xu124b53d2007-04-16 20:49:20 +1000235
236 rctx = ablkcipher_request_ctx(req);
237
Herbert Xu93aa7f82008-05-07 21:10:13 +0800238 if (unlikely(err == -EINPROGRESS))
239 goto out;
Herbert Xu124b53d2007-04-16 20:49:20 +1000240
241 desc.tfm = child;
242 desc.info = req->info;
243 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
244
245 err = crypt(&desc, req->dst, req->src, req->nbytes);
246
247 req->base.complete = rctx->complete;
248
Herbert Xu93aa7f82008-05-07 21:10:13 +0800249out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800250 tfm = crypto_ablkcipher_reqtfm(req);
251 ctx = crypto_ablkcipher_ctx(tfm);
252 refcnt = atomic_read(&ctx->refcnt);
253
Herbert Xu124b53d2007-04-16 20:49:20 +1000254 local_bh_disable();
Herbert Xu93aa7f82008-05-07 21:10:13 +0800255 rctx->complete(&req->base, err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000256 local_bh_enable();
Herbert Xu81760ea2016-06-21 16:55:13 +0800257
258 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
259 crypto_free_ablkcipher(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000260}
261
262static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
263{
264 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
265 struct crypto_blkcipher *child = ctx->child;
266
267 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
268 crypto_blkcipher_crt(child)->encrypt);
269}
270
271static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
272{
273 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
274 struct crypto_blkcipher *child = ctx->child;
275
276 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
277 crypto_blkcipher_crt(child)->decrypt);
278}
279
280static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700281 crypto_completion_t compl)
Herbert Xu124b53d2007-04-16 20:49:20 +1000282{
283 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
284 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800285 struct cryptd_queue *queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000286
Huang Ying254eff72009-02-19 14:42:19 +0800287 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
Herbert Xu124b53d2007-04-16 20:49:20 +1000288 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700289 req->base.complete = compl;
Herbert Xu124b53d2007-04-16 20:49:20 +1000290
Huang Ying254eff72009-02-19 14:42:19 +0800291 return cryptd_enqueue_request(queue, &req->base);
Herbert Xu124b53d2007-04-16 20:49:20 +1000292}
293
294static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
295{
296 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
297}
298
299static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
300{
301 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
302}
303
304static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
305{
306 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
307 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
308 struct crypto_spawn *spawn = &ictx->spawn;
309 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
310 struct crypto_blkcipher *cipher;
311
312 cipher = crypto_spawn_blkcipher(spawn);
313 if (IS_ERR(cipher))
314 return PTR_ERR(cipher);
315
316 ctx->child = cipher;
317 tfm->crt_ablkcipher.reqsize =
318 sizeof(struct cryptd_blkcipher_request_ctx);
319 return 0;
320}
321
322static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
323{
324 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu124b53d2007-04-16 20:49:20 +1000325
326 crypto_free_blkcipher(ctx->child);
327}
328
Herbert Xu9b8c4562015-05-21 15:10:57 +0800329static int cryptd_init_instance(struct crypto_instance *inst,
330 struct crypto_alg *alg)
331{
332 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
333 "cryptd(%s)",
334 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
335 return -ENAMETOOLONG;
336
337 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
338
339 inst->alg.cra_priority = alg->cra_priority + 50;
340 inst->alg.cra_blocksize = alg->cra_blocksize;
341 inst->alg.cra_alignmask = alg->cra_alignmask;
342
343 return 0;
344}
345
Herbert Xu0b535ad2009-07-14 19:11:32 +0800346static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
347 unsigned int tail)
Herbert Xu124b53d2007-04-16 20:49:20 +1000348{
Herbert Xu0b535ad2009-07-14 19:11:32 +0800349 char *p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000350 struct crypto_instance *inst;
Herbert Xu124b53d2007-04-16 20:49:20 +1000351 int err;
352
Herbert Xu0b535ad2009-07-14 19:11:32 +0800353 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
354 if (!p)
355 return ERR_PTR(-ENOMEM);
356
357 inst = (void *)(p + head);
Herbert Xu124b53d2007-04-16 20:49:20 +1000358
Herbert Xu9b8c4562015-05-21 15:10:57 +0800359 err = cryptd_init_instance(inst, alg);
360 if (err)
Herbert Xu124b53d2007-04-16 20:49:20 +1000361 goto out_free_inst;
362
Herbert Xu124b53d2007-04-16 20:49:20 +1000363out:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800364 return p;
Herbert Xu124b53d2007-04-16 20:49:20 +1000365
366out_free_inst:
Herbert Xu0b535ad2009-07-14 19:11:32 +0800367 kfree(p);
368 p = ERR_PTR(err);
Herbert Xu124b53d2007-04-16 20:49:20 +1000369 goto out;
370}
371
Herbert Xu9cd899a2009-07-14 18:45:45 +0800372static int cryptd_create_blkcipher(struct crypto_template *tmpl,
373 struct rtattr **tb,
374 struct cryptd_queue *queue)
Herbert Xu124b53d2007-04-16 20:49:20 +1000375{
Herbert Xu46309d82009-07-12 21:38:59 +0800376 struct cryptd_instance_ctx *ctx;
Herbert Xu124b53d2007-04-16 20:49:20 +1000377 struct crypto_instance *inst;
378 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200379 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
380 u32 mask = CRYPTO_ALG_TYPE_MASK;
Herbert Xu46309d82009-07-12 21:38:59 +0800381 int err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000382
Stephan Mueller466a7b92015-03-30 21:57:06 +0200383 cryptd_check_internal(tb, &type, &mask);
384
385 alg = crypto_get_attr_alg(tb, type, mask);
Herbert Xu124b53d2007-04-16 20:49:20 +1000386 if (IS_ERR(alg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800387 return PTR_ERR(alg);
Herbert Xu124b53d2007-04-16 20:49:20 +1000388
Herbert Xu0b535ad2009-07-14 19:11:32 +0800389 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800390 err = PTR_ERR(inst);
Herbert Xu124b53d2007-04-16 20:49:20 +1000391 if (IS_ERR(inst))
392 goto out_put_alg;
393
Herbert Xu46309d82009-07-12 21:38:59 +0800394 ctx = crypto_instance_ctx(inst);
395 ctx->queue = queue;
396
397 err = crypto_init_spawn(&ctx->spawn, alg, inst,
398 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
399 if (err)
400 goto out_free_inst;
401
Stephan Mueller466a7b92015-03-30 21:57:06 +0200402 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
403 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
404 type |= CRYPTO_ALG_INTERNAL;
405 inst->alg.cra_flags = type;
Herbert Xu124b53d2007-04-16 20:49:20 +1000406 inst->alg.cra_type = &crypto_ablkcipher_type;
407
408 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
409 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
410 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
411
Herbert Xu927eead2007-11-27 21:15:31 +0800412 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
413
Herbert Xu124b53d2007-04-16 20:49:20 +1000414 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
415
416 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
417 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
418
419 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
420 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
421 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
422
Herbert Xu9cd899a2009-07-14 18:45:45 +0800423 err = crypto_register_instance(tmpl, inst);
424 if (err) {
425 crypto_drop_spawn(&ctx->spawn);
426out_free_inst:
427 kfree(inst);
428 }
429
Herbert Xu124b53d2007-04-16 20:49:20 +1000430out_put_alg:
431 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800432 return err;
Herbert Xu124b53d2007-04-16 20:49:20 +1000433}
434
Loc Hob8a28252008-05-14 21:23:00 +0800435static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
436{
437 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800438 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
439 struct crypto_shash_spawn *spawn = &ictx->spawn;
Loc Hob8a28252008-05-14 21:23:00 +0800440 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Herbert Xu46309d82009-07-12 21:38:59 +0800441 struct crypto_shash *hash;
Loc Hob8a28252008-05-14 21:23:00 +0800442
Herbert Xu46309d82009-07-12 21:38:59 +0800443 hash = crypto_spawn_shash(spawn);
444 if (IS_ERR(hash))
445 return PTR_ERR(hash);
Loc Hob8a28252008-05-14 21:23:00 +0800446
Herbert Xu46309d82009-07-12 21:38:59 +0800447 ctx->child = hash;
Herbert Xu0d6669e2009-07-12 23:06:33 +0800448 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
449 sizeof(struct cryptd_hash_request_ctx) +
450 crypto_shash_descsize(hash));
Loc Hob8a28252008-05-14 21:23:00 +0800451 return 0;
452}
453
454static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
455{
456 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Loc Hob8a28252008-05-14 21:23:00 +0800457
Herbert Xu46309d82009-07-12 21:38:59 +0800458 crypto_free_shash(ctx->child);
Loc Hob8a28252008-05-14 21:23:00 +0800459}
460
461static int cryptd_hash_setkey(struct crypto_ahash *parent,
462 const u8 *key, unsigned int keylen)
463{
464 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
Herbert Xu46309d82009-07-12 21:38:59 +0800465 struct crypto_shash *child = ctx->child;
Loc Hob8a28252008-05-14 21:23:00 +0800466 int err;
467
Herbert Xu46309d82009-07-12 21:38:59 +0800468 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
469 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
470 CRYPTO_TFM_REQ_MASK);
471 err = crypto_shash_setkey(child, key, keylen);
472 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
473 CRYPTO_TFM_RES_MASK);
Loc Hob8a28252008-05-14 21:23:00 +0800474 return err;
475}
476
477static int cryptd_hash_enqueue(struct ahash_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700478 crypto_completion_t compl)
Loc Hob8a28252008-05-14 21:23:00 +0800479{
480 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
481 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
Huang Ying254eff72009-02-19 14:42:19 +0800482 struct cryptd_queue *queue =
483 cryptd_get_queue(crypto_ahash_tfm(tfm));
Loc Hob8a28252008-05-14 21:23:00 +0800484
485 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700486 req->base.complete = compl;
Loc Hob8a28252008-05-14 21:23:00 +0800487
Huang Ying254eff72009-02-19 14:42:19 +0800488 return cryptd_enqueue_request(queue, &req->base);
Loc Hob8a28252008-05-14 21:23:00 +0800489}
490
Herbert Xu81760ea2016-06-21 16:55:13 +0800491static void cryptd_hash_complete(struct ahash_request *req, int err)
492{
493 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
494 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
495 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
496 int refcnt = atomic_read(&ctx->refcnt);
497
498 local_bh_disable();
499 rctx->complete(&req->base, err);
500 local_bh_enable();
501
502 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
503 crypto_free_ahash(tfm);
504}
505
Loc Hob8a28252008-05-14 21:23:00 +0800506static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
507{
Herbert Xu46309d82009-07-12 21:38:59 +0800508 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
509 struct crypto_shash *child = ctx->child;
510 struct ahash_request *req = ahash_request_cast(req_async);
511 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
512 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800513
514 if (unlikely(err == -EINPROGRESS))
515 goto out;
516
Herbert Xu46309d82009-07-12 21:38:59 +0800517 desc->tfm = child;
518 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800519
Herbert Xu46309d82009-07-12 21:38:59 +0800520 err = crypto_shash_init(desc);
Loc Hob8a28252008-05-14 21:23:00 +0800521
522 req->base.complete = rctx->complete;
523
524out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800525 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800526}
527
528static int cryptd_hash_init_enqueue(struct ahash_request *req)
529{
530 return cryptd_hash_enqueue(req, cryptd_hash_init);
531}
532
533static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
534{
Herbert Xu46309d82009-07-12 21:38:59 +0800535 struct ahash_request *req = ahash_request_cast(req_async);
Loc Hob8a28252008-05-14 21:23:00 +0800536 struct cryptd_hash_request_ctx *rctx;
Loc Hob8a28252008-05-14 21:23:00 +0800537
538 rctx = ahash_request_ctx(req);
539
540 if (unlikely(err == -EINPROGRESS))
541 goto out;
542
Herbert Xu46309d82009-07-12 21:38:59 +0800543 err = shash_ahash_update(req, &rctx->desc);
Loc Hob8a28252008-05-14 21:23:00 +0800544
545 req->base.complete = rctx->complete;
546
547out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800548 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800549}
550
551static int cryptd_hash_update_enqueue(struct ahash_request *req)
552{
553 return cryptd_hash_enqueue(req, cryptd_hash_update);
554}
555
556static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
557{
Herbert Xu46309d82009-07-12 21:38:59 +0800558 struct ahash_request *req = ahash_request_cast(req_async);
559 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Loc Hob8a28252008-05-14 21:23:00 +0800560
561 if (unlikely(err == -EINPROGRESS))
562 goto out;
563
Herbert Xu46309d82009-07-12 21:38:59 +0800564 err = crypto_shash_final(&rctx->desc, req->result);
Loc Hob8a28252008-05-14 21:23:00 +0800565
566 req->base.complete = rctx->complete;
567
568out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800569 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800570}
571
572static int cryptd_hash_final_enqueue(struct ahash_request *req)
573{
574 return cryptd_hash_enqueue(req, cryptd_hash_final);
575}
576
Herbert Xu6fba00d2009-07-22 11:10:22 +0800577static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
578{
579 struct ahash_request *req = ahash_request_cast(req_async);
580 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
581
582 if (unlikely(err == -EINPROGRESS))
583 goto out;
584
585 err = shash_ahash_finup(req, &rctx->desc);
586
587 req->base.complete = rctx->complete;
588
589out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800590 cryptd_hash_complete(req, err);
Herbert Xu6fba00d2009-07-22 11:10:22 +0800591}
592
593static int cryptd_hash_finup_enqueue(struct ahash_request *req)
594{
595 return cryptd_hash_enqueue(req, cryptd_hash_finup);
596}
597
Loc Hob8a28252008-05-14 21:23:00 +0800598static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
599{
Herbert Xu46309d82009-07-12 21:38:59 +0800600 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
601 struct crypto_shash *child = ctx->child;
602 struct ahash_request *req = ahash_request_cast(req_async);
603 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
604 struct shash_desc *desc = &rctx->desc;
Loc Hob8a28252008-05-14 21:23:00 +0800605
606 if (unlikely(err == -EINPROGRESS))
607 goto out;
608
Herbert Xu46309d82009-07-12 21:38:59 +0800609 desc->tfm = child;
610 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Loc Hob8a28252008-05-14 21:23:00 +0800611
Herbert Xu46309d82009-07-12 21:38:59 +0800612 err = shash_ahash_digest(req, desc);
Loc Hob8a28252008-05-14 21:23:00 +0800613
614 req->base.complete = rctx->complete;
615
616out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800617 cryptd_hash_complete(req, err);
Loc Hob8a28252008-05-14 21:23:00 +0800618}
619
620static int cryptd_hash_digest_enqueue(struct ahash_request *req)
621{
622 return cryptd_hash_enqueue(req, cryptd_hash_digest);
623}
624
Herbert Xu6fba00d2009-07-22 11:10:22 +0800625static int cryptd_hash_export(struct ahash_request *req, void *out)
626{
627 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
628
629 return crypto_shash_export(&rctx->desc, out);
630}
631
632static int cryptd_hash_import(struct ahash_request *req, const void *in)
633{
634 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
635
636 return crypto_shash_import(&rctx->desc, in);
637}
638
Herbert Xu9cd899a2009-07-14 18:45:45 +0800639static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
640 struct cryptd_queue *queue)
Loc Hob8a28252008-05-14 21:23:00 +0800641{
Herbert Xu46309d82009-07-12 21:38:59 +0800642 struct hashd_instance_ctx *ctx;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800643 struct ahash_instance *inst;
Herbert Xu46309d82009-07-12 21:38:59 +0800644 struct shash_alg *salg;
Loc Hob8a28252008-05-14 21:23:00 +0800645 struct crypto_alg *alg;
Stephan Mueller466a7b92015-03-30 21:57:06 +0200646 u32 type = 0;
647 u32 mask = 0;
Herbert Xu46309d82009-07-12 21:38:59 +0800648 int err;
Loc Hob8a28252008-05-14 21:23:00 +0800649
Stephan Mueller466a7b92015-03-30 21:57:06 +0200650 cryptd_check_internal(tb, &type, &mask);
651
652 salg = shash_attr_alg(tb[1], type, mask);
Herbert Xu46309d82009-07-12 21:38:59 +0800653 if (IS_ERR(salg))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800654 return PTR_ERR(salg);
Loc Hob8a28252008-05-14 21:23:00 +0800655
Herbert Xu46309d82009-07-12 21:38:59 +0800656 alg = &salg->base;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800657 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
658 sizeof(*ctx));
Steffen Klassert05ed8752009-07-15 16:51:04 +0800659 err = PTR_ERR(inst);
Loc Hob8a28252008-05-14 21:23:00 +0800660 if (IS_ERR(inst))
661 goto out_put_alg;
662
Herbert Xu0b535ad2009-07-14 19:11:32 +0800663 ctx = ahash_instance_ctx(inst);
Herbert Xu46309d82009-07-12 21:38:59 +0800664 ctx->queue = queue;
665
Herbert Xu0b535ad2009-07-14 19:11:32 +0800666 err = crypto_init_shash_spawn(&ctx->spawn, salg,
667 ahash_crypto_instance(inst));
Herbert Xu46309d82009-07-12 21:38:59 +0800668 if (err)
669 goto out_free_inst;
670
Stephan Mueller466a7b92015-03-30 21:57:06 +0200671 type = CRYPTO_ALG_ASYNC;
672 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
673 type |= CRYPTO_ALG_INTERNAL;
674 inst->alg.halg.base.cra_flags = type;
Loc Hob8a28252008-05-14 21:23:00 +0800675
Herbert Xu0b535ad2009-07-14 19:11:32 +0800676 inst->alg.halg.digestsize = salg->digestsize;
Wang, Rui Y1a078342015-11-29 22:45:34 +0800677 inst->alg.halg.statesize = salg->statesize;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800678 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
Loc Hob8a28252008-05-14 21:23:00 +0800679
Herbert Xu0b535ad2009-07-14 19:11:32 +0800680 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
681 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
Loc Hob8a28252008-05-14 21:23:00 +0800682
Herbert Xu0b535ad2009-07-14 19:11:32 +0800683 inst->alg.init = cryptd_hash_init_enqueue;
684 inst->alg.update = cryptd_hash_update_enqueue;
685 inst->alg.final = cryptd_hash_final_enqueue;
Herbert Xu6fba00d2009-07-22 11:10:22 +0800686 inst->alg.finup = cryptd_hash_finup_enqueue;
687 inst->alg.export = cryptd_hash_export;
688 inst->alg.import = cryptd_hash_import;
Herbert Xu0b535ad2009-07-14 19:11:32 +0800689 inst->alg.setkey = cryptd_hash_setkey;
690 inst->alg.digest = cryptd_hash_digest_enqueue;
Loc Hob8a28252008-05-14 21:23:00 +0800691
Herbert Xu0b535ad2009-07-14 19:11:32 +0800692 err = ahash_register_instance(tmpl, inst);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800693 if (err) {
694 crypto_drop_shash(&ctx->spawn);
695out_free_inst:
696 kfree(inst);
697 }
698
Loc Hob8a28252008-05-14 21:23:00 +0800699out_put_alg:
700 crypto_mod_put(alg);
Herbert Xu9cd899a2009-07-14 18:45:45 +0800701 return err;
Loc Hob8a28252008-05-14 21:23:00 +0800702}
703
Herbert Xu92b98762015-05-28 22:08:01 +0800704static int cryptd_aead_setkey(struct crypto_aead *parent,
705 const u8 *key, unsigned int keylen)
706{
707 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
708 struct crypto_aead *child = ctx->child;
709
710 return crypto_aead_setkey(child, key, keylen);
711}
712
713static int cryptd_aead_setauthsize(struct crypto_aead *parent,
714 unsigned int authsize)
715{
716 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
717 struct crypto_aead *child = ctx->child;
718
719 return crypto_aead_setauthsize(child, authsize);
720}
721
Adrian Hoban298c9262010-09-20 16:05:12 +0800722static void cryptd_aead_crypt(struct aead_request *req,
723 struct crypto_aead *child,
724 int err,
725 int (*crypt)(struct aead_request *req))
726{
727 struct cryptd_aead_request_ctx *rctx;
Herbert Xu81760ea2016-06-21 16:55:13 +0800728 struct cryptd_aead_ctx *ctx;
Herbert Xuec9f2002015-07-06 19:11:03 +0800729 crypto_completion_t compl;
Herbert Xu81760ea2016-06-21 16:55:13 +0800730 struct crypto_aead *tfm;
731 int refcnt;
Herbert Xuec9f2002015-07-06 19:11:03 +0800732
Adrian Hoban298c9262010-09-20 16:05:12 +0800733 rctx = aead_request_ctx(req);
Herbert Xuec9f2002015-07-06 19:11:03 +0800734 compl = rctx->complete;
Adrian Hoban298c9262010-09-20 16:05:12 +0800735
736 if (unlikely(err == -EINPROGRESS))
737 goto out;
738 aead_request_set_tfm(req, child);
739 err = crypt( req );
Herbert Xu81760ea2016-06-21 16:55:13 +0800740
Adrian Hoban298c9262010-09-20 16:05:12 +0800741out:
Herbert Xu81760ea2016-06-21 16:55:13 +0800742 tfm = crypto_aead_reqtfm(req);
743 ctx = crypto_aead_ctx(tfm);
744 refcnt = atomic_read(&ctx->refcnt);
745
Adrian Hoban298c9262010-09-20 16:05:12 +0800746 local_bh_disable();
Herbert Xuec9f2002015-07-06 19:11:03 +0800747 compl(&req->base, err);
Adrian Hoban298c9262010-09-20 16:05:12 +0800748 local_bh_enable();
Herbert Xu81760ea2016-06-21 16:55:13 +0800749
750 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
751 crypto_free_aead(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800752}
753
754static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
755{
756 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
757 struct crypto_aead *child = ctx->child;
758 struct aead_request *req;
759
760 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800761 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800762}
763
764static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
765{
766 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
767 struct crypto_aead *child = ctx->child;
768 struct aead_request *req;
769
770 req = container_of(areq, struct aead_request, base);
Herbert Xuba3749a2015-08-13 17:29:02 +0800771 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
Adrian Hoban298c9262010-09-20 16:05:12 +0800772}
773
774static int cryptd_aead_enqueue(struct aead_request *req,
Mark Rustad3e3dc252014-07-25 02:53:38 -0700775 crypto_completion_t compl)
Adrian Hoban298c9262010-09-20 16:05:12 +0800776{
777 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
778 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
779 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
780
781 rctx->complete = req->base.complete;
Mark Rustad3e3dc252014-07-25 02:53:38 -0700782 req->base.complete = compl;
Adrian Hoban298c9262010-09-20 16:05:12 +0800783 return cryptd_enqueue_request(queue, &req->base);
784}
785
786static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
787{
788 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
789}
790
791static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
792{
793 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
794}
795
Herbert Xuf614e542015-05-28 22:08:04 +0800796static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800797{
Herbert Xuf614e542015-05-28 22:08:04 +0800798 struct aead_instance *inst = aead_alg_instance(tfm);
799 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800800 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
Herbert Xuf614e542015-05-28 22:08:04 +0800801 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800802 struct crypto_aead *cipher;
803
804 cipher = crypto_spawn_aead(spawn);
805 if (IS_ERR(cipher))
806 return PTR_ERR(cipher);
807
Adrian Hoban298c9262010-09-20 16:05:12 +0800808 ctx->child = cipher;
Herbert Xuec9f2002015-07-06 19:11:03 +0800809 crypto_aead_set_reqsize(
810 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
811 crypto_aead_reqsize(cipher)));
Adrian Hoban298c9262010-09-20 16:05:12 +0800812 return 0;
813}
814
Herbert Xuf614e542015-05-28 22:08:04 +0800815static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
Adrian Hoban298c9262010-09-20 16:05:12 +0800816{
Herbert Xuf614e542015-05-28 22:08:04 +0800817 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
Adrian Hoban298c9262010-09-20 16:05:12 +0800818 crypto_free_aead(ctx->child);
819}
820
821static int cryptd_create_aead(struct crypto_template *tmpl,
822 struct rtattr **tb,
823 struct cryptd_queue *queue)
824{
825 struct aead_instance_ctx *ctx;
Herbert Xuf614e542015-05-28 22:08:04 +0800826 struct aead_instance *inst;
827 struct aead_alg *alg;
Herbert Xu9b8c4562015-05-21 15:10:57 +0800828 const char *name;
829 u32 type = 0;
Herbert Xuec9f2002015-07-06 19:11:03 +0800830 u32 mask = CRYPTO_ALG_ASYNC;
Adrian Hoban298c9262010-09-20 16:05:12 +0800831 int err;
832
Stephan Mueller466a7b92015-03-30 21:57:06 +0200833 cryptd_check_internal(tb, &type, &mask);
834
Herbert Xu9b8c4562015-05-21 15:10:57 +0800835 name = crypto_attr_alg_name(tb[1]);
836 if (IS_ERR(name))
837 return PTR_ERR(name);
Adrian Hoban298c9262010-09-20 16:05:12 +0800838
Herbert Xu9b8c4562015-05-21 15:10:57 +0800839 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
840 if (!inst)
841 return -ENOMEM;
Adrian Hoban298c9262010-09-20 16:05:12 +0800842
Herbert Xuf614e542015-05-28 22:08:04 +0800843 ctx = aead_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800844 ctx->queue = queue;
845
Herbert Xuf614e542015-05-28 22:08:04 +0800846 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
Herbert Xu9b8c4562015-05-21 15:10:57 +0800847 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
Adrian Hoban298c9262010-09-20 16:05:12 +0800848 if (err)
849 goto out_free_inst;
850
Herbert Xuf614e542015-05-28 22:08:04 +0800851 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
852 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
Herbert Xu9b8c4562015-05-21 15:10:57 +0800853 if (err)
854 goto out_drop_aead;
855
Herbert Xuf614e542015-05-28 22:08:04 +0800856 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800857 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
Herbert Xuf614e542015-05-28 22:08:04 +0800858 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
Adrian Hoban298c9262010-09-20 16:05:12 +0800859
Herbert Xuf614e542015-05-28 22:08:04 +0800860 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
861 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
862
863 inst->alg.init = cryptd_aead_init_tfm;
864 inst->alg.exit = cryptd_aead_exit_tfm;
865 inst->alg.setkey = cryptd_aead_setkey;
866 inst->alg.setauthsize = cryptd_aead_setauthsize;
867 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
868 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
869
870 err = aead_register_instance(tmpl, inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800871 if (err) {
Herbert Xu9b8c4562015-05-21 15:10:57 +0800872out_drop_aead:
873 crypto_drop_aead(&ctx->aead_spawn);
Adrian Hoban298c9262010-09-20 16:05:12 +0800874out_free_inst:
875 kfree(inst);
876 }
Adrian Hoban298c9262010-09-20 16:05:12 +0800877 return err;
878}
879
Huang Ying254eff72009-02-19 14:42:19 +0800880static struct cryptd_queue queue;
Herbert Xu124b53d2007-04-16 20:49:20 +1000881
Herbert Xu9cd899a2009-07-14 18:45:45 +0800882static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
Herbert Xu124b53d2007-04-16 20:49:20 +1000883{
884 struct crypto_attr_type *algt;
885
886 algt = crypto_get_attr_type(tb);
887 if (IS_ERR(algt))
Herbert Xu9cd899a2009-07-14 18:45:45 +0800888 return PTR_ERR(algt);
Herbert Xu124b53d2007-04-16 20:49:20 +1000889
890 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
891 case CRYPTO_ALG_TYPE_BLKCIPHER:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800892 return cryptd_create_blkcipher(tmpl, tb, &queue);
Loc Hob8a28252008-05-14 21:23:00 +0800893 case CRYPTO_ALG_TYPE_DIGEST:
Herbert Xu9cd899a2009-07-14 18:45:45 +0800894 return cryptd_create_hash(tmpl, tb, &queue);
Adrian Hoban298c9262010-09-20 16:05:12 +0800895 case CRYPTO_ALG_TYPE_AEAD:
896 return cryptd_create_aead(tmpl, tb, &queue);
Herbert Xu124b53d2007-04-16 20:49:20 +1000897 }
898
Herbert Xu9cd899a2009-07-14 18:45:45 +0800899 return -EINVAL;
Herbert Xu124b53d2007-04-16 20:49:20 +1000900}
901
902static void cryptd_free(struct crypto_instance *inst)
903{
904 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800905 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
Adrian Hoban298c9262010-09-20 16:05:12 +0800906 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800907
908 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
909 case CRYPTO_ALG_TYPE_AHASH:
910 crypto_drop_shash(&hctx->spawn);
911 kfree(ahash_instance(inst));
912 return;
Adrian Hoban298c9262010-09-20 16:05:12 +0800913 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuf614e542015-05-28 22:08:04 +0800914 crypto_drop_aead(&aead_ctx->aead_spawn);
915 kfree(aead_instance(inst));
Adrian Hoban298c9262010-09-20 16:05:12 +0800916 return;
917 default:
918 crypto_drop_spawn(&ctx->spawn);
919 kfree(inst);
Herbert Xu0b535ad2009-07-14 19:11:32 +0800920 }
Herbert Xu124b53d2007-04-16 20:49:20 +1000921}
922
923static struct crypto_template cryptd_tmpl = {
924 .name = "cryptd",
Herbert Xu9cd899a2009-07-14 18:45:45 +0800925 .create = cryptd_create,
Herbert Xu124b53d2007-04-16 20:49:20 +1000926 .free = cryptd_free,
927 .module = THIS_MODULE,
928};
929
Huang Ying1cac2cb2009-01-18 16:19:46 +1100930struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
931 u32 type, u32 mask)
932{
933 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +0800934 struct cryptd_blkcipher_ctx *ctx;
Huang Ying505fd212009-03-29 15:33:53 +0800935 struct crypto_tfm *tfm;
Huang Ying1cac2cb2009-01-18 16:19:46 +1100936
937 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
938 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
939 return ERR_PTR(-EINVAL);
Alexander Kuleshovc012a792015-11-25 23:48:28 +0600940 type = crypto_skcipher_type(type);
Huang Ying505fd212009-03-29 15:33:53 +0800941 mask &= ~CRYPTO_ALG_TYPE_MASK;
942 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
943 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100944 if (IS_ERR(tfm))
945 return ERR_CAST(tfm);
Huang Ying505fd212009-03-29 15:33:53 +0800946 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
947 crypto_free_tfm(tfm);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100948 return ERR_PTR(-EINVAL);
949 }
950
Herbert Xu81760ea2016-06-21 16:55:13 +0800951 ctx = crypto_tfm_ctx(tfm);
952 atomic_set(&ctx->refcnt, 1);
953
Huang Ying505fd212009-03-29 15:33:53 +0800954 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
Huang Ying1cac2cb2009-01-18 16:19:46 +1100955}
956EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
957
958struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
959{
960 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
961 return ctx->child;
962}
963EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
964
Herbert Xu81760ea2016-06-21 16:55:13 +0800965bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
966{
967 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
968
969 return atomic_read(&ctx->refcnt) - 1;
970}
971EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
972
Huang Ying1cac2cb2009-01-18 16:19:46 +1100973void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
974{
Herbert Xu81760ea2016-06-21 16:55:13 +0800975 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
976
977 if (atomic_dec_and_test(&ctx->refcnt))
978 crypto_free_ablkcipher(&tfm->base);
Huang Ying1cac2cb2009-01-18 16:19:46 +1100979}
980EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
981
Huang Yingace13662009-08-06 15:35:20 +1000982struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
983 u32 type, u32 mask)
984{
985 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +0800986 struct cryptd_hash_ctx *ctx;
Huang Yingace13662009-08-06 15:35:20 +1000987 struct crypto_ahash *tfm;
988
989 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
990 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
991 return ERR_PTR(-EINVAL);
992 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
993 if (IS_ERR(tfm))
994 return ERR_CAST(tfm);
995 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
996 crypto_free_ahash(tfm);
997 return ERR_PTR(-EINVAL);
998 }
999
Herbert Xu81760ea2016-06-21 16:55:13 +08001000 ctx = crypto_ahash_ctx(tfm);
1001 atomic_set(&ctx->refcnt, 1);
1002
Huang Yingace13662009-08-06 15:35:20 +10001003 return __cryptd_ahash_cast(tfm);
1004}
1005EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1006
1007struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1008{
1009 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1010
1011 return ctx->child;
1012}
1013EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1014
Huang Ying0e1227d2009-10-19 11:53:06 +09001015struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1016{
1017 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1018 return &rctx->desc;
1019}
1020EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1021
Herbert Xu81760ea2016-06-21 16:55:13 +08001022bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1023{
1024 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1025
1026 return atomic_read(&ctx->refcnt) - 1;
1027}
1028EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1029
Huang Yingace13662009-08-06 15:35:20 +10001030void cryptd_free_ahash(struct cryptd_ahash *tfm)
1031{
Herbert Xu81760ea2016-06-21 16:55:13 +08001032 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1033
1034 if (atomic_dec_and_test(&ctx->refcnt))
1035 crypto_free_ahash(&tfm->base);
Huang Yingace13662009-08-06 15:35:20 +10001036}
1037EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1038
Adrian Hoban298c9262010-09-20 16:05:12 +08001039struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1040 u32 type, u32 mask)
1041{
1042 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
Herbert Xu81760ea2016-06-21 16:55:13 +08001043 struct cryptd_aead_ctx *ctx;
Adrian Hoban298c9262010-09-20 16:05:12 +08001044 struct crypto_aead *tfm;
1045
1046 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1047 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1048 return ERR_PTR(-EINVAL);
1049 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1050 if (IS_ERR(tfm))
1051 return ERR_CAST(tfm);
1052 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1053 crypto_free_aead(tfm);
1054 return ERR_PTR(-EINVAL);
1055 }
Herbert Xu81760ea2016-06-21 16:55:13 +08001056
1057 ctx = crypto_aead_ctx(tfm);
1058 atomic_set(&ctx->refcnt, 1);
1059
Adrian Hoban298c9262010-09-20 16:05:12 +08001060 return __cryptd_aead_cast(tfm);
1061}
1062EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1063
1064struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1065{
1066 struct cryptd_aead_ctx *ctx;
1067 ctx = crypto_aead_ctx(&tfm->base);
1068 return ctx->child;
1069}
1070EXPORT_SYMBOL_GPL(cryptd_aead_child);
1071
Herbert Xu81760ea2016-06-21 16:55:13 +08001072bool cryptd_aead_queued(struct cryptd_aead *tfm)
1073{
1074 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1075
1076 return atomic_read(&ctx->refcnt) - 1;
1077}
1078EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1079
Adrian Hoban298c9262010-09-20 16:05:12 +08001080void cryptd_free_aead(struct cryptd_aead *tfm)
1081{
Herbert Xu81760ea2016-06-21 16:55:13 +08001082 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1083
1084 if (atomic_dec_and_test(&ctx->refcnt))
1085 crypto_free_aead(&tfm->base);
Adrian Hoban298c9262010-09-20 16:05:12 +08001086}
1087EXPORT_SYMBOL_GPL(cryptd_free_aead);
1088
Herbert Xu124b53d2007-04-16 20:49:20 +10001089static int __init cryptd_init(void)
1090{
1091 int err;
1092
Huang Ying254eff72009-02-19 14:42:19 +08001093 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
Herbert Xu124b53d2007-04-16 20:49:20 +10001094 if (err)
1095 return err;
1096
1097 err = crypto_register_template(&cryptd_tmpl);
1098 if (err)
Huang Ying254eff72009-02-19 14:42:19 +08001099 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001100
1101 return err;
1102}
1103
1104static void __exit cryptd_exit(void)
1105{
Huang Ying254eff72009-02-19 14:42:19 +08001106 cryptd_fini_queue(&queue);
Herbert Xu124b53d2007-04-16 20:49:20 +10001107 crypto_unregister_template(&cryptd_tmpl);
1108}
1109
Herbert Xub2bac6a2011-08-19 16:11:23 +08001110subsys_initcall(cryptd_init);
Herbert Xu124b53d2007-04-16 20:49:20 +10001111module_exit(cryptd_exit);
1112
1113MODULE_LICENSE("GPL");
1114MODULE_DESCRIPTION("Software async crypto daemon");
Kees Cook4943ba12014-11-24 16:32:38 -08001115MODULE_ALIAS_CRYPTO("cryptd");