blob: eca04d3729b37c696c2dac4b0ac472422f30615d [file] [log] [blame]
Tim Chen1e65b812014-07-31 10:29:51 -07001/*
2 * Software multibuffer async crypto daemon.
3 *
4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
5 *
6 * Adapted from crypto daemon.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/algapi.h>
16#include <crypto/internal/hash.h>
17#include <crypto/internal/aead.h>
18#include <crypto/mcryptd.h>
19#include <crypto/crypto_wq.h>
20#include <linux/err.h>
21#include <linux/init.h>
22#include <linux/kernel.h>
23#include <linux/list.h>
24#include <linux/module.h>
25#include <linux/scatterlist.h>
26#include <linux/sched.h>
Ingo Molnar03441a32017-02-08 18:51:35 +010027#include <linux/sched/stat.h>
Tim Chen1e65b812014-07-31 10:29:51 -070028#include <linux/slab.h>
29#include <linux/hardirq.h>
30
31#define MCRYPTD_MAX_CPU_QLEN 100
32#define MCRYPTD_BATCH 9
33
34static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
35 unsigned int tail);
36
37struct mcryptd_flush_list {
38 struct list_head list;
39 struct mutex lock;
40};
41
Fengguang Wu1f6e97f2014-08-26 14:40:39 +080042static struct mcryptd_flush_list __percpu *mcryptd_flist;
Tim Chen1e65b812014-07-31 10:29:51 -070043
44struct hashd_instance_ctx {
Megha Dey331bf732016-06-21 18:21:46 -070045 struct crypto_ahash_spawn spawn;
Tim Chen1e65b812014-07-31 10:29:51 -070046 struct mcryptd_queue *queue;
47};
48
49static void mcryptd_queue_worker(struct work_struct *work);
50
51void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
52{
53 struct mcryptd_flush_list *flist;
54
55 if (!cstate->flusher_engaged) {
56 /* put the flusher on the flush list */
57 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
58 mutex_lock(&flist->lock);
59 list_add_tail(&cstate->flush_list, &flist->list);
60 cstate->flusher_engaged = true;
61 cstate->next_flush = jiffies + delay;
62 queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
63 &cstate->flush, delay);
64 mutex_unlock(&flist->lock);
65 }
66}
67EXPORT_SYMBOL(mcryptd_arm_flusher);
68
69static int mcryptd_init_queue(struct mcryptd_queue *queue,
70 unsigned int max_cpu_qlen)
71{
72 int cpu;
73 struct mcryptd_cpu_queue *cpu_queue;
74
75 queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
76 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
77 if (!queue->cpu_queue)
78 return -ENOMEM;
79 for_each_possible_cpu(cpu) {
80 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
81 pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
82 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
83 INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +010084 spin_lock_init(&cpu_queue->q_lock);
Tim Chen1e65b812014-07-31 10:29:51 -070085 }
86 return 0;
87}
88
89static void mcryptd_fini_queue(struct mcryptd_queue *queue)
90{
91 int cpu;
92 struct mcryptd_cpu_queue *cpu_queue;
93
94 for_each_possible_cpu(cpu) {
95 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
96 BUG_ON(cpu_queue->queue.qlen);
97 }
98 free_percpu(queue->cpu_queue);
99}
100
101static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
102 struct crypto_async_request *request,
103 struct mcryptd_hash_request_ctx *rctx)
104{
105 int cpu, err;
106 struct mcryptd_cpu_queue *cpu_queue;
107
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +0100108 cpu_queue = raw_cpu_ptr(queue->cpu_queue);
109 spin_lock(&cpu_queue->q_lock);
110 cpu = smp_processor_id();
111 rctx->tag.cpu = smp_processor_id();
Tim Chen1e65b812014-07-31 10:29:51 -0700112
113 err = crypto_enqueue_request(&cpu_queue->queue, request);
114 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
115 cpu, cpu_queue, request);
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +0100116 spin_unlock(&cpu_queue->q_lock);
Tim Chen1e65b812014-07-31 10:29:51 -0700117 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
Tim Chen1e65b812014-07-31 10:29:51 -0700118
119 return err;
120}
121
122/*
123 * Try to opportunisticlly flush the partially completed jobs if
124 * crypto daemon is the only task running.
125 */
126static void mcryptd_opportunistic_flush(void)
127{
128 struct mcryptd_flush_list *flist;
129 struct mcryptd_alg_cstate *cstate;
130
131 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
132 while (single_task_running()) {
133 mutex_lock(&flist->lock);
Geliang Tang08346172015-11-16 22:37:15 +0800134 cstate = list_first_entry_or_null(&flist->list,
Tim Chen1e65b812014-07-31 10:29:51 -0700135 struct mcryptd_alg_cstate, flush_list);
Geliang Tang08346172015-11-16 22:37:15 +0800136 if (!cstate || !cstate->flusher_engaged) {
Tim Chen1e65b812014-07-31 10:29:51 -0700137 mutex_unlock(&flist->lock);
138 return;
139 }
140 list_del(&cstate->flush_list);
141 cstate->flusher_engaged = false;
142 mutex_unlock(&flist->lock);
143 cstate->alg_state->flusher(cstate);
144 }
145}
146
147/*
148 * Called in workqueue context, do one real cryption work (via
149 * req->complete) and reschedule itself if there are more work to
150 * do.
151 */
152static void mcryptd_queue_worker(struct work_struct *work)
153{
154 struct mcryptd_cpu_queue *cpu_queue;
155 struct crypto_async_request *req, *backlog;
156 int i;
157
158 /*
159 * Need to loop through more than once for multi-buffer to
160 * be effective.
161 */
162
163 cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
164 i = 0;
165 while (i < MCRYPTD_BATCH || single_task_running()) {
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +0100166
167 spin_lock_bh(&cpu_queue->q_lock);
Tim Chen1e65b812014-07-31 10:29:51 -0700168 backlog = crypto_get_backlog(&cpu_queue->queue);
169 req = crypto_dequeue_request(&cpu_queue->queue);
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +0100170 spin_unlock_bh(&cpu_queue->q_lock);
Tim Chen1e65b812014-07-31 10:29:51 -0700171
172 if (!req) {
173 mcryptd_opportunistic_flush();
174 return;
175 }
176
177 if (backlog)
178 backlog->complete(backlog, -EINPROGRESS);
179 req->complete(req, 0);
180 if (!cpu_queue->queue.qlen)
181 return;
182 ++i;
183 }
184 if (cpu_queue->queue.qlen)
Sebastian Andrzej Siewior9abffc62017-11-30 13:39:27 +0100185 queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work);
Tim Chen1e65b812014-07-31 10:29:51 -0700186}
187
188void mcryptd_flusher(struct work_struct *__work)
189{
190 struct mcryptd_alg_cstate *alg_cpu_state;
191 struct mcryptd_alg_state *alg_state;
192 struct mcryptd_flush_list *flist;
193 int cpu;
194
195 cpu = smp_processor_id();
196 alg_cpu_state = container_of(to_delayed_work(__work),
197 struct mcryptd_alg_cstate, flush);
198 alg_state = alg_cpu_state->alg_state;
199 if (alg_cpu_state->cpu != cpu)
200 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
201 cpu, alg_cpu_state->cpu);
202
203 if (alg_cpu_state->flusher_engaged) {
204 flist = per_cpu_ptr(mcryptd_flist, cpu);
205 mutex_lock(&flist->lock);
206 list_del(&alg_cpu_state->flush_list);
207 alg_cpu_state->flusher_engaged = false;
208 mutex_unlock(&flist->lock);
209 alg_state->flusher(alg_cpu_state);
210 }
211}
212EXPORT_SYMBOL_GPL(mcryptd_flusher);
213
214static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
215{
216 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
217 struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
218
219 return ictx->queue;
220}
221
222static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
223 unsigned int tail)
224{
225 char *p;
226 struct crypto_instance *inst;
227 int err;
228
229 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
230 if (!p)
231 return ERR_PTR(-ENOMEM);
232
233 inst = (void *)(p + head);
234
235 err = -ENAMETOOLONG;
236 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
237 "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
238 goto out_free_inst;
239
240 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
241
242 inst->alg.cra_priority = alg->cra_priority + 50;
243 inst->alg.cra_blocksize = alg->cra_blocksize;
244 inst->alg.cra_alignmask = alg->cra_alignmask;
245
246out:
247 return p;
248
249out_free_inst:
250 kfree(p);
251 p = ERR_PTR(err);
252 goto out;
253}
254
tim48a99272016-12-05 11:46:31 -0800255static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type,
Stephan Muellerf52bbf52015-03-30 22:10:58 +0200256 u32 *mask)
257{
258 struct crypto_attr_type *algt;
259
260 algt = crypto_get_attr_type(tb);
261 if (IS_ERR(algt))
tim48a99272016-12-05 11:46:31 -0800262 return false;
263
264 *type |= algt->type & CRYPTO_ALG_INTERNAL;
265 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
266
267 if (*type & *mask & CRYPTO_ALG_INTERNAL)
268 return true;
269 else
270 return false;
Stephan Muellerf52bbf52015-03-30 22:10:58 +0200271}
272
Tim Chen1e65b812014-07-31 10:29:51 -0700273static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
274{
275 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
276 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
Megha Dey331bf732016-06-21 18:21:46 -0700277 struct crypto_ahash_spawn *spawn = &ictx->spawn;
Tim Chen1e65b812014-07-31 10:29:51 -0700278 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
Megha Dey331bf732016-06-21 18:21:46 -0700279 struct crypto_ahash *hash;
Tim Chen1e65b812014-07-31 10:29:51 -0700280
Megha Dey331bf732016-06-21 18:21:46 -0700281 hash = crypto_spawn_ahash(spawn);
Tim Chen1e65b812014-07-31 10:29:51 -0700282 if (IS_ERR(hash))
283 return PTR_ERR(hash);
284
285 ctx->child = hash;
286 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
287 sizeof(struct mcryptd_hash_request_ctx) +
Megha Dey331bf732016-06-21 18:21:46 -0700288 crypto_ahash_reqsize(hash));
Tim Chen1e65b812014-07-31 10:29:51 -0700289 return 0;
290}
291
292static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
293{
294 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
295
Megha Dey331bf732016-06-21 18:21:46 -0700296 crypto_free_ahash(ctx->child);
Tim Chen1e65b812014-07-31 10:29:51 -0700297}
298
299static int mcryptd_hash_setkey(struct crypto_ahash *parent,
300 const u8 *key, unsigned int keylen)
301{
302 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
Megha Dey331bf732016-06-21 18:21:46 -0700303 struct crypto_ahash *child = ctx->child;
Tim Chen1e65b812014-07-31 10:29:51 -0700304 int err;
305
Megha Dey331bf732016-06-21 18:21:46 -0700306 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
307 crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) &
Tim Chen1e65b812014-07-31 10:29:51 -0700308 CRYPTO_TFM_REQ_MASK);
Megha Dey331bf732016-06-21 18:21:46 -0700309 err = crypto_ahash_setkey(child, key, keylen);
310 crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) &
Tim Chen1e65b812014-07-31 10:29:51 -0700311 CRYPTO_TFM_RES_MASK);
312 return err;
313}
314
315static int mcryptd_hash_enqueue(struct ahash_request *req,
316 crypto_completion_t complete)
317{
318 int ret;
319
320 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
321 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
322 struct mcryptd_queue *queue =
323 mcryptd_get_queue(crypto_ahash_tfm(tfm));
324
325 rctx->complete = req->base.complete;
326 req->base.complete = complete;
327
328 ret = mcryptd_enqueue_request(queue, &req->base, rctx);
329
330 return ret;
331}
332
333static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
334{
335 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
Megha Dey331bf732016-06-21 18:21:46 -0700336 struct crypto_ahash *child = ctx->child;
Tim Chen1e65b812014-07-31 10:29:51 -0700337 struct ahash_request *req = ahash_request_cast(req_async);
338 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Megha Dey331bf732016-06-21 18:21:46 -0700339 struct ahash_request *desc = &rctx->areq;
Tim Chen1e65b812014-07-31 10:29:51 -0700340
341 if (unlikely(err == -EINPROGRESS))
342 goto out;
343
Megha Dey331bf732016-06-21 18:21:46 -0700344 ahash_request_set_tfm(desc, child);
345 ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
346 rctx->complete, req_async);
Tim Chen1e65b812014-07-31 10:29:51 -0700347
Megha Dey331bf732016-06-21 18:21:46 -0700348 rctx->out = req->result;
349 err = crypto_ahash_init(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700350
351out:
352 local_bh_disable();
353 rctx->complete(&req->base, err);
354 local_bh_enable();
355}
356
357static int mcryptd_hash_init_enqueue(struct ahash_request *req)
358{
359 return mcryptd_hash_enqueue(req, mcryptd_hash_init);
360}
361
362static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
363{
364 struct ahash_request *req = ahash_request_cast(req_async);
365 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
366
367 if (unlikely(err == -EINPROGRESS))
368 goto out;
369
Megha Dey331bf732016-06-21 18:21:46 -0700370 rctx->out = req->result;
371 err = ahash_mcryptd_update(&rctx->areq);
Tim Chen1e65b812014-07-31 10:29:51 -0700372 if (err) {
373 req->base.complete = rctx->complete;
374 goto out;
375 }
376
377 return;
378out:
379 local_bh_disable();
380 rctx->complete(&req->base, err);
381 local_bh_enable();
382}
383
384static int mcryptd_hash_update_enqueue(struct ahash_request *req)
385{
386 return mcryptd_hash_enqueue(req, mcryptd_hash_update);
387}
388
389static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
390{
391 struct ahash_request *req = ahash_request_cast(req_async);
392 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
393
394 if (unlikely(err == -EINPROGRESS))
395 goto out;
396
Megha Dey331bf732016-06-21 18:21:46 -0700397 rctx->out = req->result;
398 err = ahash_mcryptd_final(&rctx->areq);
Tim Chen1e65b812014-07-31 10:29:51 -0700399 if (err) {
400 req->base.complete = rctx->complete;
401 goto out;
402 }
403
404 return;
405out:
406 local_bh_disable();
407 rctx->complete(&req->base, err);
408 local_bh_enable();
409}
410
411static int mcryptd_hash_final_enqueue(struct ahash_request *req)
412{
413 return mcryptd_hash_enqueue(req, mcryptd_hash_final);
414}
415
416static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
417{
418 struct ahash_request *req = ahash_request_cast(req_async);
419 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
420
421 if (unlikely(err == -EINPROGRESS))
422 goto out;
Megha Dey331bf732016-06-21 18:21:46 -0700423 rctx->out = req->result;
424 err = ahash_mcryptd_finup(&rctx->areq);
Tim Chen1e65b812014-07-31 10:29:51 -0700425
426 if (err) {
427 req->base.complete = rctx->complete;
428 goto out;
429 }
430
431 return;
432out:
433 local_bh_disable();
434 rctx->complete(&req->base, err);
435 local_bh_enable();
436}
437
438static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
439{
440 return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
441}
442
443static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
444{
445 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
Megha Dey331bf732016-06-21 18:21:46 -0700446 struct crypto_ahash *child = ctx->child;
Tim Chen1e65b812014-07-31 10:29:51 -0700447 struct ahash_request *req = ahash_request_cast(req_async);
448 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Megha Dey331bf732016-06-21 18:21:46 -0700449 struct ahash_request *desc = &rctx->areq;
Tim Chen1e65b812014-07-31 10:29:51 -0700450
451 if (unlikely(err == -EINPROGRESS))
452 goto out;
453
Megha Dey331bf732016-06-21 18:21:46 -0700454 ahash_request_set_tfm(desc, child);
455 ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
456 rctx->complete, req_async);
Tim Chen1e65b812014-07-31 10:29:51 -0700457
Megha Dey331bf732016-06-21 18:21:46 -0700458 rctx->out = req->result;
459 err = ahash_mcryptd_digest(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700460
Tim Chen1e65b812014-07-31 10:29:51 -0700461out:
462 local_bh_disable();
463 rctx->complete(&req->base, err);
464 local_bh_enable();
465}
466
467static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
468{
469 return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
470}
471
472static int mcryptd_hash_export(struct ahash_request *req, void *out)
473{
474 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
475
Megha Dey331bf732016-06-21 18:21:46 -0700476 return crypto_ahash_export(&rctx->areq, out);
Tim Chen1e65b812014-07-31 10:29:51 -0700477}
478
479static int mcryptd_hash_import(struct ahash_request *req, const void *in)
480{
481 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
482
Megha Dey331bf732016-06-21 18:21:46 -0700483 return crypto_ahash_import(&rctx->areq, in);
Tim Chen1e65b812014-07-31 10:29:51 -0700484}
485
486static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
487 struct mcryptd_queue *queue)
488{
489 struct hashd_instance_ctx *ctx;
490 struct ahash_instance *inst;
Megha Dey331bf732016-06-21 18:21:46 -0700491 struct hash_alg_common *halg;
Tim Chen1e65b812014-07-31 10:29:51 -0700492 struct crypto_alg *alg;
Stephan Muellerf52bbf52015-03-30 22:10:58 +0200493 u32 type = 0;
494 u32 mask = 0;
Tim Chen1e65b812014-07-31 10:29:51 -0700495 int err;
496
tim48a99272016-12-05 11:46:31 -0800497 if (!mcryptd_check_internal(tb, &type, &mask))
498 return -EINVAL;
Stephan Muellerf52bbf52015-03-30 22:10:58 +0200499
Megha Dey331bf732016-06-21 18:21:46 -0700500 halg = ahash_attr_alg(tb[1], type, mask);
501 if (IS_ERR(halg))
502 return PTR_ERR(halg);
Tim Chen1e65b812014-07-31 10:29:51 -0700503
Megha Dey331bf732016-06-21 18:21:46 -0700504 alg = &halg->base;
Tim Chen1e65b812014-07-31 10:29:51 -0700505 pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
506 inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
507 sizeof(*ctx));
508 err = PTR_ERR(inst);
509 if (IS_ERR(inst))
510 goto out_put_alg;
511
512 ctx = ahash_instance_ctx(inst);
513 ctx->queue = queue;
514
Megha Dey331bf732016-06-21 18:21:46 -0700515 err = crypto_init_ahash_spawn(&ctx->spawn, halg,
Tim Chen1e65b812014-07-31 10:29:51 -0700516 ahash_crypto_instance(inst));
517 if (err)
518 goto out_free_inst;
519
Stephan Muellerf52bbf52015-03-30 22:10:58 +0200520 type = CRYPTO_ALG_ASYNC;
521 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
522 type |= CRYPTO_ALG_INTERNAL;
523 inst->alg.halg.base.cra_flags = type;
Tim Chen1e65b812014-07-31 10:29:51 -0700524
Megha Dey331bf732016-06-21 18:21:46 -0700525 inst->alg.halg.digestsize = halg->digestsize;
526 inst->alg.halg.statesize = halg->statesize;
Tim Chen1e65b812014-07-31 10:29:51 -0700527 inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
528
529 inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
530 inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
531
532 inst->alg.init = mcryptd_hash_init_enqueue;
533 inst->alg.update = mcryptd_hash_update_enqueue;
534 inst->alg.final = mcryptd_hash_final_enqueue;
535 inst->alg.finup = mcryptd_hash_finup_enqueue;
536 inst->alg.export = mcryptd_hash_export;
537 inst->alg.import = mcryptd_hash_import;
538 inst->alg.setkey = mcryptd_hash_setkey;
539 inst->alg.digest = mcryptd_hash_digest_enqueue;
540
541 err = ahash_register_instance(tmpl, inst);
542 if (err) {
Megha Dey331bf732016-06-21 18:21:46 -0700543 crypto_drop_ahash(&ctx->spawn);
Tim Chen1e65b812014-07-31 10:29:51 -0700544out_free_inst:
545 kfree(inst);
546 }
547
548out_put_alg:
549 crypto_mod_put(alg);
550 return err;
551}
552
553static struct mcryptd_queue mqueue;
554
555static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
556{
557 struct crypto_attr_type *algt;
558
559 algt = crypto_get_attr_type(tb);
560 if (IS_ERR(algt))
561 return PTR_ERR(algt);
562
563 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
564 case CRYPTO_ALG_TYPE_DIGEST:
565 return mcryptd_create_hash(tmpl, tb, &mqueue);
566 break;
567 }
568
569 return -EINVAL;
570}
571
572static void mcryptd_free(struct crypto_instance *inst)
573{
574 struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
575 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
576
577 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
578 case CRYPTO_ALG_TYPE_AHASH:
Megha Dey331bf732016-06-21 18:21:46 -0700579 crypto_drop_ahash(&hctx->spawn);
Tim Chen1e65b812014-07-31 10:29:51 -0700580 kfree(ahash_instance(inst));
581 return;
582 default:
583 crypto_drop_spawn(&ctx->spawn);
584 kfree(inst);
585 }
586}
587
588static struct crypto_template mcryptd_tmpl = {
589 .name = "mcryptd",
590 .create = mcryptd_create,
591 .free = mcryptd_free,
592 .module = THIS_MODULE,
593};
594
595struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
596 u32 type, u32 mask)
597{
598 char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
599 struct crypto_ahash *tfm;
600
601 if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
602 "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
603 return ERR_PTR(-EINVAL);
604 tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
605 if (IS_ERR(tfm))
606 return ERR_CAST(tfm);
607 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
608 crypto_free_ahash(tfm);
609 return ERR_PTR(-EINVAL);
610 }
611
612 return __mcryptd_ahash_cast(tfm);
613}
614EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
615
Megha Dey331bf732016-06-21 18:21:46 -0700616int ahash_mcryptd_digest(struct ahash_request *desc)
Tim Chen1e65b812014-07-31 10:29:51 -0700617{
Masahiro Yamada36e09e12016-09-13 04:27:54 +0900618 return crypto_ahash_init(desc) ?: ahash_mcryptd_finup(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700619}
Tim Chen1e65b812014-07-31 10:29:51 -0700620
Megha Dey331bf732016-06-21 18:21:46 -0700621int ahash_mcryptd_update(struct ahash_request *desc)
Tim Chen1e65b812014-07-31 10:29:51 -0700622{
Tim Chen1e65b812014-07-31 10:29:51 -0700623 /* alignment is to be done by multi-buffer crypto algorithm if needed */
624
Megha Dey331bf732016-06-21 18:21:46 -0700625 return crypto_ahash_update(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700626}
Tim Chen1e65b812014-07-31 10:29:51 -0700627
Megha Dey331bf732016-06-21 18:21:46 -0700628int ahash_mcryptd_finup(struct ahash_request *desc)
Tim Chen1e65b812014-07-31 10:29:51 -0700629{
Tim Chen1e65b812014-07-31 10:29:51 -0700630 /* alignment is to be done by multi-buffer crypto algorithm if needed */
631
Megha Dey331bf732016-06-21 18:21:46 -0700632 return crypto_ahash_finup(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700633}
Tim Chen1e65b812014-07-31 10:29:51 -0700634
Megha Dey331bf732016-06-21 18:21:46 -0700635int ahash_mcryptd_final(struct ahash_request *desc)
Tim Chen1e65b812014-07-31 10:29:51 -0700636{
Tim Chen1e65b812014-07-31 10:29:51 -0700637 /* alignment is to be done by multi-buffer crypto algorithm if needed */
638
Megha Dey331bf732016-06-21 18:21:46 -0700639 return crypto_ahash_final(desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700640}
Tim Chen1e65b812014-07-31 10:29:51 -0700641
Megha Dey331bf732016-06-21 18:21:46 -0700642struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
Tim Chen1e65b812014-07-31 10:29:51 -0700643{
644 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
645
646 return ctx->child;
647}
648EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
649
Megha Dey331bf732016-06-21 18:21:46 -0700650struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req)
Tim Chen1e65b812014-07-31 10:29:51 -0700651{
652 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
Megha Dey331bf732016-06-21 18:21:46 -0700653 return &rctx->areq;
Tim Chen1e65b812014-07-31 10:29:51 -0700654}
Megha Dey331bf732016-06-21 18:21:46 -0700655EXPORT_SYMBOL_GPL(mcryptd_ahash_desc);
Tim Chen1e65b812014-07-31 10:29:51 -0700656
657void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
658{
659 crypto_free_ahash(&tfm->base);
660}
661EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
662
Tim Chen1e65b812014-07-31 10:29:51 -0700663static int __init mcryptd_init(void)
664{
665 int err, cpu;
666 struct mcryptd_flush_list *flist;
667
668 mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
669 for_each_possible_cpu(cpu) {
670 flist = per_cpu_ptr(mcryptd_flist, cpu);
671 INIT_LIST_HEAD(&flist->list);
672 mutex_init(&flist->lock);
673 }
674
675 err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
676 if (err) {
677 free_percpu(mcryptd_flist);
678 return err;
679 }
680
681 err = crypto_register_template(&mcryptd_tmpl);
682 if (err) {
683 mcryptd_fini_queue(&mqueue);
684 free_percpu(mcryptd_flist);
685 }
686
687 return err;
688}
689
690static void __exit mcryptd_exit(void)
691{
692 mcryptd_fini_queue(&mqueue);
693 crypto_unregister_template(&mcryptd_tmpl);
694 free_percpu(mcryptd_flist);
695}
696
697subsys_initcall(mcryptd_init);
698module_exit(mcryptd_exit);
699
700MODULE_LICENSE("GPL");
701MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
Kees Cook4943ba12014-11-24 16:32:38 -0800702MODULE_ALIAS_CRYPTO("mcryptd");