blob: aaf2f810d170f2e31039dd5e2b607220645b4122 [file] [log] [blame]
Herbert Xu8ff59092010-10-19 21:31:55 +08001/*
2 * algif_skcipher: User-space interface for skcipher algorithms
3 *
4 * This file provides the user-space API for symmetric key ciphers.
5 *
6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/scatterwalk.h>
16#include <crypto/skcipher.h>
17#include <crypto/if_alg.h>
18#include <linux/init.h>
19#include <linux/list.h>
20#include <linux/kernel.h>
21#include <linux/mm.h>
22#include <linux/module.h>
23#include <linux/net.h>
24#include <net/sock.h>
25
26struct skcipher_sg_list {
27 struct list_head list;
28
29 int cur;
30
31 struct scatterlist sg[0];
32};
33
Herbert Xudd504582015-12-25 15:40:05 +080034struct skcipher_tfm {
35 struct crypto_skcipher *skcipher;
36 bool has_key;
37};
38
Herbert Xu8ff59092010-10-19 21:31:55 +080039struct skcipher_ctx {
40 struct list_head tsgl;
41 struct af_alg_sgl rsgl;
42
43 void *iv;
44
45 struct af_alg_completion completion;
46
Tadeusz Struka5969992015-03-19 12:31:40 -070047 atomic_t inflight;
LABBE Corentin652d5b82015-10-23 14:10:36 +020048 size_t used;
Herbert Xu8ff59092010-10-19 21:31:55 +080049
50 unsigned int len;
51 bool more;
52 bool merge;
53 bool enc;
54
Herbert Xu0d96e4b2015-12-18 19:16:57 +080055 struct skcipher_request req;
Herbert Xu8ff59092010-10-19 21:31:55 +080056};
57
Tadeusz Struka5969992015-03-19 12:31:40 -070058struct skcipher_async_rsgl {
59 struct af_alg_sgl sgl;
60 struct list_head list;
61};
62
63struct skcipher_async_req {
64 struct kiocb *iocb;
65 struct skcipher_async_rsgl first_sgl;
66 struct list_head list;
67 struct scatterlist *tsg;
Herbert Xuec69bbf2016-02-03 21:39:24 +080068 atomic_t *inflight;
69 struct skcipher_request req;
Tadeusz Struka5969992015-03-19 12:31:40 -070070};
71
Ondrej Kozinae2cffb52014-08-25 11:49:54 +020072#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
Herbert Xu8ff59092010-10-19 21:31:55 +080073 sizeof(struct scatterlist) - 1)
74
Tadeusz Struka5969992015-03-19 12:31:40 -070075static void skcipher_free_async_sgls(struct skcipher_async_req *sreq)
76{
77 struct skcipher_async_rsgl *rsgl, *tmp;
78 struct scatterlist *sgl;
79 struct scatterlist *sg;
80 int i, n;
81
82 list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) {
83 af_alg_free_sg(&rsgl->sgl);
84 if (rsgl != &sreq->first_sgl)
85 kfree(rsgl);
86 }
87 sgl = sreq->tsg;
88 n = sg_nents(sgl);
Stephan Mueller9e0a6432017-08-16 11:56:24 +020089 for_each_sg(sgl, sg, n, i) {
90 struct page *page = sg_page(sg);
91
92 /* some SGs may not have a page mapped */
93 if (page && page_ref_count(page))
94 put_page(page);
95 }
Tadeusz Struka5969992015-03-19 12:31:40 -070096
97 kfree(sreq->tsg);
98}
99
100static void skcipher_async_cb(struct crypto_async_request *req, int err)
101{
Herbert Xuec69bbf2016-02-03 21:39:24 +0800102 struct skcipher_async_req *sreq = req->data;
Tadeusz Struka5969992015-03-19 12:31:40 -0700103 struct kiocb *iocb = sreq->iocb;
104
Herbert Xuec69bbf2016-02-03 21:39:24 +0800105 atomic_dec(sreq->inflight);
Tadeusz Struka5969992015-03-19 12:31:40 -0700106 skcipher_free_async_sgls(sreq);
Herbert Xuec69bbf2016-02-03 21:39:24 +0800107 kzfree(sreq);
Al Viro237dae82015-04-09 00:00:30 -0400108 iocb->ki_complete(iocb, err, err);
Tadeusz Struka5969992015-03-19 12:31:40 -0700109}
110
Herbert Xu0f6bb832010-11-30 16:49:02 +0800111static inline int skcipher_sndbuf(struct sock *sk)
Herbert Xu8ff59092010-10-19 21:31:55 +0800112{
113 struct alg_sock *ask = alg_sk(sk);
114 struct skcipher_ctx *ctx = ask->private;
115
Herbert Xu0f6bb832010-11-30 16:49:02 +0800116 return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
117 ctx->used, 0);
118}
119
120static inline bool skcipher_writable(struct sock *sk)
121{
122 return PAGE_SIZE <= skcipher_sndbuf(sk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800123}
124
125static int skcipher_alloc_sgl(struct sock *sk)
126{
127 struct alg_sock *ask = alg_sk(sk);
128 struct skcipher_ctx *ctx = ask->private;
129 struct skcipher_sg_list *sgl;
130 struct scatterlist *sg = NULL;
131
132 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
133 if (!list_empty(&ctx->tsgl))
134 sg = sgl->sg;
135
136 if (!sg || sgl->cur >= MAX_SGL_ENTS) {
137 sgl = sock_kmalloc(sk, sizeof(*sgl) +
138 sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
139 GFP_KERNEL);
140 if (!sgl)
141 return -ENOMEM;
142
143 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
144 sgl->cur = 0;
145
Stephan Muellere684db92017-09-21 10:16:53 +0200146 if (sg) {
Dan Williamsc56f6d12015-08-07 18:15:13 +0200147 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
Stephan Muellere684db92017-09-21 10:16:53 +0200148 sg_unmark_end(sg + (MAX_SGL_ENTS - 1));
149 }
Herbert Xu8ff59092010-10-19 21:31:55 +0800150
151 list_add_tail(&sgl->list, &ctx->tsgl);
152 }
153
154 return 0;
155}
156
LABBE Corentin652d5b82015-10-23 14:10:36 +0200157static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
Herbert Xu8ff59092010-10-19 21:31:55 +0800158{
159 struct alg_sock *ask = alg_sk(sk);
160 struct skcipher_ctx *ctx = ask->private;
161 struct skcipher_sg_list *sgl;
162 struct scatterlist *sg;
163 int i;
164
165 while (!list_empty(&ctx->tsgl)) {
166 sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
167 list);
168 sg = sgl->sg;
169
170 for (i = 0; i < sgl->cur; i++) {
LABBE Corentin652d5b82015-10-23 14:10:36 +0200171 size_t plen = min_t(size_t, used, sg[i].length);
Herbert Xu8ff59092010-10-19 21:31:55 +0800172
173 if (!sg_page(sg + i))
174 continue;
175
176 sg[i].length -= plen;
177 sg[i].offset += plen;
178
179 used -= plen;
180 ctx->used -= plen;
181
182 if (sg[i].length)
183 return;
Tadeusz Struka5969992015-03-19 12:31:40 -0700184 if (put)
185 put_page(sg_page(sg + i));
Herbert Xu8ff59092010-10-19 21:31:55 +0800186 sg_assign_page(sg + i, NULL);
187 }
188
189 list_del(&sgl->list);
190 sock_kfree_s(sk, sgl,
191 sizeof(*sgl) + sizeof(sgl->sg[0]) *
192 (MAX_SGL_ENTS + 1));
193 }
194
195 if (!ctx->used)
196 ctx->merge = 0;
197}
198
199static void skcipher_free_sgl(struct sock *sk)
200{
201 struct alg_sock *ask = alg_sk(sk);
202 struct skcipher_ctx *ctx = ask->private;
203
Tadeusz Struka5969992015-03-19 12:31:40 -0700204 skcipher_pull_sgl(sk, ctx->used, 1);
Herbert Xu8ff59092010-10-19 21:31:55 +0800205}
206
207static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
208{
209 long timeout;
210 DEFINE_WAIT(wait);
211 int err = -ERESTARTSYS;
212
213 if (flags & MSG_DONTWAIT)
214 return -EAGAIN;
215
Eric Dumazet9cd3e072015-11-29 20:03:10 -0800216 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800217
218 for (;;) {
219 if (signal_pending(current))
220 break;
221 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
222 timeout = MAX_SCHEDULE_TIMEOUT;
223 if (sk_wait_event(sk, &timeout, skcipher_writable(sk))) {
224 err = 0;
225 break;
226 }
227 }
228 finish_wait(sk_sleep(sk), &wait);
229
230 return err;
231}
232
233static void skcipher_wmem_wakeup(struct sock *sk)
234{
235 struct socket_wq *wq;
236
237 if (!skcipher_writable(sk))
238 return;
239
240 rcu_read_lock();
241 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +0800242 if (skwq_has_sleeper(wq))
Herbert Xu8ff59092010-10-19 21:31:55 +0800243 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
244 POLLRDNORM |
245 POLLRDBAND);
246 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
247 rcu_read_unlock();
248}
249
250static int skcipher_wait_for_data(struct sock *sk, unsigned flags)
251{
252 struct alg_sock *ask = alg_sk(sk);
253 struct skcipher_ctx *ctx = ask->private;
254 long timeout;
255 DEFINE_WAIT(wait);
256 int err = -ERESTARTSYS;
257
258 if (flags & MSG_DONTWAIT) {
259 return -EAGAIN;
260 }
261
Eric Dumazet9cd3e072015-11-29 20:03:10 -0800262 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800263
264 for (;;) {
265 if (signal_pending(current))
266 break;
267 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
268 timeout = MAX_SCHEDULE_TIMEOUT;
269 if (sk_wait_event(sk, &timeout, ctx->used)) {
270 err = 0;
271 break;
272 }
273 }
274 finish_wait(sk_sleep(sk), &wait);
275
Eric Dumazet9cd3e072015-11-29 20:03:10 -0800276 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800277
278 return err;
279}
280
281static void skcipher_data_wakeup(struct sock *sk)
282{
283 struct alg_sock *ask = alg_sk(sk);
284 struct skcipher_ctx *ctx = ask->private;
285 struct socket_wq *wq;
286
287 if (!ctx->used)
288 return;
289
290 rcu_read_lock();
291 wq = rcu_dereference(sk->sk_wq);
Herbert Xu1ce0bf52015-11-26 13:55:39 +0800292 if (skwq_has_sleeper(wq))
Herbert Xu8ff59092010-10-19 21:31:55 +0800293 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
294 POLLRDNORM |
295 POLLRDBAND);
296 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
297 rcu_read_unlock();
298}
299
Ying Xue1b784142015-03-02 15:37:48 +0800300static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
301 size_t size)
Herbert Xu8ff59092010-10-19 21:31:55 +0800302{
303 struct sock *sk = sock->sk;
304 struct alg_sock *ask = alg_sk(sk);
Herbert Xu6454c2b2016-02-03 21:39:26 +0800305 struct sock *psk = ask->parent;
306 struct alg_sock *pask = alg_sk(psk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800307 struct skcipher_ctx *ctx = ask->private;
Herbert Xu6454c2b2016-02-03 21:39:26 +0800308 struct skcipher_tfm *skc = pask->private;
309 struct crypto_skcipher *tfm = skc->skcipher;
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800310 unsigned ivsize = crypto_skcipher_ivsize(tfm);
Herbert Xu8ff59092010-10-19 21:31:55 +0800311 struct skcipher_sg_list *sgl;
312 struct af_alg_control con = {};
313 long copied = 0;
314 bool enc = 0;
Stephan Muellerf26b7b82014-11-30 10:55:26 +0100315 bool init = 0;
Herbert Xu8ff59092010-10-19 21:31:55 +0800316 int err;
317 int i;
318
319 if (msg->msg_controllen) {
320 err = af_alg_cmsg_send(msg, &con);
321 if (err)
322 return err;
323
Stephan Muellerf26b7b82014-11-30 10:55:26 +0100324 init = 1;
Herbert Xu8ff59092010-10-19 21:31:55 +0800325 switch (con.op) {
326 case ALG_OP_ENCRYPT:
327 enc = 1;
328 break;
329 case ALG_OP_DECRYPT:
330 enc = 0;
331 break;
332 default:
333 return -EINVAL;
334 }
335
336 if (con.iv && con.iv->ivlen != ivsize)
337 return -EINVAL;
338 }
339
340 err = -EINVAL;
341
342 lock_sock(sk);
343 if (!ctx->more && ctx->used)
344 goto unlock;
345
Stephan Muellerf26b7b82014-11-30 10:55:26 +0100346 if (init) {
Herbert Xu8ff59092010-10-19 21:31:55 +0800347 ctx->enc = enc;
348 if (con.iv)
349 memcpy(ctx->iv, con.iv->iv, ivsize);
350 }
351
Herbert Xu8ff59092010-10-19 21:31:55 +0800352 while (size) {
353 struct scatterlist *sg;
354 unsigned long len = size;
LABBE Corentin652d5b82015-10-23 14:10:36 +0200355 size_t plen;
Herbert Xu8ff59092010-10-19 21:31:55 +0800356
357 if (ctx->merge) {
358 sgl = list_entry(ctx->tsgl.prev,
359 struct skcipher_sg_list, list);
360 sg = sgl->sg + sgl->cur - 1;
361 len = min_t(unsigned long, len,
362 PAGE_SIZE - sg->offset - sg->length);
363
Al Viro6ce8e9c2014-04-06 21:25:44 -0400364 err = memcpy_from_msg(page_address(sg_page(sg)) +
365 sg->offset + sg->length,
366 msg, len);
Herbert Xu8ff59092010-10-19 21:31:55 +0800367 if (err)
368 goto unlock;
369
370 sg->length += len;
371 ctx->merge = (sg->offset + sg->length) &
372 (PAGE_SIZE - 1);
373
374 ctx->used += len;
375 copied += len;
376 size -= len;
Herbert Xu8ff59092010-10-19 21:31:55 +0800377 continue;
378 }
379
Herbert Xu0f6bb832010-11-30 16:49:02 +0800380 if (!skcipher_writable(sk)) {
Herbert Xu8ff59092010-10-19 21:31:55 +0800381 err = skcipher_wait_for_wmem(sk, msg->msg_flags);
382 if (err)
383 goto unlock;
Herbert Xu8ff59092010-10-19 21:31:55 +0800384 }
385
Herbert Xu0f6bb832010-11-30 16:49:02 +0800386 len = min_t(unsigned long, len, skcipher_sndbuf(sk));
Herbert Xu8ff59092010-10-19 21:31:55 +0800387
388 err = skcipher_alloc_sgl(sk);
389 if (err)
390 goto unlock;
391
392 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
393 sg = sgl->sg;
Herbert Xu202736d2016-01-19 21:23:57 +0800394 if (sgl->cur)
395 sg_unmark_end(sg + sgl->cur - 1);
Herbert Xu8ff59092010-10-19 21:31:55 +0800396 do {
397 i = sgl->cur;
LABBE Corentin652d5b82015-10-23 14:10:36 +0200398 plen = min_t(size_t, len, PAGE_SIZE);
Herbert Xu8ff59092010-10-19 21:31:55 +0800399
400 sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
401 err = -ENOMEM;
402 if (!sg_page(sg + i))
403 goto unlock;
404
Al Viro6ce8e9c2014-04-06 21:25:44 -0400405 err = memcpy_from_msg(page_address(sg_page(sg + i)),
406 msg, plen);
Herbert Xu8ff59092010-10-19 21:31:55 +0800407 if (err) {
408 __free_page(sg_page(sg + i));
409 sg_assign_page(sg + i, NULL);
410 goto unlock;
411 }
412
413 sg[i].length = plen;
414 len -= plen;
415 ctx->used += plen;
416 copied += plen;
417 size -= plen;
Herbert Xu8ff59092010-10-19 21:31:55 +0800418 sgl->cur++;
419 } while (len && sgl->cur < MAX_SGL_ENTS);
420
Tadeusz Struk0f477b62014-12-08 12:03:42 -0800421 if (!size)
422 sg_mark_end(sg + sgl->cur - 1);
423
Herbert Xu8ff59092010-10-19 21:31:55 +0800424 ctx->merge = plen & (PAGE_SIZE - 1);
425 }
426
427 err = 0;
428
429 ctx->more = msg->msg_flags & MSG_MORE;
Herbert Xu8ff59092010-10-19 21:31:55 +0800430
431unlock:
432 skcipher_data_wakeup(sk);
433 release_sock(sk);
434
435 return copied ?: err;
436}
437
438static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
439 int offset, size_t size, int flags)
440{
441 struct sock *sk = sock->sk;
442 struct alg_sock *ask = alg_sk(sk);
443 struct skcipher_ctx *ctx = ask->private;
444 struct skcipher_sg_list *sgl;
445 int err = -EINVAL;
Herbert Xu8ff59092010-10-19 21:31:55 +0800446
Shawn Landdend3f7d562013-11-24 22:36:28 -0800447 if (flags & MSG_SENDPAGE_NOTLAST)
448 flags |= MSG_MORE;
449
Herbert Xu8ff59092010-10-19 21:31:55 +0800450 lock_sock(sk);
451 if (!ctx->more && ctx->used)
452 goto unlock;
453
454 if (!size)
455 goto done;
456
Herbert Xu0f6bb832010-11-30 16:49:02 +0800457 if (!skcipher_writable(sk)) {
Herbert Xu8ff59092010-10-19 21:31:55 +0800458 err = skcipher_wait_for_wmem(sk, flags);
459 if (err)
460 goto unlock;
Herbert Xu8ff59092010-10-19 21:31:55 +0800461 }
462
463 err = skcipher_alloc_sgl(sk);
464 if (err)
465 goto unlock;
466
467 ctx->merge = 0;
468 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
469
Tadeusz Struk0f477b62014-12-08 12:03:42 -0800470 if (sgl->cur)
471 sg_unmark_end(sgl->sg + sgl->cur - 1);
472
473 sg_mark_end(sgl->sg + sgl->cur);
Herbert Xu8ff59092010-10-19 21:31:55 +0800474 get_page(page);
475 sg_set_page(sgl->sg + sgl->cur, page, size, offset);
476 sgl->cur++;
477 ctx->used += size;
478
479done:
480 ctx->more = flags & MSG_MORE;
Herbert Xu8ff59092010-10-19 21:31:55 +0800481
482unlock:
483 skcipher_data_wakeup(sk);
484 release_sock(sk);
485
486 return err ?: size;
487}
488
Tadeusz Struka5969992015-03-19 12:31:40 -0700489static int skcipher_all_sg_nents(struct skcipher_ctx *ctx)
490{
491 struct skcipher_sg_list *sgl;
492 struct scatterlist *sg;
493 int nents = 0;
494
495 list_for_each_entry(sgl, &ctx->tsgl, list) {
496 sg = sgl->sg;
497
498 while (!sg->length)
499 sg++;
500
501 nents += sg_nents(sg);
502 }
503 return nents;
504}
505
506static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
507 int flags)
508{
509 struct sock *sk = sock->sk;
510 struct alg_sock *ask = alg_sk(sk);
Herbert Xuec69bbf2016-02-03 21:39:24 +0800511 struct sock *psk = ask->parent;
512 struct alg_sock *pask = alg_sk(psk);
Tadeusz Struka5969992015-03-19 12:31:40 -0700513 struct skcipher_ctx *ctx = ask->private;
Herbert Xuec69bbf2016-02-03 21:39:24 +0800514 struct skcipher_tfm *skc = pask->private;
515 struct crypto_skcipher *tfm = skc->skcipher;
Tadeusz Struka5969992015-03-19 12:31:40 -0700516 struct skcipher_sg_list *sgl;
517 struct scatterlist *sg;
518 struct skcipher_async_req *sreq;
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800519 struct skcipher_request *req;
Tadeusz Struka5969992015-03-19 12:31:40 -0700520 struct skcipher_async_rsgl *last_rsgl = NULL;
Herbert Xu6454c2b2016-02-03 21:39:26 +0800521 unsigned int txbufs = 0, len = 0, tx_nents;
Herbert Xuec69bbf2016-02-03 21:39:24 +0800522 unsigned int reqsize = crypto_skcipher_reqsize(tfm);
523 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
Tadeusz Struka5969992015-03-19 12:31:40 -0700524 int err = -ENOMEM;
tadeusz.struk@intel.com033f46b2015-04-01 13:53:06 -0700525 bool mark = false;
Herbert Xuec69bbf2016-02-03 21:39:24 +0800526 char *iv;
527
528 sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL);
529 if (unlikely(!sreq))
530 goto out;
531
532 req = &sreq->req;
533 iv = (char *)(req + 1) + reqsize;
534 sreq->iocb = msg->msg_iocb;
535 INIT_LIST_HEAD(&sreq->list);
536 sreq->inflight = &ctx->inflight;
Tadeusz Struka5969992015-03-19 12:31:40 -0700537
538 lock_sock(sk);
Herbert Xu6454c2b2016-02-03 21:39:26 +0800539 tx_nents = skcipher_all_sg_nents(ctx);
Tadeusz Struka5969992015-03-19 12:31:40 -0700540 sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL);
Herbert Xuec69bbf2016-02-03 21:39:24 +0800541 if (unlikely(!sreq->tsg))
Tadeusz Struka5969992015-03-19 12:31:40 -0700542 goto unlock;
Tadeusz Struka5969992015-03-19 12:31:40 -0700543 sg_init_table(sreq->tsg, tx_nents);
Herbert Xuec69bbf2016-02-03 21:39:24 +0800544 memcpy(iv, ctx->iv, ivsize);
545 skcipher_request_set_tfm(req, tfm);
Herbert Xudad41992016-02-03 21:39:27 +0800546 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
Herbert Xuec69bbf2016-02-03 21:39:24 +0800547 skcipher_async_cb, sreq);
Tadeusz Struka5969992015-03-19 12:31:40 -0700548
549 while (iov_iter_count(&msg->msg_iter)) {
550 struct skcipher_async_rsgl *rsgl;
tadeusz.struk@intel.comac110f42015-03-25 07:29:19 -0700551 int used;
Tadeusz Struka5969992015-03-19 12:31:40 -0700552
553 if (!ctx->used) {
554 err = skcipher_wait_for_data(sk, flags);
555 if (err)
556 goto free;
557 }
558 sgl = list_first_entry(&ctx->tsgl,
559 struct skcipher_sg_list, list);
560 sg = sgl->sg;
561
562 while (!sg->length)
563 sg++;
564
565 used = min_t(unsigned long, ctx->used,
566 iov_iter_count(&msg->msg_iter));
567 used = min_t(unsigned long, used, sg->length);
568
tadeusz.struk@intel.com033f46b2015-04-01 13:53:06 -0700569 if (txbufs == tx_nents) {
Tadeusz Struka5969992015-03-19 12:31:40 -0700570 struct scatterlist *tmp;
571 int x;
572 /* Ran out of tx slots in async request
573 * need to expand */
574 tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
575 GFP_KERNEL);
576 if (!tmp)
577 goto free;
578
579 sg_init_table(tmp, tx_nents * 2);
580 for (x = 0; x < tx_nents; x++)
581 sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]),
582 sreq->tsg[x].length,
583 sreq->tsg[x].offset);
584 kfree(sreq->tsg);
585 sreq->tsg = tmp;
586 tx_nents *= 2;
tadeusz.struk@intel.com033f46b2015-04-01 13:53:06 -0700587 mark = true;
Tadeusz Struka5969992015-03-19 12:31:40 -0700588 }
589 /* Need to take over the tx sgl from ctx
590 * to the asynch req - these sgls will be freed later */
tadeusz.struk@intel.com033f46b2015-04-01 13:53:06 -0700591 sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length,
Tadeusz Struka5969992015-03-19 12:31:40 -0700592 sg->offset);
593
594 if (list_empty(&sreq->list)) {
595 rsgl = &sreq->first_sgl;
596 list_add_tail(&rsgl->list, &sreq->list);
597 } else {
Tadeusz Struk82d92922015-03-30 14:25:44 -0700598 rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL);
Tadeusz Struka5969992015-03-19 12:31:40 -0700599 if (!rsgl) {
600 err = -ENOMEM;
601 goto free;
602 }
603 list_add_tail(&rsgl->list, &sreq->list);
604 }
605
606 used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used);
607 err = used;
608 if (used < 0)
609 goto free;
610 if (last_rsgl)
611 af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
612
613 last_rsgl = rsgl;
614 len += used;
615 skcipher_pull_sgl(sk, used, 0);
616 iov_iter_advance(&msg->msg_iter, used);
617 }
618
tadeusz.struk@intel.com033f46b2015-04-01 13:53:06 -0700619 if (mark)
620 sg_mark_end(sreq->tsg + txbufs - 1);
621
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800622 skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg,
Herbert Xuec69bbf2016-02-03 21:39:24 +0800623 len, iv);
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800624 err = ctx->enc ? crypto_skcipher_encrypt(req) :
625 crypto_skcipher_decrypt(req);
Tadeusz Struka5969992015-03-19 12:31:40 -0700626 if (err == -EINPROGRESS) {
627 atomic_inc(&ctx->inflight);
628 err = -EIOCBQUEUED;
Herbert Xuec69bbf2016-02-03 21:39:24 +0800629 sreq = NULL;
Tadeusz Struka5969992015-03-19 12:31:40 -0700630 goto unlock;
631 }
632free:
633 skcipher_free_async_sgls(sreq);
Tadeusz Struka5969992015-03-19 12:31:40 -0700634unlock:
635 skcipher_wmem_wakeup(sk);
636 release_sock(sk);
Herbert Xuec69bbf2016-02-03 21:39:24 +0800637 kzfree(sreq);
638out:
Tadeusz Struka5969992015-03-19 12:31:40 -0700639 return err;
640}
641
642static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg,
643 int flags)
Herbert Xu8ff59092010-10-19 21:31:55 +0800644{
645 struct sock *sk = sock->sk;
646 struct alg_sock *ask = alg_sk(sk);
Herbert Xu6454c2b2016-02-03 21:39:26 +0800647 struct sock *psk = ask->parent;
648 struct alg_sock *pask = alg_sk(psk);
Herbert Xu8ff59092010-10-19 21:31:55 +0800649 struct skcipher_ctx *ctx = ask->private;
Herbert Xu6454c2b2016-02-03 21:39:26 +0800650 struct skcipher_tfm *skc = pask->private;
651 struct crypto_skcipher *tfm = skc->skcipher;
652 unsigned bs = crypto_skcipher_blocksize(tfm);
Herbert Xu8ff59092010-10-19 21:31:55 +0800653 struct skcipher_sg_list *sgl;
654 struct scatterlist *sg;
Herbert Xu8ff59092010-10-19 21:31:55 +0800655 int err = -EAGAIN;
656 int used;
657 long copied = 0;
658
659 lock_sock(sk);
Al Viro01e97e62014-12-15 21:39:31 -0500660 while (msg_data_left(msg)) {
Linus Torvalds9399f0c2015-02-10 19:55:45 -0800661 if (!ctx->used) {
Al Viro1d10eb22014-11-28 16:39:25 -0500662 err = skcipher_wait_for_data(sk, flags);
663 if (err)
Herbert Xubc97e572010-11-30 17:04:31 +0800664 goto unlock;
Al Viro1d10eb22014-11-28 16:39:25 -0500665 }
Herbert Xubc97e572010-11-30 17:04:31 +0800666
Al Viro01e97e62014-12-15 21:39:31 -0500667 used = min_t(unsigned long, ctx->used, msg_data_left(msg));
Herbert Xu8ff59092010-10-19 21:31:55 +0800668
Al Viro1d10eb22014-11-28 16:39:25 -0500669 used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used);
670 err = used;
671 if (err < 0)
672 goto unlock;
Herbert Xu8ff59092010-10-19 21:31:55 +0800673
Al Viro1d10eb22014-11-28 16:39:25 -0500674 if (ctx->more || used < ctx->used)
675 used -= used % bs;
Herbert Xu8ff59092010-10-19 21:31:55 +0800676
Al Viro1d10eb22014-11-28 16:39:25 -0500677 err = -EINVAL;
678 if (!used)
679 goto free;
680
Herbert Xu4f0414e2016-01-18 18:46:10 +0800681 sgl = list_first_entry(&ctx->tsgl,
682 struct skcipher_sg_list, list);
683 sg = sgl->sg;
684
685 while (!sg->length)
686 sg++;
687
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800688 skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used,
689 ctx->iv);
Al Viro1d10eb22014-11-28 16:39:25 -0500690
691 err = af_alg_wait_for_completion(
Herbert Xu8ff59092010-10-19 21:31:55 +0800692 ctx->enc ?
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800693 crypto_skcipher_encrypt(&ctx->req) :
694 crypto_skcipher_decrypt(&ctx->req),
Herbert Xu8ff59092010-10-19 21:31:55 +0800695 &ctx->completion);
696
Herbert Xubc97e572010-11-30 17:04:31 +0800697free:
Al Viro1d10eb22014-11-28 16:39:25 -0500698 af_alg_free_sg(&ctx->rsgl);
Herbert Xu8ff59092010-10-19 21:31:55 +0800699
Al Viro1d10eb22014-11-28 16:39:25 -0500700 if (err)
701 goto unlock;
Herbert Xu8ff59092010-10-19 21:31:55 +0800702
Al Viro1d10eb22014-11-28 16:39:25 -0500703 copied += used;
Tadeusz Struka5969992015-03-19 12:31:40 -0700704 skcipher_pull_sgl(sk, used, 1);
Al Viro1d10eb22014-11-28 16:39:25 -0500705 iov_iter_advance(&msg->msg_iter, used);
Herbert Xu8ff59092010-10-19 21:31:55 +0800706 }
707
708 err = 0;
709
710unlock:
711 skcipher_wmem_wakeup(sk);
712 release_sock(sk);
713
714 return copied ?: err;
715}
716
Tadeusz Struka5969992015-03-19 12:31:40 -0700717static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
718 size_t ignored, int flags)
719{
720 return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
721 skcipher_recvmsg_async(sock, msg, flags) :
722 skcipher_recvmsg_sync(sock, msg, flags);
723}
Herbert Xu8ff59092010-10-19 21:31:55 +0800724
725static unsigned int skcipher_poll(struct file *file, struct socket *sock,
726 poll_table *wait)
727{
728 struct sock *sk = sock->sk;
729 struct alg_sock *ask = alg_sk(sk);
730 struct skcipher_ctx *ctx = ask->private;
731 unsigned int mask;
732
733 sock_poll_wait(file, sk_sleep(sk), wait);
734 mask = 0;
735
736 if (ctx->used)
737 mask |= POLLIN | POLLRDNORM;
738
739 if (skcipher_writable(sk))
740 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
741
742 return mask;
743}
744
745static struct proto_ops algif_skcipher_ops = {
746 .family = PF_ALG,
747
748 .connect = sock_no_connect,
749 .socketpair = sock_no_socketpair,
750 .getname = sock_no_getname,
751 .ioctl = sock_no_ioctl,
752 .listen = sock_no_listen,
753 .shutdown = sock_no_shutdown,
754 .getsockopt = sock_no_getsockopt,
755 .mmap = sock_no_mmap,
756 .bind = sock_no_bind,
757 .accept = sock_no_accept,
758 .setsockopt = sock_no_setsockopt,
759
760 .release = af_alg_release,
761 .sendmsg = skcipher_sendmsg,
762 .sendpage = skcipher_sendpage,
763 .recvmsg = skcipher_recvmsg,
764 .poll = skcipher_poll,
765};
766
Herbert Xua0fa2d02016-01-04 13:36:12 +0900767static int skcipher_check_key(struct socket *sock)
768{
Herbert Xu18227932016-01-15 22:02:20 +0800769 int err = 0;
Herbert Xua0fa2d02016-01-04 13:36:12 +0900770 struct sock *psk;
771 struct alg_sock *pask;
772 struct skcipher_tfm *tfm;
773 struct sock *sk = sock->sk;
774 struct alg_sock *ask = alg_sk(sk);
775
Herbert Xu18227932016-01-15 22:02:20 +0800776 lock_sock(sk);
Herbert Xua0fa2d02016-01-04 13:36:12 +0900777 if (ask->refcnt)
Herbert Xu18227932016-01-15 22:02:20 +0800778 goto unlock_child;
Herbert Xua0fa2d02016-01-04 13:36:12 +0900779
780 psk = ask->parent;
781 pask = alg_sk(ask->parent);
782 tfm = pask->private;
783
784 err = -ENOKEY;
Herbert Xu18227932016-01-15 22:02:20 +0800785 lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
Herbert Xua0fa2d02016-01-04 13:36:12 +0900786 if (!tfm->has_key)
787 goto unlock;
788
789 if (!pask->refcnt++)
790 sock_hold(psk);
791
792 ask->refcnt = 1;
793 sock_put(psk);
794
795 err = 0;
796
797unlock:
798 release_sock(psk);
Herbert Xu18227932016-01-15 22:02:20 +0800799unlock_child:
800 release_sock(sk);
Herbert Xua0fa2d02016-01-04 13:36:12 +0900801
802 return err;
803}
804
805static int skcipher_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
806 size_t size)
807{
808 int err;
809
810 err = skcipher_check_key(sock);
811 if (err)
812 return err;
813
814 return skcipher_sendmsg(sock, msg, size);
815}
816
817static ssize_t skcipher_sendpage_nokey(struct socket *sock, struct page *page,
818 int offset, size_t size, int flags)
819{
820 int err;
821
822 err = skcipher_check_key(sock);
823 if (err)
824 return err;
825
826 return skcipher_sendpage(sock, page, offset, size, flags);
827}
828
829static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
830 size_t ignored, int flags)
831{
832 int err;
833
834 err = skcipher_check_key(sock);
835 if (err)
836 return err;
837
838 return skcipher_recvmsg(sock, msg, ignored, flags);
839}
840
841static struct proto_ops algif_skcipher_ops_nokey = {
842 .family = PF_ALG,
843
844 .connect = sock_no_connect,
845 .socketpair = sock_no_socketpair,
846 .getname = sock_no_getname,
847 .ioctl = sock_no_ioctl,
848 .listen = sock_no_listen,
849 .shutdown = sock_no_shutdown,
850 .getsockopt = sock_no_getsockopt,
851 .mmap = sock_no_mmap,
852 .bind = sock_no_bind,
853 .accept = sock_no_accept,
854 .setsockopt = sock_no_setsockopt,
855
856 .release = af_alg_release,
857 .sendmsg = skcipher_sendmsg_nokey,
858 .sendpage = skcipher_sendpage_nokey,
859 .recvmsg = skcipher_recvmsg_nokey,
860 .poll = skcipher_poll,
861};
862
Herbert Xu8ff59092010-10-19 21:31:55 +0800863static void *skcipher_bind(const char *name, u32 type, u32 mask)
864{
Herbert Xudd504582015-12-25 15:40:05 +0800865 struct skcipher_tfm *tfm;
866 struct crypto_skcipher *skcipher;
867
868 tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
869 if (!tfm)
870 return ERR_PTR(-ENOMEM);
871
872 skcipher = crypto_alloc_skcipher(name, type, mask);
873 if (IS_ERR(skcipher)) {
874 kfree(tfm);
875 return ERR_CAST(skcipher);
876 }
877
878 tfm->skcipher = skcipher;
879
880 return tfm;
Herbert Xu8ff59092010-10-19 21:31:55 +0800881}
882
883static void skcipher_release(void *private)
884{
Herbert Xudd504582015-12-25 15:40:05 +0800885 struct skcipher_tfm *tfm = private;
886
887 crypto_free_skcipher(tfm->skcipher);
888 kfree(tfm);
Herbert Xu8ff59092010-10-19 21:31:55 +0800889}
890
891static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
892{
Herbert Xudd504582015-12-25 15:40:05 +0800893 struct skcipher_tfm *tfm = private;
894 int err;
895
896 err = crypto_skcipher_setkey(tfm->skcipher, key, keylen);
897 tfm->has_key = !err;
898
899 return err;
Herbert Xu8ff59092010-10-19 21:31:55 +0800900}
901
Tadeusz Struka5969992015-03-19 12:31:40 -0700902static void skcipher_wait(struct sock *sk)
903{
904 struct alg_sock *ask = alg_sk(sk);
905 struct skcipher_ctx *ctx = ask->private;
906 int ctr = 0;
907
908 while (atomic_read(&ctx->inflight) && ctr++ < 100)
909 msleep(100);
910}
911
Herbert Xu8ff59092010-10-19 21:31:55 +0800912static void skcipher_sock_destruct(struct sock *sk)
913{
914 struct alg_sock *ask = alg_sk(sk);
915 struct skcipher_ctx *ctx = ask->private;
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800916 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req);
Herbert Xu8ff59092010-10-19 21:31:55 +0800917
Tadeusz Struka5969992015-03-19 12:31:40 -0700918 if (atomic_read(&ctx->inflight))
919 skcipher_wait(sk);
920
Herbert Xu8ff59092010-10-19 21:31:55 +0800921 skcipher_free_sgl(sk);
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800922 sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
Herbert Xu8ff59092010-10-19 21:31:55 +0800923 sock_kfree_s(sk, ctx, ctx->len);
924 af_alg_release_parent(sk);
925}
926
Herbert Xud7b65ae2016-01-13 15:01:06 +0800927static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
Herbert Xu8ff59092010-10-19 21:31:55 +0800928{
929 struct skcipher_ctx *ctx;
930 struct alg_sock *ask = alg_sk(sk);
Herbert Xudd504582015-12-25 15:40:05 +0800931 struct skcipher_tfm *tfm = private;
932 struct crypto_skcipher *skcipher = tfm->skcipher;
933 unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher);
Herbert Xu8ff59092010-10-19 21:31:55 +0800934
935 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
936 if (!ctx)
937 return -ENOMEM;
938
Herbert Xudd504582015-12-25 15:40:05 +0800939 ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(skcipher),
Herbert Xu8ff59092010-10-19 21:31:55 +0800940 GFP_KERNEL);
941 if (!ctx->iv) {
942 sock_kfree_s(sk, ctx, len);
943 return -ENOMEM;
944 }
945
Herbert Xudd504582015-12-25 15:40:05 +0800946 memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
Herbert Xu8ff59092010-10-19 21:31:55 +0800947
948 INIT_LIST_HEAD(&ctx->tsgl);
949 ctx->len = len;
950 ctx->used = 0;
951 ctx->more = 0;
952 ctx->merge = 0;
953 ctx->enc = 0;
Tadeusz Struka5969992015-03-19 12:31:40 -0700954 atomic_set(&ctx->inflight, 0);
Herbert Xu8ff59092010-10-19 21:31:55 +0800955 af_alg_init_completion(&ctx->completion);
956
957 ask->private = ctx;
958
Herbert Xudd504582015-12-25 15:40:05 +0800959 skcipher_request_set_tfm(&ctx->req, skcipher);
Herbert Xudad41992016-02-03 21:39:27 +0800960 skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP |
961 CRYPTO_TFM_REQ_MAY_BACKLOG,
Herbert Xu0d96e4b2015-12-18 19:16:57 +0800962 af_alg_complete, &ctx->completion);
Herbert Xu8ff59092010-10-19 21:31:55 +0800963
964 sk->sk_destruct = skcipher_sock_destruct;
965
966 return 0;
967}
968
Herbert Xua0fa2d02016-01-04 13:36:12 +0900969static int skcipher_accept_parent(void *private, struct sock *sk)
970{
971 struct skcipher_tfm *tfm = private;
972
Herbert Xu6e8d8ec2016-01-11 21:29:41 +0800973 if (!tfm->has_key && crypto_skcipher_has_setkey(tfm->skcipher))
Herbert Xua0fa2d02016-01-04 13:36:12 +0900974 return -ENOKEY;
975
Herbert Xud7b65ae2016-01-13 15:01:06 +0800976 return skcipher_accept_parent_nokey(private, sk);
Herbert Xua0fa2d02016-01-04 13:36:12 +0900977}
978
Herbert Xu8ff59092010-10-19 21:31:55 +0800979static const struct af_alg_type algif_type_skcipher = {
980 .bind = skcipher_bind,
981 .release = skcipher_release,
982 .setkey = skcipher_setkey,
983 .accept = skcipher_accept_parent,
Herbert Xua0fa2d02016-01-04 13:36:12 +0900984 .accept_nokey = skcipher_accept_parent_nokey,
Herbert Xu8ff59092010-10-19 21:31:55 +0800985 .ops = &algif_skcipher_ops,
Herbert Xua0fa2d02016-01-04 13:36:12 +0900986 .ops_nokey = &algif_skcipher_ops_nokey,
Herbert Xu8ff59092010-10-19 21:31:55 +0800987 .name = "skcipher",
988 .owner = THIS_MODULE
989};
990
991static int __init algif_skcipher_init(void)
992{
993 return af_alg_register_type(&algif_type_skcipher);
994}
995
996static void __exit algif_skcipher_exit(void)
997{
998 int err = af_alg_unregister_type(&algif_type_skcipher);
999 BUG_ON(err);
1000}
1001
1002module_init(algif_skcipher_init);
1003module_exit(algif_skcipher_exit);
1004MODULE_LICENSE("GPL");