blob: 96d842a13ffc9ae2f28872c85fae2fa289245c80 [file] [log] [blame]
Martin Willi71ebc4d2015-06-01 13:44:00 +02001/*
2 * ChaCha20-Poly1305 AEAD, RFC7539
3 *
4 * Copyright (C) 2015 Martin Willi
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <crypto/internal/aead.h>
13#include <crypto/internal/hash.h>
14#include <crypto/internal/skcipher.h>
15#include <crypto/scatterwalk.h>
Martin Willi31d72472015-07-16 19:14:00 +020016#include <crypto/chacha20.h>
Martin Willi2546f812015-07-16 19:14:05 +020017#include <crypto/poly1305.h>
Martin Willi71ebc4d2015-06-01 13:44:00 +020018#include <linux/err.h>
19#include <linux/init.h>
20#include <linux/kernel.h>
21#include <linux/module.h>
22
23#include "internal.h"
24
Martin Willi71ebc4d2015-06-01 13:44:00 +020025#define CHACHAPOLY_IV_SIZE 12
26
27struct chachapoly_instance_ctx {
28 struct crypto_skcipher_spawn chacha;
29 struct crypto_ahash_spawn poly;
30 unsigned int saltlen;
31};
32
33struct chachapoly_ctx {
Herbert Xu1e1f0062016-07-12 13:17:40 +080034 struct crypto_skcipher *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +020035 struct crypto_ahash *poly;
36 /* key bytes we use for the ChaCha20 IV */
37 unsigned int saltlen;
38 u8 salt[];
39};
40
41struct poly_req {
42 /* zero byte padding for AD/ciphertext, as needed */
43 u8 pad[POLY1305_BLOCK_SIZE];
44 /* tail data with AD/ciphertext lengths */
45 struct {
46 __le64 assoclen;
47 __le64 cryptlen;
48 } tail;
49 struct scatterlist src[1];
50 struct ahash_request req; /* must be last member */
51};
52
53struct chacha_req {
Martin Willi71ebc4d2015-06-01 13:44:00 +020054 u8 iv[CHACHA20_IV_SIZE];
55 struct scatterlist src[1];
Herbert Xu1e1f0062016-07-12 13:17:40 +080056 struct skcipher_request req; /* must be last member */
Martin Willi71ebc4d2015-06-01 13:44:00 +020057};
58
59struct chachapoly_req_ctx {
Herbert Xu74790922015-07-16 12:35:08 +080060 struct scatterlist src[2];
61 struct scatterlist dst[2];
Martin Willic2b7b20a2015-06-16 11:34:16 +020062 /* the key we generate for Poly1305 using Chacha20 */
63 u8 key[POLY1305_KEY_SIZE];
Martin Willi71ebc4d2015-06-01 13:44:00 +020064 /* calculated Poly1305 tag */
65 u8 tag[POLY1305_DIGEST_SIZE];
66 /* length of data to en/decrypt, without ICV */
67 unsigned int cryptlen;
Herbert Xu74790922015-07-16 12:35:08 +080068 /* Actual AD, excluding IV */
69 unsigned int assoclen;
Eric Biggers16102fb2019-05-31 11:12:30 -070070 /* request flags, with MAY_SLEEP cleared if needed */
71 u32 flags;
Martin Willi71ebc4d2015-06-01 13:44:00 +020072 union {
73 struct poly_req poly;
74 struct chacha_req chacha;
75 } u;
76};
77
78static inline void async_done_continue(struct aead_request *req, int err,
79 int (*cont)(struct aead_request *))
80{
Eric Biggers16102fb2019-05-31 11:12:30 -070081 if (!err) {
82 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
83
84 rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
Martin Willi71ebc4d2015-06-01 13:44:00 +020085 err = cont(req);
Eric Biggers16102fb2019-05-31 11:12:30 -070086 }
Martin Willi71ebc4d2015-06-01 13:44:00 +020087
88 if (err != -EINPROGRESS && err != -EBUSY)
89 aead_request_complete(req, err);
90}
91
92static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)
93{
94 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
95 __le32 leicb = cpu_to_le32(icb);
96
97 memcpy(iv, &leicb, sizeof(leicb));
98 memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen);
99 memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv,
100 CHACHA20_IV_SIZE - sizeof(leicb) - ctx->saltlen);
101}
102
103static int poly_verify_tag(struct aead_request *req)
104{
105 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
106 u8 tag[sizeof(rctx->tag)];
107
Herbert Xu74790922015-07-16 12:35:08 +0800108 scatterwalk_map_and_copy(tag, req->src,
109 req->assoclen + rctx->cryptlen,
110 sizeof(tag), 0);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200111 if (crypto_memneq(tag, rctx->tag, sizeof(tag)))
112 return -EBADMSG;
113 return 0;
114}
115
116static int poly_copy_tag(struct aead_request *req)
117{
118 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
119
Herbert Xu74790922015-07-16 12:35:08 +0800120 scatterwalk_map_and_copy(rctx->tag, req->dst,
121 req->assoclen + rctx->cryptlen,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200122 sizeof(rctx->tag), 1);
123 return 0;
124}
125
126static void chacha_decrypt_done(struct crypto_async_request *areq, int err)
127{
128 async_done_continue(areq->data, err, poly_verify_tag);
129}
130
131static int chacha_decrypt(struct aead_request *req)
132{
133 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
134 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
135 struct chacha_req *creq = &rctx->u.chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800136 struct scatterlist *src, *dst;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200137 int err;
138
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100139 if (rctx->cryptlen == 0)
140 goto skip;
141
Martin Willi71ebc4d2015-06-01 13:44:00 +0200142 chacha_iv(creq->iv, req, 1);
143
Herbert Xu74790922015-07-16 12:35:08 +0800144 sg_init_table(rctx->src, 2);
145 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
146 dst = src;
147
148 if (req->src != req->dst) {
149 sg_init_table(rctx->dst, 2);
150 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
151 }
152
Eric Biggers16102fb2019-05-31 11:12:30 -0700153 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800154 chacha_decrypt_done, req);
155 skcipher_request_set_tfm(&creq->req, ctx->chacha);
156 skcipher_request_set_crypt(&creq->req, src, dst,
157 rctx->cryptlen, creq->iv);
158 err = crypto_skcipher_decrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200159 if (err)
160 return err;
161
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100162skip:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200163 return poly_verify_tag(req);
164}
165
166static int poly_tail_continue(struct aead_request *req)
167{
168 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
169
170 if (rctx->cryptlen == req->cryptlen) /* encrypting */
171 return poly_copy_tag(req);
172
173 return chacha_decrypt(req);
174}
175
176static void poly_tail_done(struct crypto_async_request *areq, int err)
177{
178 async_done_continue(areq->data, err, poly_tail_continue);
179}
180
181static int poly_tail(struct aead_request *req)
182{
Herbert Xu74790922015-07-16 12:35:08 +0800183 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
184 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200185 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
186 struct poly_req *preq = &rctx->u.poly;
187 __le64 len;
188 int err;
189
190 sg_init_table(preq->src, 1);
Herbert Xu74790922015-07-16 12:35:08 +0800191 len = cpu_to_le64(rctx->assoclen);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200192 memcpy(&preq->tail.assoclen, &len, sizeof(len));
193 len = cpu_to_le64(rctx->cryptlen);
194 memcpy(&preq->tail.cryptlen, &len, sizeof(len));
195 sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail));
196
Eric Biggers16102fb2019-05-31 11:12:30 -0700197 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200198 poly_tail_done, req);
199 ahash_request_set_tfm(&preq->req, ctx->poly);
200 ahash_request_set_crypt(&preq->req, preq->src,
201 rctx->tag, sizeof(preq->tail));
202
203 err = crypto_ahash_finup(&preq->req);
204 if (err)
205 return err;
206
207 return poly_tail_continue(req);
208}
209
210static void poly_cipherpad_done(struct crypto_async_request *areq, int err)
211{
212 async_done_continue(areq->data, err, poly_tail);
213}
214
215static int poly_cipherpad(struct aead_request *req)
216{
217 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
218 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
219 struct poly_req *preq = &rctx->u.poly;
220 unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
221 int err;
222
223 padlen = (bs - (rctx->cryptlen % bs)) % bs;
224 memset(preq->pad, 0, sizeof(preq->pad));
225 sg_init_table(preq->src, 1);
226 sg_set_buf(preq->src, &preq->pad, padlen);
227
Eric Biggers16102fb2019-05-31 11:12:30 -0700228 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200229 poly_cipherpad_done, req);
230 ahash_request_set_tfm(&preq->req, ctx->poly);
231 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
232
233 err = crypto_ahash_update(&preq->req);
234 if (err)
235 return err;
236
237 return poly_tail(req);
238}
239
240static void poly_cipher_done(struct crypto_async_request *areq, int err)
241{
242 async_done_continue(areq->data, err, poly_cipherpad);
243}
244
245static int poly_cipher(struct aead_request *req)
246{
247 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
248 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
249 struct poly_req *preq = &rctx->u.poly;
250 struct scatterlist *crypt = req->src;
251 int err;
252
253 if (rctx->cryptlen == req->cryptlen) /* encrypting */
254 crypt = req->dst;
255
Herbert Xu74790922015-07-16 12:35:08 +0800256 sg_init_table(rctx->src, 2);
257 crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);
258
Eric Biggers16102fb2019-05-31 11:12:30 -0700259 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200260 poly_cipher_done, req);
261 ahash_request_set_tfm(&preq->req, ctx->poly);
262 ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
263
264 err = crypto_ahash_update(&preq->req);
265 if (err)
266 return err;
267
268 return poly_cipherpad(req);
269}
270
271static void poly_adpad_done(struct crypto_async_request *areq, int err)
272{
273 async_done_continue(areq->data, err, poly_cipher);
274}
275
276static int poly_adpad(struct aead_request *req)
277{
278 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
279 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
280 struct poly_req *preq = &rctx->u.poly;
281 unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
282 int err;
283
Herbert Xu74790922015-07-16 12:35:08 +0800284 padlen = (bs - (rctx->assoclen % bs)) % bs;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200285 memset(preq->pad, 0, sizeof(preq->pad));
286 sg_init_table(preq->src, 1);
287 sg_set_buf(preq->src, preq->pad, padlen);
288
Eric Biggers16102fb2019-05-31 11:12:30 -0700289 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200290 poly_adpad_done, req);
291 ahash_request_set_tfm(&preq->req, ctx->poly);
292 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
293
294 err = crypto_ahash_update(&preq->req);
295 if (err)
296 return err;
297
298 return poly_cipher(req);
299}
300
301static void poly_ad_done(struct crypto_async_request *areq, int err)
302{
303 async_done_continue(areq->data, err, poly_adpad);
304}
305
306static int poly_ad(struct aead_request *req)
307{
308 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
309 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
310 struct poly_req *preq = &rctx->u.poly;
311 int err;
312
Eric Biggers16102fb2019-05-31 11:12:30 -0700313 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200314 poly_ad_done, req);
315 ahash_request_set_tfm(&preq->req, ctx->poly);
Herbert Xu74790922015-07-16 12:35:08 +0800316 ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200317
318 err = crypto_ahash_update(&preq->req);
319 if (err)
320 return err;
321
322 return poly_adpad(req);
323}
324
Martin Willic2b7b20a2015-06-16 11:34:16 +0200325static void poly_setkey_done(struct crypto_async_request *areq, int err)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200326{
327 async_done_continue(areq->data, err, poly_ad);
328}
329
Martin Willic2b7b20a2015-06-16 11:34:16 +0200330static int poly_setkey(struct aead_request *req)
331{
332 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
333 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
334 struct poly_req *preq = &rctx->u.poly;
335 int err;
336
337 sg_init_table(preq->src, 1);
338 sg_set_buf(preq->src, rctx->key, sizeof(rctx->key));
339
Eric Biggers16102fb2019-05-31 11:12:30 -0700340 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willic2b7b20a2015-06-16 11:34:16 +0200341 poly_setkey_done, req);
342 ahash_request_set_tfm(&preq->req, ctx->poly);
343 ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
344
345 err = crypto_ahash_update(&preq->req);
346 if (err)
347 return err;
348
349 return poly_ad(req);
350}
351
352static void poly_init_done(struct crypto_async_request *areq, int err)
353{
354 async_done_continue(areq->data, err, poly_setkey);
355}
356
Martin Willi71ebc4d2015-06-01 13:44:00 +0200357static int poly_init(struct aead_request *req)
358{
359 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
360 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
361 struct poly_req *preq = &rctx->u.poly;
362 int err;
363
Eric Biggers16102fb2019-05-31 11:12:30 -0700364 ahash_request_set_callback(&preq->req, rctx->flags,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200365 poly_init_done, req);
366 ahash_request_set_tfm(&preq->req, ctx->poly);
367
368 err = crypto_ahash_init(&preq->req);
369 if (err)
370 return err;
371
Martin Willic2b7b20a2015-06-16 11:34:16 +0200372 return poly_setkey(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200373}
374
375static void poly_genkey_done(struct crypto_async_request *areq, int err)
376{
Martin Willic2b7b20a2015-06-16 11:34:16 +0200377 async_done_continue(areq->data, err, poly_init);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200378}
379
380static int poly_genkey(struct aead_request *req)
381{
Herbert Xu74790922015-07-16 12:35:08 +0800382 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
383 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200384 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
385 struct chacha_req *creq = &rctx->u.chacha;
386 int err;
387
Herbert Xu74790922015-07-16 12:35:08 +0800388 rctx->assoclen = req->assoclen;
389
390 if (crypto_aead_ivsize(tfm) == 8) {
391 if (rctx->assoclen < 8)
392 return -EINVAL;
393 rctx->assoclen -= 8;
394 }
395
Martin Willi71ebc4d2015-06-01 13:44:00 +0200396 sg_init_table(creq->src, 1);
Martin Willic2b7b20a2015-06-16 11:34:16 +0200397 memset(rctx->key, 0, sizeof(rctx->key));
398 sg_set_buf(creq->src, rctx->key, sizeof(rctx->key));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200399
400 chacha_iv(creq->iv, req, 0);
401
Eric Biggers16102fb2019-05-31 11:12:30 -0700402 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800403 poly_genkey_done, req);
404 skcipher_request_set_tfm(&creq->req, ctx->chacha);
405 skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
406 POLY1305_KEY_SIZE, creq->iv);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200407
Herbert Xu1e1f0062016-07-12 13:17:40 +0800408 err = crypto_skcipher_decrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200409 if (err)
410 return err;
411
Martin Willic2b7b20a2015-06-16 11:34:16 +0200412 return poly_init(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200413}
414
415static void chacha_encrypt_done(struct crypto_async_request *areq, int err)
416{
417 async_done_continue(areq->data, err, poly_genkey);
418}
419
420static int chacha_encrypt(struct aead_request *req)
421{
422 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
423 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
424 struct chacha_req *creq = &rctx->u.chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800425 struct scatterlist *src, *dst;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200426 int err;
427
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100428 if (req->cryptlen == 0)
429 goto skip;
430
Martin Willi71ebc4d2015-06-01 13:44:00 +0200431 chacha_iv(creq->iv, req, 1);
432
Herbert Xu74790922015-07-16 12:35:08 +0800433 sg_init_table(rctx->src, 2);
434 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
435 dst = src;
436
437 if (req->src != req->dst) {
438 sg_init_table(rctx->dst, 2);
439 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
440 }
441
Eric Biggers16102fb2019-05-31 11:12:30 -0700442 skcipher_request_set_callback(&creq->req, rctx->flags,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800443 chacha_encrypt_done, req);
444 skcipher_request_set_tfm(&creq->req, ctx->chacha);
445 skcipher_request_set_crypt(&creq->req, src, dst,
446 req->cryptlen, creq->iv);
447 err = crypto_skcipher_encrypt(&creq->req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200448 if (err)
449 return err;
450
Jason A. Donenfeld161151d2015-12-06 02:51:38 +0100451skip:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200452 return poly_genkey(req);
453}
454
455static int chachapoly_encrypt(struct aead_request *req)
456{
457 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
458
459 rctx->cryptlen = req->cryptlen;
Eric Biggers16102fb2019-05-31 11:12:30 -0700460 rctx->flags = aead_request_flags(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200461
462 /* encrypt call chain:
463 * - chacha_encrypt/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200464 * - poly_genkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200465 * - poly_init/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200466 * - poly_setkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200467 * - poly_ad/done()
468 * - poly_adpad/done()
469 * - poly_cipher/done()
470 * - poly_cipherpad/done()
471 * - poly_tail/done/continue()
472 * - poly_copy_tag()
473 */
474 return chacha_encrypt(req);
475}
476
477static int chachapoly_decrypt(struct aead_request *req)
478{
479 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
480
Martin Willi71ebc4d2015-06-01 13:44:00 +0200481 rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
Eric Biggers16102fb2019-05-31 11:12:30 -0700482 rctx->flags = aead_request_flags(req);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200483
484 /* decrypt call chain:
Martin Willic2b7b20a2015-06-16 11:34:16 +0200485 * - poly_genkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200486 * - poly_init/done()
Martin Willic2b7b20a2015-06-16 11:34:16 +0200487 * - poly_setkey/done()
Martin Willi71ebc4d2015-06-01 13:44:00 +0200488 * - poly_ad/done()
489 * - poly_adpad/done()
490 * - poly_cipher/done()
491 * - poly_cipherpad/done()
492 * - poly_tail/done/continue()
493 * - chacha_decrypt/done()
494 * - poly_verify_tag()
495 */
496 return poly_genkey(req);
497}
498
499static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
500 unsigned int keylen)
501{
502 struct chachapoly_ctx *ctx = crypto_aead_ctx(aead);
503 int err;
504
505 if (keylen != ctx->saltlen + CHACHA20_KEY_SIZE)
506 return -EINVAL;
507
508 keylen -= ctx->saltlen;
509 memcpy(ctx->salt, key + keylen, ctx->saltlen);
510
Herbert Xu1e1f0062016-07-12 13:17:40 +0800511 crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
512 crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
513 CRYPTO_TFM_REQ_MASK);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200514
Herbert Xu1e1f0062016-07-12 13:17:40 +0800515 err = crypto_skcipher_setkey(ctx->chacha, key, keylen);
516 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctx->chacha) &
517 CRYPTO_TFM_RES_MASK);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200518 return err;
519}
520
521static int chachapoly_setauthsize(struct crypto_aead *tfm,
522 unsigned int authsize)
523{
524 if (authsize != POLY1305_DIGEST_SIZE)
525 return -EINVAL;
526
527 return 0;
528}
529
Herbert Xu74790922015-07-16 12:35:08 +0800530static int chachapoly_init(struct crypto_aead *tfm)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200531{
Herbert Xu74790922015-07-16 12:35:08 +0800532 struct aead_instance *inst = aead_alg_instance(tfm);
533 struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);
534 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu1e1f0062016-07-12 13:17:40 +0800535 struct crypto_skcipher *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200536 struct crypto_ahash *poly;
537 unsigned long align;
538
539 poly = crypto_spawn_ahash(&ictx->poly);
540 if (IS_ERR(poly))
541 return PTR_ERR(poly);
542
Herbert Xu1e1f0062016-07-12 13:17:40 +0800543 chacha = crypto_spawn_skcipher2(&ictx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200544 if (IS_ERR(chacha)) {
545 crypto_free_ahash(poly);
546 return PTR_ERR(chacha);
547 }
548
549 ctx->chacha = chacha;
550 ctx->poly = poly;
551 ctx->saltlen = ictx->saltlen;
552
Herbert Xu74790922015-07-16 12:35:08 +0800553 align = crypto_aead_alignmask(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200554 align &= ~(crypto_tfm_ctx_alignment() - 1);
Herbert Xu74790922015-07-16 12:35:08 +0800555 crypto_aead_set_reqsize(
556 tfm,
557 align + offsetof(struct chachapoly_req_ctx, u) +
558 max(offsetof(struct chacha_req, req) +
Herbert Xu1e1f0062016-07-12 13:17:40 +0800559 sizeof(struct skcipher_request) +
560 crypto_skcipher_reqsize(chacha),
Herbert Xu74790922015-07-16 12:35:08 +0800561 offsetof(struct poly_req, req) +
562 sizeof(struct ahash_request) +
563 crypto_ahash_reqsize(poly)));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200564
565 return 0;
566}
567
Herbert Xu74790922015-07-16 12:35:08 +0800568static void chachapoly_exit(struct crypto_aead *tfm)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200569{
Herbert Xu74790922015-07-16 12:35:08 +0800570 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200571
572 crypto_free_ahash(ctx->poly);
Herbert Xu1e1f0062016-07-12 13:17:40 +0800573 crypto_free_skcipher(ctx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200574}
575
Herbert Xu74790922015-07-16 12:35:08 +0800576static void chachapoly_free(struct aead_instance *inst)
577{
578 struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst);
579
580 crypto_drop_skcipher(&ctx->chacha);
581 crypto_drop_ahash(&ctx->poly);
582 kfree(inst);
583}
584
585static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
586 const char *name, unsigned int ivsize)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200587{
588 struct crypto_attr_type *algt;
Herbert Xu74790922015-07-16 12:35:08 +0800589 struct aead_instance *inst;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800590 struct skcipher_alg *chacha;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200591 struct crypto_alg *poly;
Herbert Xu74790922015-07-16 12:35:08 +0800592 struct hash_alg_common *poly_hash;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200593 struct chachapoly_instance_ctx *ctx;
594 const char *chacha_name, *poly_name;
595 int err;
596
597 if (ivsize > CHACHAPOLY_IV_SIZE)
Herbert Xu74790922015-07-16 12:35:08 +0800598 return -EINVAL;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200599
600 algt = crypto_get_attr_type(tb);
601 if (IS_ERR(algt))
Herbert Xu74790922015-07-16 12:35:08 +0800602 return PTR_ERR(algt);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200603
Herbert Xu5e4b8c12015-08-13 17:29:06 +0800604 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
Herbert Xu74790922015-07-16 12:35:08 +0800605 return -EINVAL;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200606
607 chacha_name = crypto_attr_alg_name(tb[1]);
608 if (IS_ERR(chacha_name))
Herbert Xu74790922015-07-16 12:35:08 +0800609 return PTR_ERR(chacha_name);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200610 poly_name = crypto_attr_alg_name(tb[2]);
611 if (IS_ERR(poly_name))
Herbert Xu74790922015-07-16 12:35:08 +0800612 return PTR_ERR(poly_name);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200613
614 poly = crypto_find_alg(poly_name, &crypto_ahash_type,
615 CRYPTO_ALG_TYPE_HASH,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800616 CRYPTO_ALG_TYPE_AHASH_MASK |
617 crypto_requires_sync(algt->type,
618 algt->mask));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200619 if (IS_ERR(poly))
Herbert Xu74790922015-07-16 12:35:08 +0800620 return PTR_ERR(poly);
Eric Biggers868f50b2017-12-11 12:15:17 -0800621 poly_hash = __crypto_hash_alg_common(poly);
622
623 err = -EINVAL;
624 if (poly_hash->digestsize != POLY1305_DIGEST_SIZE)
625 goto out_put_poly;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200626
627 err = -ENOMEM;
628 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
629 if (!inst)
630 goto out_put_poly;
631
Herbert Xu74790922015-07-16 12:35:08 +0800632 ctx = aead_instance_ctx(inst);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200633 ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize;
Herbert Xu74790922015-07-16 12:35:08 +0800634 err = crypto_init_ahash_spawn(&ctx->poly, poly_hash,
635 aead_crypto_instance(inst));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200636 if (err)
637 goto err_free_inst;
638
Herbert Xu74790922015-07-16 12:35:08 +0800639 crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
Herbert Xu1e1f0062016-07-12 13:17:40 +0800640 err = crypto_grab_skcipher2(&ctx->chacha, chacha_name, 0,
641 crypto_requires_sync(algt->type,
642 algt->mask));
Martin Willi71ebc4d2015-06-01 13:44:00 +0200643 if (err)
644 goto err_drop_poly;
645
Herbert Xu1e1f0062016-07-12 13:17:40 +0800646 chacha = crypto_spawn_skcipher_alg(&ctx->chacha);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200647
648 err = -EINVAL;
649 /* Need 16-byte IV size, including Initial Block Counter value */
Herbert Xu1e1f0062016-07-12 13:17:40 +0800650 if (crypto_skcipher_alg_ivsize(chacha) != CHACHA20_IV_SIZE)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200651 goto out_drop_chacha;
652 /* Not a stream cipher? */
Herbert Xu1e1f0062016-07-12 13:17:40 +0800653 if (chacha->base.cra_blocksize != 1)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200654 goto out_drop_chacha;
655
656 err = -ENAMETOOLONG;
Herbert Xu74790922015-07-16 12:35:08 +0800657 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
Eric Biggersfa2876c2019-03-31 13:04:16 -0700658 "%s(%s,%s)", name, chacha->base.cra_name,
659 poly->cra_name) >= CRYPTO_MAX_ALG_NAME)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200660 goto out_drop_chacha;
Herbert Xu74790922015-07-16 12:35:08 +0800661 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
Herbert Xu1e1f0062016-07-12 13:17:40 +0800662 "%s(%s,%s)", name, chacha->base.cra_driver_name,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200663 poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
664 goto out_drop_chacha;
665
Herbert Xu1e1f0062016-07-12 13:17:40 +0800666 inst->alg.base.cra_flags = (chacha->base.cra_flags | poly->cra_flags) &
Herbert Xu74790922015-07-16 12:35:08 +0800667 CRYPTO_ALG_ASYNC;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800668 inst->alg.base.cra_priority = (chacha->base.cra_priority +
Herbert Xu74790922015-07-16 12:35:08 +0800669 poly->cra_priority) / 2;
670 inst->alg.base.cra_blocksize = 1;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800671 inst->alg.base.cra_alignmask = chacha->base.cra_alignmask |
Herbert Xu74790922015-07-16 12:35:08 +0800672 poly->cra_alignmask;
673 inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +
674 ctx->saltlen;
675 inst->alg.ivsize = ivsize;
Herbert Xu1e1f0062016-07-12 13:17:40 +0800676 inst->alg.chunksize = crypto_skcipher_alg_chunksize(chacha);
Herbert Xu74790922015-07-16 12:35:08 +0800677 inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;
678 inst->alg.init = chachapoly_init;
679 inst->alg.exit = chachapoly_exit;
680 inst->alg.encrypt = chachapoly_encrypt;
681 inst->alg.decrypt = chachapoly_decrypt;
682 inst->alg.setkey = chachapoly_setkey;
683 inst->alg.setauthsize = chachapoly_setauthsize;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200684
Herbert Xu74790922015-07-16 12:35:08 +0800685 inst->free = chachapoly_free;
686
687 err = aead_register_instance(tmpl, inst);
688 if (err)
689 goto out_drop_chacha;
690
691out_put_poly:
Martin Willi71ebc4d2015-06-01 13:44:00 +0200692 crypto_mod_put(poly);
Herbert Xu74790922015-07-16 12:35:08 +0800693 return err;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200694
695out_drop_chacha:
696 crypto_drop_skcipher(&ctx->chacha);
697err_drop_poly:
698 crypto_drop_ahash(&ctx->poly);
699err_free_inst:
700 kfree(inst);
Herbert Xu74790922015-07-16 12:35:08 +0800701 goto out_put_poly;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200702}
703
Herbert Xu74790922015-07-16 12:35:08 +0800704static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb)
Martin Willi71ebc4d2015-06-01 13:44:00 +0200705{
Herbert Xu74790922015-07-16 12:35:08 +0800706 return chachapoly_create(tmpl, tb, "rfc7539", 12);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200707}
708
Herbert Xu74790922015-07-16 12:35:08 +0800709static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb)
Martin Willi4db4ad22015-06-01 13:44:02 +0200710{
Herbert Xu74790922015-07-16 12:35:08 +0800711 return chachapoly_create(tmpl, tb, "rfc7539esp", 8);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200712}
713
714static struct crypto_template rfc7539_tmpl = {
715 .name = "rfc7539",
Herbert Xu74790922015-07-16 12:35:08 +0800716 .create = rfc7539_create,
Martin Willi71ebc4d2015-06-01 13:44:00 +0200717 .module = THIS_MODULE,
718};
719
Martin Willi4db4ad22015-06-01 13:44:02 +0200720static struct crypto_template rfc7539esp_tmpl = {
721 .name = "rfc7539esp",
Herbert Xu74790922015-07-16 12:35:08 +0800722 .create = rfc7539esp_create,
Martin Willi4db4ad22015-06-01 13:44:02 +0200723 .module = THIS_MODULE,
724};
725
Martin Willi71ebc4d2015-06-01 13:44:00 +0200726static int __init chacha20poly1305_module_init(void)
727{
Martin Willi4db4ad22015-06-01 13:44:02 +0200728 int err;
729
730 err = crypto_register_template(&rfc7539_tmpl);
731 if (err)
732 return err;
733
734 err = crypto_register_template(&rfc7539esp_tmpl);
735 if (err)
736 crypto_unregister_template(&rfc7539_tmpl);
737
738 return err;
Martin Willi71ebc4d2015-06-01 13:44:00 +0200739}
740
741static void __exit chacha20poly1305_module_exit(void)
742{
Martin Willi4db4ad22015-06-01 13:44:02 +0200743 crypto_unregister_template(&rfc7539esp_tmpl);
Martin Willi71ebc4d2015-06-01 13:44:00 +0200744 crypto_unregister_template(&rfc7539_tmpl);
745}
746
747module_init(chacha20poly1305_module_init);
748module_exit(chacha20poly1305_module_exit);
749
750MODULE_LICENSE("GPL");
751MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
752MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD");
Martin Willi71ebc4d2015-06-01 13:44:00 +0200753MODULE_ALIAS_CRYPTO("rfc7539");
Martin Willi4db4ad22015-06-01 13:44:02 +0200754MODULE_ALIAS_CRYPTO("rfc7539esp");