blob: 27483f361e8027cecdfa67068cc5e028d91ae0d5 [file] [log] [blame]
Mikko Herranen28db8e32007-11-26 22:24:11 +08001/*
2 * GCM: Galois/Counter Mode.
3 *
4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <crypto/algapi.h>
12#include <crypto/gf128mul.h>
Herbert Xu42c271c2007-12-07 18:52:49 +080013#include <crypto/scatterwalk.h>
Mikko Herranen28db8e32007-11-26 22:24:11 +080014#include <linux/err.h>
15#include <linux/init.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19
Herbert Xu42c271c2007-12-07 18:52:49 +080020#include "internal.h"
Mikko Herranen28db8e32007-11-26 22:24:11 +080021
22struct gcm_instance_ctx {
23 struct crypto_spawn ctr;
24};
25
26struct crypto_gcm_ctx {
27 struct crypto_ablkcipher *ctr;
28 struct gf128mul_4k *gf128;
29};
30
31struct crypto_gcm_ghash_ctx {
32 u32 bytes;
33 u32 flags;
34 struct gf128mul_4k *gf128;
35 u8 buffer[16];
36};
37
38struct crypto_gcm_req_priv_ctx {
39 u8 auth_tag[16];
Herbert Xu6160b282007-12-04 19:17:50 +110040 u8 iauth_tag[16];
Mikko Herranen28db8e32007-11-26 22:24:11 +080041 u8 counter[16];
42 struct crypto_gcm_ghash_ctx ghash;
43};
44
45static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
46 struct gf128mul_4k *gf128)
47{
48 ctx->bytes = 0;
49 ctx->flags = flags;
50 ctx->gf128 = gf128;
51 memset(ctx->buffer, 0, 16);
52}
53
54static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
55 const u8 *src, unsigned int srclen)
56{
57 u8 *dst = ctx->buffer;
58
59 if (ctx->bytes) {
60 int n = min(srclen, ctx->bytes);
61 u8 *pos = dst + (16 - ctx->bytes);
62
63 ctx->bytes -= n;
64 srclen -= n;
65
66 while (n--)
67 *pos++ ^= *src++;
68
69 if (!ctx->bytes)
70 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
71 }
72
73 while (srclen >= 16) {
74 crypto_xor(dst, src, 16);
75 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
76 src += 16;
77 srclen -= 16;
78 }
79
80 if (srclen) {
81 ctx->bytes = 16 - srclen;
82 while (srclen--)
83 *dst++ ^= *src++;
84 }
85}
86
87static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
88 struct scatterlist *sg, int len)
89{
90 struct scatter_walk walk;
91 u8 *src;
92 int n;
93
Herbert Xu6160b282007-12-04 19:17:50 +110094 if (!len)
95 return;
96
Mikko Herranen28db8e32007-11-26 22:24:11 +080097 scatterwalk_start(&walk, sg);
98
99 while (len) {
100 n = scatterwalk_clamp(&walk, len);
101
102 if (!n) {
103 scatterwalk_start(&walk, sg_next(walk.sg));
104 n = scatterwalk_clamp(&walk, len);
105 }
106
107 src = scatterwalk_map(&walk, 0);
108
109 crypto_gcm_ghash_update(ctx, src, n);
110 len -= n;
111
112 scatterwalk_unmap(src, 0);
113 scatterwalk_advance(&walk, n);
114 scatterwalk_done(&walk, 0, len);
115 if (len)
116 crypto_yield(ctx->flags);
117 }
118}
119
120static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
121{
122 u8 *dst = ctx->buffer;
123
124 if (ctx->bytes) {
125 u8 *tmp = dst + (16 - ctx->bytes);
126
127 while (ctx->bytes--)
128 *tmp++ ^= 0;
129
130 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
131 }
132
133 ctx->bytes = 0;
134}
135
136static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
137 unsigned int authlen,
138 unsigned int cryptlen, u8 *dst)
139{
140 u8 *buf = ctx->buffer;
141 u128 lengths;
142
143 lengths.a = cpu_to_be64(authlen * 8);
144 lengths.b = cpu_to_be64(cryptlen * 8);
145
146 crypto_gcm_ghash_flush(ctx);
147 crypto_xor(buf, (u8 *)&lengths, 16);
148 gf128mul_4k_lle((be128 *)buf, ctx->gf128);
149 crypto_xor(dst, buf, 16);
150}
151
152static inline void crypto_gcm_set_counter(u8 *counterblock, u32 value)
153{
154 *((u32 *)&counterblock[12]) = cpu_to_be32(value);
155}
156
157static int crypto_gcm_encrypt_counter(struct crypto_aead *aead, u8 *block,
158 u32 value, const u8 *iv)
159{
160 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
161 struct crypto_ablkcipher *ctr = ctx->ctr;
162 struct ablkcipher_request req;
163 struct scatterlist sg;
164 u8 counterblock[16];
165
166 if (iv == NULL)
167 memset(counterblock, 0, 12);
168 else
169 memcpy(counterblock, iv, 12);
170
171 crypto_gcm_set_counter(counterblock, value);
172
173 sg_init_one(&sg, block, 16);
174 ablkcipher_request_set_tfm(&req, ctr);
175 ablkcipher_request_set_crypt(&req, &sg, &sg, 16, counterblock);
176 ablkcipher_request_set_callback(&req, 0, NULL, NULL);
177 memset(block, 0, 16);
178 return crypto_ablkcipher_encrypt(&req);
179}
180
181static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
182 unsigned int keylen)
183{
184 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
185 struct crypto_ablkcipher *ctr = ctx->ctr;
186 int alignmask = crypto_ablkcipher_alignmask(ctr);
187 u8 alignbuf[16+alignmask];
188 u8 *hash = (u8 *)ALIGN((unsigned long)alignbuf, alignmask+1);
189 int err = 0;
190
191 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
192 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
193 CRYPTO_TFM_REQ_MASK);
194
195 err = crypto_ablkcipher_setkey(ctr, key, keylen);
196 if (err)
197 goto out;
198
199 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
200 CRYPTO_TFM_RES_MASK);
201
202 err = crypto_gcm_encrypt_counter(aead, hash, -1, NULL);
203 if (err)
204 goto out;
205
206 if (ctx->gf128 != NULL)
207 gf128mul_free_4k(ctx->gf128);
208
209 ctx->gf128 = gf128mul_init_4k_lle((be128 *)hash);
210
211 if (ctx->gf128 == NULL)
212 err = -ENOMEM;
213
214 out:
215 return err;
216}
217
218static int crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
Herbert Xu6160b282007-12-04 19:17:50 +1100219 struct aead_request *req,
220 unsigned int cryptlen,
221 void (*done)(struct crypto_async_request *,
222 int))
Mikko Herranen28db8e32007-11-26 22:24:11 +0800223{
224 struct crypto_aead *aead = crypto_aead_reqtfm(req);
225 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
226 struct crypto_gcm_req_priv_ctx *pctx = aead_request_ctx(req);
227 u32 flags = req->base.tfm->crt_flags;
228 u8 *auth_tag = pctx->auth_tag;
229 u8 *counter = pctx->counter;
230 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
231 int err = 0;
232
233 ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
234 ablkcipher_request_set_callback(ablk_req, aead_request_flags(req),
235 done, req);
236 ablkcipher_request_set_crypt(ablk_req, req->src, req->dst,
Herbert Xu6160b282007-12-04 19:17:50 +1100237 cryptlen, counter);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800238
239 err = crypto_gcm_encrypt_counter(aead, auth_tag, 0, req->iv);
240 if (err)
241 goto out;
242
243 memcpy(counter, req->iv, 12);
244 crypto_gcm_set_counter(counter, 1);
245
246 crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
247
Herbert Xu6160b282007-12-04 19:17:50 +1100248 crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
249 crypto_gcm_ghash_flush(ghash);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800250
251 out:
252 return err;
253}
254
Herbert Xu6160b282007-12-04 19:17:50 +1100255static int crypto_gcm_hash(struct aead_request *req)
Mikko Herranen28db8e32007-11-26 22:24:11 +0800256{
Herbert Xu6160b282007-12-04 19:17:50 +1100257 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800258 struct crypto_gcm_req_priv_ctx *pctx = aead_request_ctx(req);
259 u8 *auth_tag = pctx->auth_tag;
260 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
261
262 crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
263 crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
264 auth_tag);
265
Herbert Xu6160b282007-12-04 19:17:50 +1100266 scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
267 crypto_aead_authsize(aead), 1);
268 return 0;
269}
270
271static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
272{
273 struct aead_request *req = areq->data;
274
275 if (!err)
276 err = crypto_gcm_hash(req);
277
Mikko Herranen28db8e32007-11-26 22:24:11 +0800278 aead_request_complete(req, err);
279}
280
281static int crypto_gcm_encrypt(struct aead_request *req)
282{
283 struct ablkcipher_request abreq;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800284 int err = 0;
285
Herbert Xu6160b282007-12-04 19:17:50 +1100286 err = crypto_gcm_init_crypt(&abreq, req, req->cryptlen,
287 crypto_gcm_encrypt_done);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800288 if (err)
289 return err;
290
291 if (req->cryptlen) {
292 err = crypto_ablkcipher_encrypt(&abreq);
293 if (err)
294 return err;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800295 }
296
Herbert Xu6160b282007-12-04 19:17:50 +1100297 return crypto_gcm_hash(req);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800298}
299
300static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
301{
302 aead_request_complete(areq->data, err);
303}
304
305static int crypto_gcm_decrypt(struct aead_request *req)
306{
307 struct ablkcipher_request abreq;
Herbert Xu6160b282007-12-04 19:17:50 +1100308 struct crypto_aead *aead = crypto_aead_reqtfm(req);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800309 struct crypto_gcm_req_priv_ctx *pctx = aead_request_ctx(req);
310 u8 *auth_tag = pctx->auth_tag;
Herbert Xu6160b282007-12-04 19:17:50 +1100311 u8 *iauth_tag = pctx->iauth_tag;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800312 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
Herbert Xu6160b282007-12-04 19:17:50 +1100313 unsigned int cryptlen = req->cryptlen;
314 unsigned int authsize = crypto_aead_authsize(aead);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800315 int err;
316
Herbert Xu6160b282007-12-04 19:17:50 +1100317 if (cryptlen < authsize)
Mikko Herranen28db8e32007-11-26 22:24:11 +0800318 return -EINVAL;
Herbert Xu6160b282007-12-04 19:17:50 +1100319 cryptlen -= authsize;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800320
Herbert Xu6160b282007-12-04 19:17:50 +1100321 err = crypto_gcm_init_crypt(&abreq, req, cryptlen,
322 crypto_gcm_decrypt_done);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800323 if (err)
324 return err;
325
Herbert Xu6160b282007-12-04 19:17:50 +1100326 crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
327 crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
Mikko Herranen28db8e32007-11-26 22:24:11 +0800328
Herbert Xu6160b282007-12-04 19:17:50 +1100329 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
330 if (memcmp(iauth_tag, auth_tag, authsize))
Herbert Xufe70f5d2007-12-04 20:07:27 +1100331 return -EBADMSG;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800332
333 return crypto_ablkcipher_decrypt(&abreq);
334}
335
336static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
337{
338 struct crypto_instance *inst = (void *)tfm->__crt_alg;
339 struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
340 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
341 struct crypto_ablkcipher *ctr;
342 unsigned long align;
343 int err;
344
345 ctr = crypto_spawn_ablkcipher(&ictx->ctr);
346 err = PTR_ERR(ctr);
347 if (IS_ERR(ctr))
348 return err;
349
350 ctx->ctr = ctr;
351 ctx->gf128 = NULL;
352
353 align = max_t(unsigned long, crypto_ablkcipher_alignmask(ctr),
354 __alignof__(u32) - 1);
355 align &= ~(crypto_tfm_ctx_alignment() - 1);
356 tfm->crt_aead.reqsize = align + sizeof(struct crypto_gcm_req_priv_ctx);
357
358 return 0;
359}
360
361static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
362{
363 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
364
365 if (ctx->gf128 != NULL)
366 gf128mul_free_4k(ctx->gf128);
367
368 crypto_free_ablkcipher(ctx->ctr);
369}
370
371static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
372{
373 struct crypto_instance *inst;
374 struct crypto_alg *ctr;
375 struct crypto_alg *cipher;
376 struct gcm_instance_ctx *ctx;
377 int err;
378 char ctr_name[CRYPTO_MAX_ALG_NAME];
379
380 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD);
381 if (err)
382 return ERR_PTR(err);
383
384 cipher = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER,
385 CRYPTO_ALG_TYPE_MASK);
386
387 inst = ERR_PTR(PTR_ERR(cipher));
388 if (IS_ERR(cipher))
389 return inst;
390
391 inst = ERR_PTR(ENAMETOOLONG);
392 if (snprintf(
393 ctr_name, CRYPTO_MAX_ALG_NAME,
394 "ctr(%s,0,16,4)", cipher->cra_name) >= CRYPTO_MAX_ALG_NAME)
395 return inst;
396
397 ctr = crypto_alg_mod_lookup(ctr_name, CRYPTO_ALG_TYPE_BLKCIPHER,
398 CRYPTO_ALG_TYPE_MASK);
399
400 if (IS_ERR(ctr))
401 return ERR_PTR(PTR_ERR(ctr));
402
403 if (cipher->cra_blocksize != 16)
404 goto out_put_ctr;
405
406 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
407 err = -ENOMEM;
408 if (!inst)
409 goto out_put_ctr;
410
411 err = -ENAMETOOLONG;
412 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
413 "gcm(%s)", cipher->cra_name) >= CRYPTO_MAX_ALG_NAME ||
414 snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
415 "gcm(%s)", cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
416 goto err_free_inst;
417
418
419 ctx = crypto_instance_ctx(inst);
420 err = crypto_init_spawn(&ctx->ctr, ctr, inst, CRYPTO_ALG_TYPE_MASK);
421 if (err)
422 goto err_free_inst;
423
424 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
425 inst->alg.cra_priority = ctr->cra_priority;
426 inst->alg.cra_blocksize = 16;
427 inst->alg.cra_alignmask = __alignof__(u32) - 1;
428 inst->alg.cra_type = &crypto_aead_type;
429 inst->alg.cra_aead.ivsize = 12;
Herbert Xu7ba683a2007-12-02 18:49:21 +1100430 inst->alg.cra_aead.maxauthsize = 16;
Mikko Herranen28db8e32007-11-26 22:24:11 +0800431 inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
432 inst->alg.cra_init = crypto_gcm_init_tfm;
433 inst->alg.cra_exit = crypto_gcm_exit_tfm;
434 inst->alg.cra_aead.setkey = crypto_gcm_setkey;
435 inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
436 inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
437
438out:
439 crypto_mod_put(ctr);
440 return inst;
441err_free_inst:
442 kfree(inst);
443out_put_ctr:
444 inst = ERR_PTR(err);
445 goto out;
446}
447
448static void crypto_gcm_free(struct crypto_instance *inst)
449{
450 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
451
452 crypto_drop_spawn(&ctx->ctr);
453 kfree(inst);
454}
455
456static struct crypto_template crypto_gcm_tmpl = {
457 .name = "gcm",
458 .alloc = crypto_gcm_alloc,
459 .free = crypto_gcm_free,
460 .module = THIS_MODULE,
461};
462
463static int __init crypto_gcm_module_init(void)
464{
465 return crypto_register_template(&crypto_gcm_tmpl);
466}
467
468static void __exit crypto_gcm_module_exit(void)
469{
470 crypto_unregister_template(&crypto_gcm_tmpl);
471}
472
473module_init(crypto_gcm_module_init);
474module_exit(crypto_gcm_module_exit);
475
476MODULE_LICENSE("GPL");
477MODULE_DESCRIPTION("Galois/Counter Mode");
478MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");