blob: fa9151df36371709b221a977f8ff6c845522a34b [file] [log] [blame]
Jussi Kivilinna8280daa2011-09-26 16:47:25 +03001/*
2 * Glue Code for 3-way parallel assembler optimized version of Twofish
3 *
4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8 * CTR part based on code (crypto/ctr.c) by:
9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
24 * USA
25 *
26 */
27
28#include <linux/crypto.h>
29#include <linux/init.h>
30#include <linux/module.h>
31#include <linux/types.h>
32#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/b128ops.h>
Jussi Kivilinna81559f92011-10-18 13:33:02 +030035#include <crypto/lrw.h>
36
37#if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
38#define HAS_LRW
39#endif
Jussi Kivilinna8280daa2011-09-26 16:47:25 +030040
41/* regular block cipher functions from twofish_x86_64 module */
42asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
43 const u8 *src);
44asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
45 const u8 *src);
46
47/* 3-way parallel cipher functions */
48asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
49 const u8 *src, bool xor);
50asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
51 const u8 *src);
52
53static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
54 const u8 *src)
55{
56 __twofish_enc_blk_3way(ctx, dst, src, false);
57}
58
59static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
60 const u8 *src)
61{
62 __twofish_enc_blk_3way(ctx, dst, src, true);
63}
64
65static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
66 void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
67 void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
68{
69 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
70 unsigned int bsize = TF_BLOCK_SIZE;
71 unsigned int nbytes;
72 int err;
73
74 err = blkcipher_walk_virt(desc, walk);
75
76 while ((nbytes = walk->nbytes)) {
77 u8 *wsrc = walk->src.virt.addr;
78 u8 *wdst = walk->dst.virt.addr;
79
80 /* Process three block batch */
81 if (nbytes >= bsize * 3) {
82 do {
83 fn_3way(ctx, wdst, wsrc);
84
85 wsrc += bsize * 3;
86 wdst += bsize * 3;
87 nbytes -= bsize * 3;
88 } while (nbytes >= bsize * 3);
89
90 if (nbytes < bsize)
91 goto done;
92 }
93
94 /* Handle leftovers */
95 do {
96 fn(ctx, wdst, wsrc);
97
98 wsrc += bsize;
99 wdst += bsize;
100 nbytes -= bsize;
101 } while (nbytes >= bsize);
102
103done:
104 err = blkcipher_walk_done(desc, walk, nbytes);
105 }
106
107 return err;
108}
109
110static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
111 struct scatterlist *src, unsigned int nbytes)
112{
113 struct blkcipher_walk walk;
114
115 blkcipher_walk_init(&walk, dst, src, nbytes);
116 return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
117}
118
119static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
120 struct scatterlist *src, unsigned int nbytes)
121{
122 struct blkcipher_walk walk;
123
124 blkcipher_walk_init(&walk, dst, src, nbytes);
125 return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
126}
127
128static struct crypto_alg blk_ecb_alg = {
129 .cra_name = "ecb(twofish)",
130 .cra_driver_name = "ecb-twofish-3way",
131 .cra_priority = 300,
132 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
133 .cra_blocksize = TF_BLOCK_SIZE,
134 .cra_ctxsize = sizeof(struct twofish_ctx),
135 .cra_alignmask = 0,
136 .cra_type = &crypto_blkcipher_type,
137 .cra_module = THIS_MODULE,
138 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
139 .cra_u = {
140 .blkcipher = {
141 .min_keysize = TF_MIN_KEY_SIZE,
142 .max_keysize = TF_MAX_KEY_SIZE,
143 .setkey = twofish_setkey,
144 .encrypt = ecb_encrypt,
145 .decrypt = ecb_decrypt,
146 },
147 },
148};
149
150static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
151 struct blkcipher_walk *walk)
152{
153 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
154 unsigned int bsize = TF_BLOCK_SIZE;
155 unsigned int nbytes = walk->nbytes;
156 u128 *src = (u128 *)walk->src.virt.addr;
157 u128 *dst = (u128 *)walk->dst.virt.addr;
158 u128 *iv = (u128 *)walk->iv;
159
160 do {
161 u128_xor(dst, src, iv);
162 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
163 iv = dst;
164
165 src += 1;
166 dst += 1;
167 nbytes -= bsize;
168 } while (nbytes >= bsize);
169
170 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
171 return nbytes;
172}
173
174static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
175 struct scatterlist *src, unsigned int nbytes)
176{
177 struct blkcipher_walk walk;
178 int err;
179
180 blkcipher_walk_init(&walk, dst, src, nbytes);
181 err = blkcipher_walk_virt(desc, &walk);
182
183 while ((nbytes = walk.nbytes)) {
184 nbytes = __cbc_encrypt(desc, &walk);
185 err = blkcipher_walk_done(desc, &walk, nbytes);
186 }
187
188 return err;
189}
190
191static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
192 struct blkcipher_walk *walk)
193{
194 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
195 unsigned int bsize = TF_BLOCK_SIZE;
196 unsigned int nbytes = walk->nbytes;
197 u128 *src = (u128 *)walk->src.virt.addr;
198 u128 *dst = (u128 *)walk->dst.virt.addr;
199 u128 ivs[3 - 1];
200 u128 last_iv;
201
202 /* Start of the last block. */
203 src += nbytes / bsize - 1;
204 dst += nbytes / bsize - 1;
205
206 last_iv = *src;
207
208 /* Process three block batch */
209 if (nbytes >= bsize * 3) {
210 do {
211 nbytes -= bsize * (3 - 1);
212 src -= 3 - 1;
213 dst -= 3 - 1;
214
215 ivs[0] = src[0];
216 ivs[1] = src[1];
217
218 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
219
220 u128_xor(dst + 1, dst + 1, ivs + 0);
221 u128_xor(dst + 2, dst + 2, ivs + 1);
222
223 nbytes -= bsize;
224 if (nbytes < bsize)
225 goto done;
226
227 u128_xor(dst, dst, src - 1);
228 src -= 1;
229 dst -= 1;
230 } while (nbytes >= bsize * 3);
231
232 if (nbytes < bsize)
233 goto done;
234 }
235
236 /* Handle leftovers */
237 for (;;) {
238 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
239
240 nbytes -= bsize;
241 if (nbytes < bsize)
242 break;
243
244 u128_xor(dst, dst, src - 1);
245 src -= 1;
246 dst -= 1;
247 }
248
249done:
250 u128_xor(dst, dst, (u128 *)walk->iv);
251 *(u128 *)walk->iv = last_iv;
252
253 return nbytes;
254}
255
256static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
257 struct scatterlist *src, unsigned int nbytes)
258{
259 struct blkcipher_walk walk;
260 int err;
261
262 blkcipher_walk_init(&walk, dst, src, nbytes);
263 err = blkcipher_walk_virt(desc, &walk);
264
265 while ((nbytes = walk.nbytes)) {
266 nbytes = __cbc_decrypt(desc, &walk);
267 err = blkcipher_walk_done(desc, &walk, nbytes);
268 }
269
270 return err;
271}
272
273static struct crypto_alg blk_cbc_alg = {
274 .cra_name = "cbc(twofish)",
275 .cra_driver_name = "cbc-twofish-3way",
276 .cra_priority = 300,
277 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
278 .cra_blocksize = TF_BLOCK_SIZE,
279 .cra_ctxsize = sizeof(struct twofish_ctx),
280 .cra_alignmask = 0,
281 .cra_type = &crypto_blkcipher_type,
282 .cra_module = THIS_MODULE,
283 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
284 .cra_u = {
285 .blkcipher = {
286 .min_keysize = TF_MIN_KEY_SIZE,
287 .max_keysize = TF_MAX_KEY_SIZE,
288 .ivsize = TF_BLOCK_SIZE,
289 .setkey = twofish_setkey,
290 .encrypt = cbc_encrypt,
291 .decrypt = cbc_decrypt,
292 },
293 },
294};
295
296static inline void u128_to_be128(be128 *dst, const u128 *src)
297{
298 dst->a = cpu_to_be64(src->a);
299 dst->b = cpu_to_be64(src->b);
300}
301
302static inline void be128_to_u128(u128 *dst, const be128 *src)
303{
304 dst->a = be64_to_cpu(src->a);
305 dst->b = be64_to_cpu(src->b);
306}
307
308static inline void u128_inc(u128 *i)
309{
310 i->b++;
311 if (!i->b)
312 i->a++;
313}
314
315static void ctr_crypt_final(struct blkcipher_desc *desc,
316 struct blkcipher_walk *walk)
317{
318 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
319 u8 *ctrblk = walk->iv;
320 u8 keystream[TF_BLOCK_SIZE];
321 u8 *src = walk->src.virt.addr;
322 u8 *dst = walk->dst.virt.addr;
323 unsigned int nbytes = walk->nbytes;
324
325 twofish_enc_blk(ctx, keystream, ctrblk);
326 crypto_xor(keystream, src, nbytes);
327 memcpy(dst, keystream, nbytes);
328
329 crypto_inc(ctrblk, TF_BLOCK_SIZE);
330}
331
332static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
333 struct blkcipher_walk *walk)
334{
335 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
336 unsigned int bsize = TF_BLOCK_SIZE;
337 unsigned int nbytes = walk->nbytes;
338 u128 *src = (u128 *)walk->src.virt.addr;
339 u128 *dst = (u128 *)walk->dst.virt.addr;
340 u128 ctrblk;
341 be128 ctrblocks[3];
342
343 be128_to_u128(&ctrblk, (be128 *)walk->iv);
344
345 /* Process three block batch */
346 if (nbytes >= bsize * 3) {
347 do {
348 if (dst != src) {
349 dst[0] = src[0];
350 dst[1] = src[1];
351 dst[2] = src[2];
352 }
353
354 /* create ctrblks for parallel encrypt */
355 u128_to_be128(&ctrblocks[0], &ctrblk);
356 u128_inc(&ctrblk);
357 u128_to_be128(&ctrblocks[1], &ctrblk);
358 u128_inc(&ctrblk);
359 u128_to_be128(&ctrblocks[2], &ctrblk);
360 u128_inc(&ctrblk);
361
362 twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
363 (u8 *)ctrblocks);
364
365 src += 3;
366 dst += 3;
367 nbytes -= bsize * 3;
368 } while (nbytes >= bsize * 3);
369
370 if (nbytes < bsize)
371 goto done;
372 }
373
374 /* Handle leftovers */
375 do {
376 if (dst != src)
377 *dst = *src;
378
379 u128_to_be128(&ctrblocks[0], &ctrblk);
380 u128_inc(&ctrblk);
381
382 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
383 u128_xor(dst, dst, (u128 *)ctrblocks);
384
385 src += 1;
386 dst += 1;
387 nbytes -= bsize;
388 } while (nbytes >= bsize);
389
390done:
391 u128_to_be128((be128 *)walk->iv, &ctrblk);
392 return nbytes;
393}
394
395static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
396 struct scatterlist *src, unsigned int nbytes)
397{
398 struct blkcipher_walk walk;
399 int err;
400
401 blkcipher_walk_init(&walk, dst, src, nbytes);
402 err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
403
404 while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
405 nbytes = __ctr_crypt(desc, &walk);
406 err = blkcipher_walk_done(desc, &walk, nbytes);
407 }
408
409 if (walk.nbytes) {
410 ctr_crypt_final(desc, &walk);
411 err = blkcipher_walk_done(desc, &walk, 0);
412 }
413
414 return err;
415}
416
417static struct crypto_alg blk_ctr_alg = {
418 .cra_name = "ctr(twofish)",
419 .cra_driver_name = "ctr-twofish-3way",
420 .cra_priority = 300,
421 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
Jussi Kivilinna906b2c92011-10-10 12:33:02 +0300422 .cra_blocksize = 1,
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300423 .cra_ctxsize = sizeof(struct twofish_ctx),
424 .cra_alignmask = 0,
425 .cra_type = &crypto_blkcipher_type,
426 .cra_module = THIS_MODULE,
427 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
428 .cra_u = {
429 .blkcipher = {
430 .min_keysize = TF_MIN_KEY_SIZE,
431 .max_keysize = TF_MAX_KEY_SIZE,
432 .ivsize = TF_BLOCK_SIZE,
433 .setkey = twofish_setkey,
434 .encrypt = ctr_crypt,
435 .decrypt = ctr_crypt,
436 },
437 },
438};
439
Jussi Kivilinna81559f92011-10-18 13:33:02 +0300440#ifdef HAS_LRW
441
442static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
443{
444 const unsigned int bsize = TF_BLOCK_SIZE;
445 struct twofish_ctx *ctx = priv;
446 int i;
447
448 if (nbytes == 3 * bsize) {
449 twofish_enc_blk_3way(ctx, srcdst, srcdst);
450 return;
451 }
452
453 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
454 twofish_enc_blk(ctx, srcdst, srcdst);
455}
456
457static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
458{
459 const unsigned int bsize = TF_BLOCK_SIZE;
460 struct twofish_ctx *ctx = priv;
461 int i;
462
463 if (nbytes == 3 * bsize) {
464 twofish_dec_blk_3way(ctx, srcdst, srcdst);
465 return;
466 }
467
468 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
469 twofish_dec_blk(ctx, srcdst, srcdst);
470}
471
472struct twofish_lrw_ctx {
473 struct lrw_table_ctx lrw_table;
474 struct twofish_ctx twofish_ctx;
475};
476
477static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
478 unsigned int keylen)
479{
480 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
481 int err;
482
483 err = __twofish_setkey(&ctx->twofish_ctx, key, keylen - TF_BLOCK_SIZE,
484 &tfm->crt_flags);
485 if (err)
486 return err;
487
488 return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
489}
490
491static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
492 struct scatterlist *src, unsigned int nbytes)
493{
494 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
495 be128 buf[3];
496 struct lrw_crypt_req req = {
497 .tbuf = buf,
498 .tbuflen = sizeof(buf),
499
500 .table_ctx = &ctx->lrw_table,
501 .crypt_ctx = &ctx->twofish_ctx,
502 .crypt_fn = encrypt_callback,
503 };
504
505 return lrw_crypt(desc, dst, src, nbytes, &req);
506}
507
508static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
509 struct scatterlist *src, unsigned int nbytes)
510{
511 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
512 be128 buf[3];
513 struct lrw_crypt_req req = {
514 .tbuf = buf,
515 .tbuflen = sizeof(buf),
516
517 .table_ctx = &ctx->lrw_table,
518 .crypt_ctx = &ctx->twofish_ctx,
519 .crypt_fn = decrypt_callback,
520 };
521
522 return lrw_crypt(desc, dst, src, nbytes, &req);
523}
524
525static void lrw_exit_tfm(struct crypto_tfm *tfm)
526{
527 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
528
529 lrw_free_table(&ctx->lrw_table);
530}
531
532static struct crypto_alg blk_lrw_alg = {
533 .cra_name = "lrw(twofish)",
534 .cra_driver_name = "lrw-twofish-3way",
535 .cra_priority = 300,
536 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
537 .cra_blocksize = TF_BLOCK_SIZE,
538 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
539 .cra_alignmask = 0,
540 .cra_type = &crypto_blkcipher_type,
541 .cra_module = THIS_MODULE,
542 .cra_list = LIST_HEAD_INIT(blk_lrw_alg.cra_list),
543 .cra_exit = lrw_exit_tfm,
544 .cra_u = {
545 .blkcipher = {
546 .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
547 .max_keysize = TF_MAX_KEY_SIZE + TF_BLOCK_SIZE,
548 .ivsize = TF_BLOCK_SIZE,
549 .setkey = lrw_twofish_setkey,
550 .encrypt = lrw_encrypt,
551 .decrypt = lrw_decrypt,
552 },
553 },
554};
555
556#endif
557
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300558int __init init(void)
559{
560 int err;
561
562 err = crypto_register_alg(&blk_ecb_alg);
563 if (err)
564 goto ecb_err;
565 err = crypto_register_alg(&blk_cbc_alg);
566 if (err)
567 goto cbc_err;
568 err = crypto_register_alg(&blk_ctr_alg);
569 if (err)
570 goto ctr_err;
Jussi Kivilinna81559f92011-10-18 13:33:02 +0300571#ifdef HAS_LRW
572 err = crypto_register_alg(&blk_lrw_alg);
573 if (err)
574 goto blk_lrw_err;
575#endif
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300576
577 return 0;
578
Jussi Kivilinna81559f92011-10-18 13:33:02 +0300579#ifdef HAS_LRW
580blk_lrw_err:
581 crypto_unregister_alg(&blk_ctr_alg);
582#endif
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300583ctr_err:
584 crypto_unregister_alg(&blk_cbc_alg);
585cbc_err:
586 crypto_unregister_alg(&blk_ecb_alg);
587ecb_err:
588 return err;
589}
590
591void __exit fini(void)
592{
Jussi Kivilinna81559f92011-10-18 13:33:02 +0300593#ifdef HAS_LRW
594 crypto_unregister_alg(&blk_lrw_alg);
595#endif
Jussi Kivilinna8280daa2011-09-26 16:47:25 +0300596 crypto_unregister_alg(&blk_ctr_alg);
597 crypto_unregister_alg(&blk_cbc_alg);
598 crypto_unregister_alg(&blk_ecb_alg);
599}
600
601module_init(init);
602module_exit(fini);
603
604MODULE_LICENSE("GPL");
605MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
606MODULE_ALIAS("twofish");
607MODULE_ALIAS("twofish-asm");