blob: b7a3904b953c6146f543a164e6456c1893102a92 [file] [log] [blame]
Johannes Goetzfried107778b2012-05-28 15:54:24 +02001/*
2 * Glue Code for AVX assembler version of Twofish Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
Jussi Kivilinna18be4522013-04-08 21:51:00 +03007 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8 *
Johannes Goetzfried107778b2012-05-28 15:54:24 +02009 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
22 * USA
23 *
24 */
25
26#include <linux/module.h>
27#include <linux/hardirq.h>
28#include <linux/types.h>
29#include <linux/crypto.h>
30#include <linux/err.h>
Ard Biesheuvel801201a2013-09-20 09:55:41 +020031#include <crypto/ablk_helper.h>
Johannes Goetzfried107778b2012-05-28 15:54:24 +020032#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
Ingo Molnardf6b35f2015-04-24 02:46:00 +020039#include <asm/fpu/api.h>
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030040#include <asm/crypto/twofish.h>
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030041#include <asm/crypto/glue_helper.h>
Johannes Goetzfried107778b2012-05-28 15:54:24 +020042#include <crypto/scatterwalk.h>
43#include <linux/workqueue.h>
44#include <linux/spinlock.h>
45
Johannes Goetzfried107778b2012-05-28 15:54:24 +020046#define TWOFISH_PARALLEL_BLOCKS 8
47
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +030048/* 8-way parallel cipher functions */
49asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
50 const u8 *src);
51asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
52 const u8 *src);
53
54asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
55 const u8 *src);
56asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
57 const u8 *src, le128 *iv);
58
Jussi Kivilinna18be4522013-04-08 21:51:00 +030059asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
60 const u8 *src, le128 *iv);
61asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
62 const u8 *src, le128 *iv);
63
Johannes Goetzfried107778b2012-05-28 15:54:24 +020064static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
65 const u8 *src)
66{
67 __twofish_enc_blk_3way(ctx, dst, src, false);
68}
69
Jussi Kivilinna99f42f92013-06-08 12:17:47 +030070static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
Jussi Kivilinna18be4522013-04-08 21:51:00 +030071{
72 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
73 GLUE_FUNC_CAST(twofish_enc_blk));
74}
75
Jussi Kivilinna99f42f92013-06-08 12:17:47 +030076static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
Jussi Kivilinna18be4522013-04-08 21:51:00 +030077{
78 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
79 GLUE_FUNC_CAST(twofish_dec_blk));
80}
81
Johannes Goetzfried107778b2012-05-28 15:54:24 +020082
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030083static const struct common_glue_ctx twofish_enc = {
84 .num_funcs = 3,
85 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
86
87 .funcs = { {
88 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +030089 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +030090 }, {
91 .num_blocks = 3,
92 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
93 }, {
94 .num_blocks = 1,
95 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
96 } }
97};
98
99static const struct common_glue_ctx twofish_ctr = {
100 .num_funcs = 3,
101 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
102
103 .funcs = { {
104 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +0300105 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300106 }, {
107 .num_blocks = 3,
108 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
109 }, {
110 .num_blocks = 1,
111 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
112 } }
113};
114
Jussi Kivilinna18be4522013-04-08 21:51:00 +0300115static const struct common_glue_ctx twofish_enc_xts = {
116 .num_funcs = 2,
117 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
118
119 .funcs = { {
120 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
121 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
122 }, {
123 .num_blocks = 1,
124 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
125 } }
126};
127
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300128static const struct common_glue_ctx twofish_dec = {
129 .num_funcs = 3,
130 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
131
132 .funcs = { {
133 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +0300134 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300135 }, {
136 .num_blocks = 3,
137 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
138 }, {
139 .num_blocks = 1,
140 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
141 } }
142};
143
144static const struct common_glue_ctx twofish_dec_cbc = {
145 .num_funcs = 3,
146 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
147
148 .funcs = { {
149 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +0300150 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300151 }, {
152 .num_blocks = 3,
153 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
154 }, {
155 .num_blocks = 1,
156 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
157 } }
158};
159
Jussi Kivilinna18be4522013-04-08 21:51:00 +0300160static const struct common_glue_ctx twofish_dec_xts = {
161 .num_funcs = 2,
162 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
163
164 .funcs = { {
165 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
166 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
167 }, {
168 .num_blocks = 1,
169 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
170 } }
171};
172
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200173static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174 struct scatterlist *src, unsigned int nbytes)
175{
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300176 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200177}
178
179static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
180 struct scatterlist *src, unsigned int nbytes)
181{
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300182 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200183}
184
185static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
186 struct scatterlist *src, unsigned int nbytes)
187{
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300188 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
189 dst, src, nbytes);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200190}
191
192static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
193 struct scatterlist *src, unsigned int nbytes)
194{
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300195 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
196 nbytes);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200197}
198
199static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
200 struct scatterlist *src, unsigned int nbytes)
201{
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300202 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
203}
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200204
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300205static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
206{
207 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
208 fpu_enabled, nbytes);
209}
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200210
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300211static inline void twofish_fpu_end(bool fpu_enabled)
212{
213 glue_fpu_end(fpu_enabled);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200214}
215
216struct crypt_priv {
217 struct twofish_ctx *ctx;
218 bool fpu_enabled;
219};
220
221static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
222{
223 const unsigned int bsize = TF_BLOCK_SIZE;
224 struct crypt_priv *ctx = priv;
225 int i;
226
227 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
228
229 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +0300230 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200231 return;
232 }
233
234 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
235 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
236
237 nbytes %= bsize * 3;
238
239 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
240 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
241}
242
243static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
244{
245 const unsigned int bsize = TF_BLOCK_SIZE;
246 struct crypt_priv *ctx = priv;
247 int i;
248
249 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
250
251 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
Jussi Kivilinna8435a3c2012-10-20 15:06:46 +0300252 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200253 return;
254 }
255
256 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
257 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
258
259 nbytes %= bsize * 3;
260
261 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
262 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
263}
264
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200265static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
266 struct scatterlist *src, unsigned int nbytes)
267{
268 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269 be128 buf[TWOFISH_PARALLEL_BLOCKS];
270 struct crypt_priv crypt_ctx = {
271 .ctx = &ctx->twofish_ctx,
272 .fpu_enabled = false,
273 };
274 struct lrw_crypt_req req = {
275 .tbuf = buf,
276 .tbuflen = sizeof(buf),
277
278 .table_ctx = &ctx->lrw_table,
279 .crypt_ctx = &crypt_ctx,
280 .crypt_fn = encrypt_callback,
281 };
282 int ret;
283
284 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
285 ret = lrw_crypt(desc, dst, src, nbytes, &req);
286 twofish_fpu_end(crypt_ctx.fpu_enabled);
287
288 return ret;
289}
290
291static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
292 struct scatterlist *src, unsigned int nbytes)
293{
294 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
295 be128 buf[TWOFISH_PARALLEL_BLOCKS];
296 struct crypt_priv crypt_ctx = {
297 .ctx = &ctx->twofish_ctx,
298 .fpu_enabled = false,
299 };
300 struct lrw_crypt_req req = {
301 .tbuf = buf,
302 .tbuflen = sizeof(buf),
303
304 .table_ctx = &ctx->lrw_table,
305 .crypt_ctx = &crypt_ctx,
306 .crypt_fn = decrypt_callback,
307 };
308 int ret;
309
310 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
311 ret = lrw_crypt(desc, dst, src, nbytes, &req);
312 twofish_fpu_end(crypt_ctx.fpu_enabled);
313
314 return ret;
315}
316
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200317static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
318 struct scatterlist *src, unsigned int nbytes)
319{
320 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200321
Jussi Kivilinna18be4522013-04-08 21:51:00 +0300322 return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
323 XTS_TWEAK_CAST(twofish_enc_blk),
324 &ctx->tweak_ctx, &ctx->crypt_ctx);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200325}
326
327static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
328 struct scatterlist *src, unsigned int nbytes)
329{
330 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200331
Jussi Kivilinna18be4522013-04-08 21:51:00 +0300332 return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
333 XTS_TWEAK_CAST(twofish_enc_blk),
334 &ctx->tweak_ctx, &ctx->crypt_ctx);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200335}
336
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200337static struct crypto_alg twofish_algs[10] = { {
338 .cra_name = "__ecb-twofish-avx",
339 .cra_driver_name = "__driver-ecb-twofish-avx",
340 .cra_priority = 0,
Stephan Mueller4dda66f2015-03-30 22:08:53 +0200341 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
342 CRYPTO_ALG_INTERNAL,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200343 .cra_blocksize = TF_BLOCK_SIZE,
344 .cra_ctxsize = sizeof(struct twofish_ctx),
345 .cra_alignmask = 0,
346 .cra_type = &crypto_blkcipher_type,
347 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200348 .cra_u = {
349 .blkcipher = {
350 .min_keysize = TF_MIN_KEY_SIZE,
351 .max_keysize = TF_MAX_KEY_SIZE,
352 .setkey = twofish_setkey,
353 .encrypt = ecb_encrypt,
354 .decrypt = ecb_decrypt,
355 },
356 },
357}, {
358 .cra_name = "__cbc-twofish-avx",
359 .cra_driver_name = "__driver-cbc-twofish-avx",
360 .cra_priority = 0,
Stephan Mueller4dda66f2015-03-30 22:08:53 +0200361 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
362 CRYPTO_ALG_INTERNAL,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200363 .cra_blocksize = TF_BLOCK_SIZE,
364 .cra_ctxsize = sizeof(struct twofish_ctx),
365 .cra_alignmask = 0,
366 .cra_type = &crypto_blkcipher_type,
367 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200368 .cra_u = {
369 .blkcipher = {
370 .min_keysize = TF_MIN_KEY_SIZE,
371 .max_keysize = TF_MAX_KEY_SIZE,
372 .setkey = twofish_setkey,
373 .encrypt = cbc_encrypt,
374 .decrypt = cbc_decrypt,
375 },
376 },
377}, {
378 .cra_name = "__ctr-twofish-avx",
379 .cra_driver_name = "__driver-ctr-twofish-avx",
380 .cra_priority = 0,
Stephan Mueller4dda66f2015-03-30 22:08:53 +0200381 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
382 CRYPTO_ALG_INTERNAL,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200383 .cra_blocksize = 1,
384 .cra_ctxsize = sizeof(struct twofish_ctx),
385 .cra_alignmask = 0,
386 .cra_type = &crypto_blkcipher_type,
387 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200388 .cra_u = {
389 .blkcipher = {
390 .min_keysize = TF_MIN_KEY_SIZE,
391 .max_keysize = TF_MAX_KEY_SIZE,
392 .ivsize = TF_BLOCK_SIZE,
393 .setkey = twofish_setkey,
394 .encrypt = ctr_crypt,
395 .decrypt = ctr_crypt,
396 },
397 },
398}, {
399 .cra_name = "__lrw-twofish-avx",
400 .cra_driver_name = "__driver-lrw-twofish-avx",
401 .cra_priority = 0,
Stephan Mueller4dda66f2015-03-30 22:08:53 +0200402 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
403 CRYPTO_ALG_INTERNAL,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200404 .cra_blocksize = TF_BLOCK_SIZE,
405 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
406 .cra_alignmask = 0,
407 .cra_type = &crypto_blkcipher_type,
408 .cra_module = THIS_MODULE,
Jussi Kivilinnaa7378d42012-06-18 14:07:39 +0300409 .cra_exit = lrw_twofish_exit_tfm,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200410 .cra_u = {
411 .blkcipher = {
412 .min_keysize = TF_MIN_KEY_SIZE +
413 TF_BLOCK_SIZE,
414 .max_keysize = TF_MAX_KEY_SIZE +
415 TF_BLOCK_SIZE,
416 .ivsize = TF_BLOCK_SIZE,
417 .setkey = lrw_twofish_setkey,
418 .encrypt = lrw_encrypt,
419 .decrypt = lrw_decrypt,
420 },
421 },
422}, {
423 .cra_name = "__xts-twofish-avx",
424 .cra_driver_name = "__driver-xts-twofish-avx",
425 .cra_priority = 0,
Stephan Mueller4dda66f2015-03-30 22:08:53 +0200426 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
427 CRYPTO_ALG_INTERNAL,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200428 .cra_blocksize = TF_BLOCK_SIZE,
429 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
430 .cra_alignmask = 0,
431 .cra_type = &crypto_blkcipher_type,
432 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200433 .cra_u = {
434 .blkcipher = {
435 .min_keysize = TF_MIN_KEY_SIZE * 2,
436 .max_keysize = TF_MAX_KEY_SIZE * 2,
437 .ivsize = TF_BLOCK_SIZE,
438 .setkey = xts_twofish_setkey,
439 .encrypt = xts_encrypt,
440 .decrypt = xts_decrypt,
441 },
442 },
443}, {
444 .cra_name = "ecb(twofish)",
445 .cra_driver_name = "ecb-twofish-avx",
446 .cra_priority = 400,
447 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
448 .cra_blocksize = TF_BLOCK_SIZE,
Jussi Kivilinna30a04002012-06-18 14:07:03 +0300449 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200450 .cra_alignmask = 0,
451 .cra_type = &crypto_ablkcipher_type,
452 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200453 .cra_init = ablk_init,
454 .cra_exit = ablk_exit,
455 .cra_u = {
456 .ablkcipher = {
457 .min_keysize = TF_MIN_KEY_SIZE,
458 .max_keysize = TF_MAX_KEY_SIZE,
459 .setkey = ablk_set_key,
460 .encrypt = ablk_encrypt,
461 .decrypt = ablk_decrypt,
462 },
463 },
464}, {
465 .cra_name = "cbc(twofish)",
466 .cra_driver_name = "cbc-twofish-avx",
467 .cra_priority = 400,
468 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
469 .cra_blocksize = TF_BLOCK_SIZE,
Jussi Kivilinna30a04002012-06-18 14:07:03 +0300470 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200471 .cra_alignmask = 0,
472 .cra_type = &crypto_ablkcipher_type,
473 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200474 .cra_init = ablk_init,
475 .cra_exit = ablk_exit,
476 .cra_u = {
477 .ablkcipher = {
478 .min_keysize = TF_MIN_KEY_SIZE,
479 .max_keysize = TF_MAX_KEY_SIZE,
480 .ivsize = TF_BLOCK_SIZE,
481 .setkey = ablk_set_key,
482 .encrypt = __ablk_encrypt,
483 .decrypt = ablk_decrypt,
484 },
485 },
486}, {
487 .cra_name = "ctr(twofish)",
488 .cra_driver_name = "ctr-twofish-avx",
489 .cra_priority = 400,
490 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
491 .cra_blocksize = 1,
Jussi Kivilinna30a04002012-06-18 14:07:03 +0300492 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200493 .cra_alignmask = 0,
494 .cra_type = &crypto_ablkcipher_type,
495 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200496 .cra_init = ablk_init,
497 .cra_exit = ablk_exit,
498 .cra_u = {
499 .ablkcipher = {
500 .min_keysize = TF_MIN_KEY_SIZE,
501 .max_keysize = TF_MAX_KEY_SIZE,
502 .ivsize = TF_BLOCK_SIZE,
503 .setkey = ablk_set_key,
504 .encrypt = ablk_encrypt,
505 .decrypt = ablk_encrypt,
506 .geniv = "chainiv",
507 },
508 },
509}, {
510 .cra_name = "lrw(twofish)",
511 .cra_driver_name = "lrw-twofish-avx",
512 .cra_priority = 400,
513 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
514 .cra_blocksize = TF_BLOCK_SIZE,
Jussi Kivilinna30a04002012-06-18 14:07:03 +0300515 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200516 .cra_alignmask = 0,
517 .cra_type = &crypto_ablkcipher_type,
518 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200519 .cra_init = ablk_init,
520 .cra_exit = ablk_exit,
521 .cra_u = {
522 .ablkcipher = {
523 .min_keysize = TF_MIN_KEY_SIZE +
524 TF_BLOCK_SIZE,
525 .max_keysize = TF_MAX_KEY_SIZE +
526 TF_BLOCK_SIZE,
527 .ivsize = TF_BLOCK_SIZE,
528 .setkey = ablk_set_key,
529 .encrypt = ablk_encrypt,
530 .decrypt = ablk_decrypt,
531 },
532 },
533}, {
534 .cra_name = "xts(twofish)",
535 .cra_driver_name = "xts-twofish-avx",
536 .cra_priority = 400,
537 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
538 .cra_blocksize = TF_BLOCK_SIZE,
Jussi Kivilinna30a04002012-06-18 14:07:03 +0300539 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200540 .cra_alignmask = 0,
541 .cra_type = &crypto_ablkcipher_type,
542 .cra_module = THIS_MODULE,
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200543 .cra_init = ablk_init,
544 .cra_exit = ablk_exit,
545 .cra_u = {
546 .ablkcipher = {
547 .min_keysize = TF_MIN_KEY_SIZE * 2,
548 .max_keysize = TF_MAX_KEY_SIZE * 2,
549 .ivsize = TF_BLOCK_SIZE,
550 .setkey = ablk_set_key,
551 .encrypt = ablk_encrypt,
552 .decrypt = ablk_decrypt,
553 },
554 },
555} };
556
557static int __init twofish_init(void)
558{
Ingo Molnar4eecd262015-04-28 10:11:24 +0200559 const char *feature_name;
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200560
Borislav Petkov158ecc32015-09-23 12:49:01 +0200561 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
Ingo Molnar4eecd262015-04-28 10:11:24 +0200562 pr_info("CPU feature '%s' is not supported.\n", feature_name);
Johannes Goetzfried107778b2012-05-28 15:54:24 +0200563 return -ENODEV;
564 }
565
566 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
567}
568
569static void __exit twofish_exit(void)
570{
571 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
572}
573
574module_init(twofish_init);
575module_exit(twofish_exit);
576
577MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
578MODULE_LICENSE("GPL");
Kees Cook5d26a102014-11-20 17:05:53 -0800579MODULE_ALIAS_CRYPTO("twofish");