blob: 2aa31ade1e68312ba48d61e56dfff1e377eba1ad [file] [log] [blame]
Johannes Goetzfried7efe4072012-06-12 16:47:43 +08001/*
2 * Glue Code for AVX assembler versions of Serpent Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Glue code based on serpent_sse2_glue.c by:
8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27#include <linux/module.h>
28#include <linux/hardirq.h>
29#include <linux/types.h>
30#include <linux/crypto.h>
31#include <linux/err.h>
32#include <crypto/algapi.h>
33#include <crypto/serpent.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080039#include <asm/xcr.h>
40#include <asm/xsave.h>
Jussi Kivilinnad4af0e92012-06-18 14:07:45 +030041#include <asm/crypto/serpent-avx.h>
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +030042#include <asm/crypto/ablk_helper.h>
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030043#include <asm/crypto/glue_helper.h>
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080044
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030045static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080046{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030047 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
48 unsigned int j;
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080049
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030050 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
51 ivs[j] = src[j];
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080052
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030053 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
54
55 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
56 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080057}
58
Jussi Kivilinna58990982012-10-20 15:06:36 +030059static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080060{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030061 be128 ctrblk;
62
Jussi Kivilinna58990982012-10-20 15:06:36 +030063 le128_to_be128(&ctrblk, iv);
64 le128_inc(iv);
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030065
66 __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
67 u128_xor(dst, src, (u128 *)&ctrblk);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080068}
69
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030070static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
Jussi Kivilinna58990982012-10-20 15:06:36 +030071 le128 *iv)
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080072{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030073 be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
74 unsigned int i;
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080075
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030076 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
77 if (dst != src)
78 dst[i] = src[i];
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080079
Jussi Kivilinna58990982012-10-20 15:06:36 +030080 le128_to_be128(&ctrblks[i], iv);
81 le128_inc(iv);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080082 }
83
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030084 serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +080085}
86
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +030087static const struct common_glue_ctx serpent_enc = {
88 .num_funcs = 2,
89 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
90
91 .funcs = { {
92 .num_blocks = SERPENT_PARALLEL_BLOCKS,
93 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
94 }, {
95 .num_blocks = 1,
96 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
97 } }
98};
99
100static const struct common_glue_ctx serpent_ctr = {
101 .num_funcs = 2,
102 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
103
104 .funcs = { {
105 .num_blocks = SERPENT_PARALLEL_BLOCKS,
106 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
107 }, {
108 .num_blocks = 1,
109 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
110 } }
111};
112
113static const struct common_glue_ctx serpent_dec = {
114 .num_funcs = 2,
115 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
116
117 .funcs = { {
118 .num_blocks = SERPENT_PARALLEL_BLOCKS,
119 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
120 }, {
121 .num_blocks = 1,
122 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
123 } }
124};
125
126static const struct common_glue_ctx serpent_dec_cbc = {
127 .num_funcs = 2,
128 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
129
130 .funcs = { {
131 .num_blocks = SERPENT_PARALLEL_BLOCKS,
132 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
133 }, {
134 .num_blocks = 1,
135 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
136 } }
137};
138
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800139static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
140 struct scatterlist *src, unsigned int nbytes)
141{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300142 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800143}
144
145static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
146 struct scatterlist *src, unsigned int nbytes)
147{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300148 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800149}
150
151static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
152 struct scatterlist *src, unsigned int nbytes)
153{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300154 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
155 dst, src, nbytes);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800156}
157
158static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
159 struct scatterlist *src, unsigned int nbytes)
160{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300161 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
162 nbytes);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800163}
164
165static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
166 struct scatterlist *src, unsigned int nbytes)
167{
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300168 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
169}
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800170
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300171static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
172{
173 return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
174 NULL, fpu_enabled, nbytes);
175}
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800176
Jussi Kivilinna1d0debb2012-06-18 14:07:24 +0300177static inline void serpent_fpu_end(bool fpu_enabled)
178{
179 glue_fpu_end(fpu_enabled);
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800180}
181
182struct crypt_priv {
183 struct serpent_ctx *ctx;
184 bool fpu_enabled;
185};
186
187static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
188{
189 const unsigned int bsize = SERPENT_BLOCK_SIZE;
190 struct crypt_priv *ctx = priv;
191 int i;
192
193 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
194
195 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
196 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
197 return;
198 }
199
200 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
201 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
202}
203
204static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
205{
206 const unsigned int bsize = SERPENT_BLOCK_SIZE;
207 struct crypt_priv *ctx = priv;
208 int i;
209
210 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
211
212 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
213 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
214 return;
215 }
216
217 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
218 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
219}
220
221struct serpent_lrw_ctx {
222 struct lrw_table_ctx lrw_table;
223 struct serpent_ctx serpent_ctx;
224};
225
226static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
227 unsigned int keylen)
228{
229 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
230 int err;
231
232 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
233 SERPENT_BLOCK_SIZE);
234 if (err)
235 return err;
236
237 return lrw_init_table(&ctx->lrw_table, key + keylen -
238 SERPENT_BLOCK_SIZE);
239}
240
241static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
242 struct scatterlist *src, unsigned int nbytes)
243{
244 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
245 be128 buf[SERPENT_PARALLEL_BLOCKS];
246 struct crypt_priv crypt_ctx = {
247 .ctx = &ctx->serpent_ctx,
248 .fpu_enabled = false,
249 };
250 struct lrw_crypt_req req = {
251 .tbuf = buf,
252 .tbuflen = sizeof(buf),
253
254 .table_ctx = &ctx->lrw_table,
255 .crypt_ctx = &crypt_ctx,
256 .crypt_fn = encrypt_callback,
257 };
258 int ret;
259
260 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
261 ret = lrw_crypt(desc, dst, src, nbytes, &req);
262 serpent_fpu_end(crypt_ctx.fpu_enabled);
263
264 return ret;
265}
266
267static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268 struct scatterlist *src, unsigned int nbytes)
269{
270 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
271 be128 buf[SERPENT_PARALLEL_BLOCKS];
272 struct crypt_priv crypt_ctx = {
273 .ctx = &ctx->serpent_ctx,
274 .fpu_enabled = false,
275 };
276 struct lrw_crypt_req req = {
277 .tbuf = buf,
278 .tbuflen = sizeof(buf),
279
280 .table_ctx = &ctx->lrw_table,
281 .crypt_ctx = &crypt_ctx,
282 .crypt_fn = decrypt_callback,
283 };
284 int ret;
285
286 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
287 ret = lrw_crypt(desc, dst, src, nbytes, &req);
288 serpent_fpu_end(crypt_ctx.fpu_enabled);
289
290 return ret;
291}
292
293static void lrw_exit_tfm(struct crypto_tfm *tfm)
294{
295 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
296
297 lrw_free_table(&ctx->lrw_table);
298}
299
300struct serpent_xts_ctx {
301 struct serpent_ctx tweak_ctx;
302 struct serpent_ctx crypt_ctx;
303};
304
305static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
306 unsigned int keylen)
307{
308 struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
309 u32 *flags = &tfm->crt_flags;
310 int err;
311
312 /* key consists of keys of equal size concatenated, therefore
313 * the length must be even
314 */
315 if (keylen % 2) {
316 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
317 return -EINVAL;
318 }
319
320 /* first half of xts-key is for crypt */
321 err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
322 if (err)
323 return err;
324
325 /* second half of xts-key is for tweak */
326 return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
327}
328
329static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330 struct scatterlist *src, unsigned int nbytes)
331{
332 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
333 be128 buf[SERPENT_PARALLEL_BLOCKS];
334 struct crypt_priv crypt_ctx = {
335 .ctx = &ctx->crypt_ctx,
336 .fpu_enabled = false,
337 };
338 struct xts_crypt_req req = {
339 .tbuf = buf,
340 .tbuflen = sizeof(buf),
341
342 .tweak_ctx = &ctx->tweak_ctx,
343 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
344 .crypt_ctx = &crypt_ctx,
345 .crypt_fn = encrypt_callback,
346 };
347 int ret;
348
349 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
350 ret = xts_crypt(desc, dst, src, nbytes, &req);
351 serpent_fpu_end(crypt_ctx.fpu_enabled);
352
353 return ret;
354}
355
356static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
357 struct scatterlist *src, unsigned int nbytes)
358{
359 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
360 be128 buf[SERPENT_PARALLEL_BLOCKS];
361 struct crypt_priv crypt_ctx = {
362 .ctx = &ctx->crypt_ctx,
363 .fpu_enabled = false,
364 };
365 struct xts_crypt_req req = {
366 .tbuf = buf,
367 .tbuflen = sizeof(buf),
368
369 .tweak_ctx = &ctx->tweak_ctx,
370 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
371 .crypt_ctx = &crypt_ctx,
372 .crypt_fn = decrypt_callback,
373 };
374 int ret;
375
376 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
377 ret = xts_crypt(desc, dst, src, nbytes, &req);
378 serpent_fpu_end(crypt_ctx.fpu_enabled);
379
380 return ret;
381}
382
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800383static struct crypto_alg serpent_algs[10] = { {
384 .cra_name = "__ecb-serpent-avx",
385 .cra_driver_name = "__driver-ecb-serpent-avx",
386 .cra_priority = 0,
387 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
388 .cra_blocksize = SERPENT_BLOCK_SIZE,
389 .cra_ctxsize = sizeof(struct serpent_ctx),
390 .cra_alignmask = 0,
391 .cra_type = &crypto_blkcipher_type,
392 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800393 .cra_u = {
394 .blkcipher = {
395 .min_keysize = SERPENT_MIN_KEY_SIZE,
396 .max_keysize = SERPENT_MAX_KEY_SIZE,
397 .setkey = serpent_setkey,
398 .encrypt = ecb_encrypt,
399 .decrypt = ecb_decrypt,
400 },
401 },
402}, {
403 .cra_name = "__cbc-serpent-avx",
404 .cra_driver_name = "__driver-cbc-serpent-avx",
405 .cra_priority = 0,
406 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
407 .cra_blocksize = SERPENT_BLOCK_SIZE,
408 .cra_ctxsize = sizeof(struct serpent_ctx),
409 .cra_alignmask = 0,
410 .cra_type = &crypto_blkcipher_type,
411 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800412 .cra_u = {
413 .blkcipher = {
414 .min_keysize = SERPENT_MIN_KEY_SIZE,
415 .max_keysize = SERPENT_MAX_KEY_SIZE,
416 .setkey = serpent_setkey,
417 .encrypt = cbc_encrypt,
418 .decrypt = cbc_decrypt,
419 },
420 },
421}, {
422 .cra_name = "__ctr-serpent-avx",
423 .cra_driver_name = "__driver-ctr-serpent-avx",
424 .cra_priority = 0,
425 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
426 .cra_blocksize = 1,
427 .cra_ctxsize = sizeof(struct serpent_ctx),
428 .cra_alignmask = 0,
429 .cra_type = &crypto_blkcipher_type,
430 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800431 .cra_u = {
432 .blkcipher = {
433 .min_keysize = SERPENT_MIN_KEY_SIZE,
434 .max_keysize = SERPENT_MAX_KEY_SIZE,
435 .ivsize = SERPENT_BLOCK_SIZE,
436 .setkey = serpent_setkey,
437 .encrypt = ctr_crypt,
438 .decrypt = ctr_crypt,
439 },
440 },
441}, {
442 .cra_name = "__lrw-serpent-avx",
443 .cra_driver_name = "__driver-lrw-serpent-avx",
444 .cra_priority = 0,
445 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
446 .cra_blocksize = SERPENT_BLOCK_SIZE,
447 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
448 .cra_alignmask = 0,
449 .cra_type = &crypto_blkcipher_type,
450 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800451 .cra_exit = lrw_exit_tfm,
452 .cra_u = {
453 .blkcipher = {
454 .min_keysize = SERPENT_MIN_KEY_SIZE +
455 SERPENT_BLOCK_SIZE,
456 .max_keysize = SERPENT_MAX_KEY_SIZE +
457 SERPENT_BLOCK_SIZE,
458 .ivsize = SERPENT_BLOCK_SIZE,
459 .setkey = lrw_serpent_setkey,
460 .encrypt = lrw_encrypt,
461 .decrypt = lrw_decrypt,
462 },
463 },
464}, {
465 .cra_name = "__xts-serpent-avx",
466 .cra_driver_name = "__driver-xts-serpent-avx",
467 .cra_priority = 0,
468 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
469 .cra_blocksize = SERPENT_BLOCK_SIZE,
470 .cra_ctxsize = sizeof(struct serpent_xts_ctx),
471 .cra_alignmask = 0,
472 .cra_type = &crypto_blkcipher_type,
473 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800474 .cra_u = {
475 .blkcipher = {
476 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
477 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
478 .ivsize = SERPENT_BLOCK_SIZE,
479 .setkey = xts_serpent_setkey,
480 .encrypt = xts_encrypt,
481 .decrypt = xts_decrypt,
482 },
483 },
484}, {
485 .cra_name = "ecb(serpent)",
486 .cra_driver_name = "ecb-serpent-avx",
487 .cra_priority = 500,
488 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
489 .cra_blocksize = SERPENT_BLOCK_SIZE,
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +0300490 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800491 .cra_alignmask = 0,
492 .cra_type = &crypto_ablkcipher_type,
493 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800494 .cra_init = ablk_init,
495 .cra_exit = ablk_exit,
496 .cra_u = {
497 .ablkcipher = {
498 .min_keysize = SERPENT_MIN_KEY_SIZE,
499 .max_keysize = SERPENT_MAX_KEY_SIZE,
500 .setkey = ablk_set_key,
501 .encrypt = ablk_encrypt,
502 .decrypt = ablk_decrypt,
503 },
504 },
505}, {
506 .cra_name = "cbc(serpent)",
507 .cra_driver_name = "cbc-serpent-avx",
508 .cra_priority = 500,
509 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
510 .cra_blocksize = SERPENT_BLOCK_SIZE,
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +0300511 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800512 .cra_alignmask = 0,
513 .cra_type = &crypto_ablkcipher_type,
514 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800515 .cra_init = ablk_init,
516 .cra_exit = ablk_exit,
517 .cra_u = {
518 .ablkcipher = {
519 .min_keysize = SERPENT_MIN_KEY_SIZE,
520 .max_keysize = SERPENT_MAX_KEY_SIZE,
521 .ivsize = SERPENT_BLOCK_SIZE,
522 .setkey = ablk_set_key,
523 .encrypt = __ablk_encrypt,
524 .decrypt = ablk_decrypt,
525 },
526 },
527}, {
528 .cra_name = "ctr(serpent)",
529 .cra_driver_name = "ctr-serpent-avx",
530 .cra_priority = 500,
531 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
532 .cra_blocksize = 1,
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +0300533 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800534 .cra_alignmask = 0,
535 .cra_type = &crypto_ablkcipher_type,
536 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800537 .cra_init = ablk_init,
538 .cra_exit = ablk_exit,
539 .cra_u = {
540 .ablkcipher = {
541 .min_keysize = SERPENT_MIN_KEY_SIZE,
542 .max_keysize = SERPENT_MAX_KEY_SIZE,
543 .ivsize = SERPENT_BLOCK_SIZE,
544 .setkey = ablk_set_key,
545 .encrypt = ablk_encrypt,
546 .decrypt = ablk_encrypt,
547 .geniv = "chainiv",
548 },
549 },
550}, {
551 .cra_name = "lrw(serpent)",
552 .cra_driver_name = "lrw-serpent-avx",
553 .cra_priority = 500,
554 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
555 .cra_blocksize = SERPENT_BLOCK_SIZE,
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +0300556 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800557 .cra_alignmask = 0,
558 .cra_type = &crypto_ablkcipher_type,
559 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800560 .cra_init = ablk_init,
561 .cra_exit = ablk_exit,
562 .cra_u = {
563 .ablkcipher = {
564 .min_keysize = SERPENT_MIN_KEY_SIZE +
565 SERPENT_BLOCK_SIZE,
566 .max_keysize = SERPENT_MAX_KEY_SIZE +
567 SERPENT_BLOCK_SIZE,
568 .ivsize = SERPENT_BLOCK_SIZE,
569 .setkey = ablk_set_key,
570 .encrypt = ablk_encrypt,
571 .decrypt = ablk_decrypt,
572 },
573 },
574}, {
575 .cra_name = "xts(serpent)",
576 .cra_driver_name = "xts-serpent-avx",
577 .cra_priority = 500,
578 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
579 .cra_blocksize = SERPENT_BLOCK_SIZE,
Jussi Kivilinnaffaf9152012-06-18 14:06:58 +0300580 .cra_ctxsize = sizeof(struct async_helper_ctx),
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800581 .cra_alignmask = 0,
582 .cra_type = &crypto_ablkcipher_type,
583 .cra_module = THIS_MODULE,
Johannes Goetzfried7efe4072012-06-12 16:47:43 +0800584 .cra_init = ablk_init,
585 .cra_exit = ablk_exit,
586 .cra_u = {
587 .ablkcipher = {
588 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
589 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
590 .ivsize = SERPENT_BLOCK_SIZE,
591 .setkey = ablk_set_key,
592 .encrypt = ablk_encrypt,
593 .decrypt = ablk_decrypt,
594 },
595 },
596} };
597
598static int __init serpent_init(void)
599{
600 u64 xcr0;
601
602 if (!cpu_has_avx || !cpu_has_osxsave) {
603 printk(KERN_INFO "AVX instructions are not detected.\n");
604 return -ENODEV;
605 }
606
607 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
608 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
609 printk(KERN_INFO "AVX detected but unusable.\n");
610 return -ENODEV;
611 }
612
613 return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
614}
615
616static void __exit serpent_exit(void)
617{
618 crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
619}
620
621module_init(serpent_init);
622module_exit(serpent_exit);
623
624MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
625MODULE_LICENSE("GPL");
626MODULE_ALIAS("serpent");