Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 1 | /* |
| 2 | * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code |
| 3 | * |
| 4 | * Copyright (C) 2015 Martin Willi |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License as published by |
| 8 | * the Free Software Foundation; either version 2 of the License, or |
| 9 | * (at your option) any later version. |
| 10 | */ |
| 11 | |
| 12 | #include <crypto/algapi.h> |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 13 | #include <crypto/chacha.h> |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 14 | #include <linux/crypto.h> |
| 15 | #include <linux/kernel.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <asm/fpu/api.h> |
| 18 | #include <asm/simd.h> |
| 19 | |
| 20 | #define CHACHA20_STATE_ALIGN 16 |
| 21 | |
| 22 | asmlinkage void chacha20_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src); |
Martin Willi | 274f938 | 2015-07-16 19:14:02 +0200 | [diff] [blame] | 23 | asmlinkage void chacha20_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src); |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 24 | #ifdef CONFIG_AS_AVX2 |
| 25 | asmlinkage void chacha20_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src); |
| 26 | static bool chacha20_use_avx2; |
| 27 | #endif |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 28 | |
| 29 | static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src, |
| 30 | unsigned int bytes) |
| 31 | { |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 32 | u8 buf[CHACHA_BLOCK_SIZE]; |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 33 | |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 34 | #ifdef CONFIG_AS_AVX2 |
| 35 | if (chacha20_use_avx2) { |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 36 | while (bytes >= CHACHA_BLOCK_SIZE * 8) { |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 37 | chacha20_8block_xor_avx2(state, dst, src); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 38 | bytes -= CHACHA_BLOCK_SIZE * 8; |
| 39 | src += CHACHA_BLOCK_SIZE * 8; |
| 40 | dst += CHACHA_BLOCK_SIZE * 8; |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 41 | state[12] += 8; |
| 42 | } |
| 43 | } |
| 44 | #endif |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 45 | while (bytes >= CHACHA_BLOCK_SIZE * 4) { |
Martin Willi | 274f938 | 2015-07-16 19:14:02 +0200 | [diff] [blame] | 46 | chacha20_4block_xor_ssse3(state, dst, src); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 47 | bytes -= CHACHA_BLOCK_SIZE * 4; |
| 48 | src += CHACHA_BLOCK_SIZE * 4; |
| 49 | dst += CHACHA_BLOCK_SIZE * 4; |
Martin Willi | 274f938 | 2015-07-16 19:14:02 +0200 | [diff] [blame] | 50 | state[12] += 4; |
| 51 | } |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 52 | while (bytes >= CHACHA_BLOCK_SIZE) { |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 53 | chacha20_block_xor_ssse3(state, dst, src); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 54 | bytes -= CHACHA_BLOCK_SIZE; |
| 55 | src += CHACHA_BLOCK_SIZE; |
| 56 | dst += CHACHA_BLOCK_SIZE; |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 57 | state[12]++; |
| 58 | } |
| 59 | if (bytes) { |
| 60 | memcpy(buf, src, bytes); |
| 61 | chacha20_block_xor_ssse3(state, buf, buf); |
| 62 | memcpy(dst, buf, bytes); |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst, |
| 67 | struct scatterlist *src, unsigned int nbytes) |
| 68 | { |
| 69 | u32 *state, state_buf[16 + (CHACHA20_STATE_ALIGN / sizeof(u32)) - 1]; |
| 70 | struct blkcipher_walk walk; |
| 71 | int err; |
| 72 | |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 73 | if (nbytes <= CHACHA_BLOCK_SIZE || !may_use_simd()) |
| 74 | return crypto_chacha_crypt(desc, dst, src, nbytes); |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 75 | |
| 76 | state = (u32 *)roundup((uintptr_t)state_buf, CHACHA20_STATE_ALIGN); |
| 77 | |
| 78 | blkcipher_walk_init(&walk, dst, src, nbytes); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 79 | err = blkcipher_walk_virt_block(desc, &walk, CHACHA_BLOCK_SIZE); |
Eric Biggers | ce8ca30 | 2019-01-07 15:15:59 -0800 | [diff] [blame] | 80 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 81 | |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 82 | crypto_chacha_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv); |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 83 | |
| 84 | kernel_fpu_begin(); |
| 85 | |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 86 | while (walk.nbytes >= CHACHA_BLOCK_SIZE) { |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 87 | chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 88 | rounddown(walk.nbytes, CHACHA_BLOCK_SIZE)); |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 89 | err = blkcipher_walk_done(desc, &walk, |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 90 | walk.nbytes % CHACHA_BLOCK_SIZE); |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 91 | } |
| 92 | |
| 93 | if (walk.nbytes) { |
| 94 | chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, |
| 95 | walk.nbytes); |
| 96 | err = blkcipher_walk_done(desc, &walk, 0); |
| 97 | } |
| 98 | |
| 99 | kernel_fpu_end(); |
| 100 | |
| 101 | return err; |
| 102 | } |
| 103 | |
| 104 | static struct crypto_alg alg = { |
| 105 | .cra_name = "chacha20", |
| 106 | .cra_driver_name = "chacha20-simd", |
| 107 | .cra_priority = 300, |
| 108 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, |
| 109 | .cra_blocksize = 1, |
| 110 | .cra_type = &crypto_blkcipher_type, |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 111 | .cra_ctxsize = sizeof(struct chacha_ctx), |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 112 | .cra_alignmask = sizeof(u32) - 1, |
| 113 | .cra_module = THIS_MODULE, |
| 114 | .cra_u = { |
| 115 | .blkcipher = { |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 116 | .min_keysize = CHACHA_KEY_SIZE, |
| 117 | .max_keysize = CHACHA_KEY_SIZE, |
| 118 | .ivsize = CHACHA_IV_SIZE, |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 119 | .geniv = "seqiv", |
| 120 | .setkey = crypto_chacha20_setkey, |
| 121 | .encrypt = chacha20_simd, |
| 122 | .decrypt = chacha20_simd, |
| 123 | }, |
| 124 | }, |
| 125 | }; |
| 126 | |
| 127 | static int __init chacha20_simd_mod_init(void) |
| 128 | { |
Borislav Petkov | 362f924 | 2015-12-07 10:39:41 +0100 | [diff] [blame] | 129 | if (!boot_cpu_has(X86_FEATURE_SSSE3)) |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 130 | return -ENODEV; |
| 131 | |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 132 | #ifdef CONFIG_AS_AVX2 |
Borislav Petkov | da154e8 | 2016-04-04 22:24:56 +0200 | [diff] [blame] | 133 | chacha20_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) && |
| 134 | boot_cpu_has(X86_FEATURE_AVX2) && |
Dave Hansen | d91cab7 | 2015-09-02 16:31:26 -0700 | [diff] [blame] | 135 | cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); |
Martin Willi | 3d1e93c | 2015-07-16 19:14:03 +0200 | [diff] [blame] | 136 | #endif |
Martin Willi | c9320b6 | 2015-07-16 19:14:01 +0200 | [diff] [blame] | 137 | return crypto_register_alg(&alg); |
| 138 | } |
| 139 | |
| 140 | static void __exit chacha20_simd_mod_fini(void) |
| 141 | { |
| 142 | crypto_unregister_alg(&alg); |
| 143 | } |
| 144 | |
| 145 | module_init(chacha20_simd_mod_init); |
| 146 | module_exit(chacha20_simd_mod_fini); |
| 147 | |
| 148 | MODULE_LICENSE("GPL"); |
| 149 | MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); |
| 150 | MODULE_DESCRIPTION("chacha20 cipher algorithm, SIMD accelerated"); |
| 151 | MODULE_ALIAS_CRYPTO("chacha20"); |
| 152 | MODULE_ALIAS_CRYPTO("chacha20-simd"); |