Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 1 | /* |
Eric Biggers | 38e3f25 | 2018-11-16 17:26:26 -0800 | [diff] [blame] | 2 | * ARM NEON accelerated ChaCha and XChaCha stream ciphers, |
| 3 | * including ChaCha20 (RFC7539) |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 4 | * |
| 5 | * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org> |
| 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | * |
| 11 | * Based on: |
| 12 | * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code |
| 13 | * |
| 14 | * Copyright (C) 2015 Martin Willi |
| 15 | * |
| 16 | * This program is free software; you can redistribute it and/or modify |
| 17 | * it under the terms of the GNU General Public License as published by |
| 18 | * the Free Software Foundation; either version 2 of the License, or |
| 19 | * (at your option) any later version. |
| 20 | */ |
| 21 | |
| 22 | #include <crypto/algapi.h> |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 23 | #include <crypto/chacha.h> |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 24 | #include <linux/crypto.h> |
| 25 | #include <linux/kernel.h> |
| 26 | #include <linux/module.h> |
| 27 | |
| 28 | #include <asm/hwcap.h> |
| 29 | #include <asm/neon.h> |
| 30 | #include <asm/simd.h> |
| 31 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 32 | asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src, |
| 33 | int nrounds); |
| 34 | asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src, |
| 35 | int nrounds); |
| 36 | asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 37 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 38 | static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, |
| 39 | unsigned int bytes, int nrounds) |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 40 | { |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 41 | u8 buf[CHACHA_BLOCK_SIZE]; |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 42 | |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 43 | while (bytes >= CHACHA_BLOCK_SIZE * 4) { |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 44 | chacha_4block_xor_neon(state, dst, src, nrounds); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 45 | bytes -= CHACHA_BLOCK_SIZE * 4; |
| 46 | src += CHACHA_BLOCK_SIZE * 4; |
| 47 | dst += CHACHA_BLOCK_SIZE * 4; |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 48 | state[12] += 4; |
| 49 | } |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 50 | while (bytes >= CHACHA_BLOCK_SIZE) { |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 51 | chacha_block_xor_neon(state, dst, src, nrounds); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 52 | bytes -= CHACHA_BLOCK_SIZE; |
| 53 | src += CHACHA_BLOCK_SIZE; |
| 54 | dst += CHACHA_BLOCK_SIZE; |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 55 | state[12]++; |
| 56 | } |
| 57 | if (bytes) { |
| 58 | memcpy(buf, src, bytes); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 59 | chacha_block_xor_neon(state, buf, buf, nrounds); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 60 | memcpy(dst, buf, bytes); |
| 61 | } |
| 62 | } |
| 63 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 64 | static int chacha_neon_stream_xor(struct blkcipher_desc *desc, |
| 65 | struct scatterlist *dst, |
| 66 | struct scatterlist *src, |
| 67 | unsigned int nbytes, |
| 68 | struct chacha_ctx *ctx, u8 *iv) |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 69 | { |
| 70 | struct blkcipher_walk walk; |
| 71 | u32 state[16]; |
| 72 | int err; |
| 73 | |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 74 | blkcipher_walk_init(&walk, dst, src, nbytes); |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 75 | err = blkcipher_walk_virt_block(desc, &walk, CHACHA_BLOCK_SIZE); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 76 | |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 77 | crypto_chacha_init(state, ctx, iv); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 78 | |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 79 | while (walk.nbytes >= CHACHA_BLOCK_SIZE) { |
Eric Biggers | d31aa62 | 2018-11-16 17:26:23 -0800 | [diff] [blame] | 80 | kernel_neon_begin(); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 81 | chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr, |
| 82 | rounddown(walk.nbytes, CHACHA_BLOCK_SIZE), |
| 83 | ctx->nrounds); |
Eric Biggers | d31aa62 | 2018-11-16 17:26:23 -0800 | [diff] [blame] | 84 | kernel_neon_end(); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 85 | err = blkcipher_walk_done(desc, &walk, |
Eric Biggers | b8181f3 | 2018-11-16 17:26:21 -0800 | [diff] [blame] | 86 | walk.nbytes % CHACHA_BLOCK_SIZE); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | if (walk.nbytes) { |
Eric Biggers | d31aa62 | 2018-11-16 17:26:23 -0800 | [diff] [blame] | 90 | kernel_neon_begin(); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 91 | chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr, |
| 92 | walk.nbytes, ctx->nrounds); |
Eric Biggers | d31aa62 | 2018-11-16 17:26:23 -0800 | [diff] [blame] | 93 | kernel_neon_end(); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 94 | err = blkcipher_walk_done(desc, &walk, 0); |
| 95 | } |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 96 | return err; |
| 97 | } |
| 98 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 99 | static int chacha_neon(struct blkcipher_desc *desc, struct scatterlist *dst, |
| 100 | struct scatterlist *src, unsigned int nbytes) |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 101 | { |
| 102 | struct chacha_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); |
| 103 | u8 *iv = desc->info; |
| 104 | |
| 105 | if (nbytes <= CHACHA_BLOCK_SIZE || !may_use_simd()) |
| 106 | return crypto_chacha_crypt(desc, dst, src, nbytes); |
| 107 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 108 | return chacha_neon_stream_xor(desc, dst, src, nbytes, ctx, iv); |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 109 | } |
| 110 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 111 | static int xchacha_neon(struct blkcipher_desc *desc, struct scatterlist *dst, |
| 112 | struct scatterlist *src, unsigned int nbytes) |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 113 | { |
| 114 | struct chacha_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); |
| 115 | u8 *iv = desc->info; |
| 116 | struct chacha_ctx subctx; |
| 117 | u32 state[16]; |
| 118 | u8 real_iv[16]; |
| 119 | |
| 120 | if (nbytes <= CHACHA_BLOCK_SIZE || !may_use_simd()) |
| 121 | return crypto_xchacha_crypt(desc, dst, src, nbytes); |
| 122 | |
| 123 | crypto_chacha_init(state, ctx, iv); |
| 124 | |
| 125 | kernel_neon_begin(); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 126 | hchacha_block_neon(state, subctx.key, ctx->nrounds); |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 127 | kernel_neon_end(); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 128 | subctx.nrounds = ctx->nrounds; |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 129 | |
| 130 | memcpy(&real_iv[0], iv + 24, 8); |
| 131 | memcpy(&real_iv[8], iv + 16, 8); |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 132 | return chacha_neon_stream_xor(desc, dst, src, nbytes, &subctx, real_iv); |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 133 | } |
| 134 | |
| 135 | static struct crypto_alg algs[] = { |
| 136 | { |
| 137 | .cra_name = "chacha20", |
| 138 | .cra_driver_name = "chacha20-neon", |
| 139 | .cra_priority = 300, |
| 140 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, |
| 141 | .cra_blocksize = 1, |
| 142 | .cra_type = &crypto_blkcipher_type, |
| 143 | .cra_ctxsize = sizeof(struct chacha_ctx), |
| 144 | .cra_alignmask = sizeof(u32) - 1, |
| 145 | .cra_module = THIS_MODULE, |
| 146 | .cra_u = { |
| 147 | .blkcipher = { |
| 148 | .min_keysize = CHACHA_KEY_SIZE, |
| 149 | .max_keysize = CHACHA_KEY_SIZE, |
| 150 | .ivsize = CHACHA_IV_SIZE, |
| 151 | .geniv = "seqiv", |
| 152 | .setkey = crypto_chacha20_setkey, |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 153 | .encrypt = chacha_neon, |
| 154 | .decrypt = chacha_neon, |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 155 | }, |
| 156 | }, |
| 157 | }, { |
| 158 | .cra_name = "xchacha20", |
| 159 | .cra_driver_name = "xchacha20-neon", |
| 160 | .cra_priority = 300, |
| 161 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, |
| 162 | .cra_blocksize = 1, |
| 163 | .cra_type = &crypto_blkcipher_type, |
| 164 | .cra_ctxsize = sizeof(struct chacha_ctx), |
| 165 | .cra_alignmask = sizeof(u32) - 1, |
| 166 | .cra_module = THIS_MODULE, |
| 167 | .cra_u = { |
| 168 | .blkcipher = { |
| 169 | .min_keysize = CHACHA_KEY_SIZE, |
| 170 | .max_keysize = CHACHA_KEY_SIZE, |
| 171 | .ivsize = XCHACHA_IV_SIZE, |
| 172 | .geniv = "seqiv", |
| 173 | .setkey = crypto_chacha20_setkey, |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 174 | .encrypt = xchacha_neon, |
| 175 | .decrypt = xchacha_neon, |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 176 | }, |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 177 | }, |
Eric Biggers | 38e3f25 | 2018-11-16 17:26:26 -0800 | [diff] [blame] | 178 | }, { |
| 179 | .cra_name = "xchacha12", |
| 180 | .cra_driver_name = "xchacha12-neon", |
| 181 | .cra_priority = 300, |
| 182 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, |
| 183 | .cra_blocksize = 1, |
| 184 | .cra_type = &crypto_blkcipher_type, |
| 185 | .cra_ctxsize = sizeof(struct chacha_ctx), |
| 186 | .cra_alignmask = sizeof(u32) - 1, |
| 187 | .cra_module = THIS_MODULE, |
| 188 | .cra_u = { |
| 189 | .blkcipher = { |
| 190 | .min_keysize = CHACHA_KEY_SIZE, |
| 191 | .max_keysize = CHACHA_KEY_SIZE, |
| 192 | .ivsize = XCHACHA_IV_SIZE, |
| 193 | .geniv = "seqiv", |
| 194 | .setkey = crypto_chacha12_setkey, |
| 195 | .encrypt = xchacha_neon, |
| 196 | .decrypt = xchacha_neon, |
| 197 | }, |
| 198 | }, |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 199 | }, |
| 200 | }; |
| 201 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 202 | static int __init chacha_simd_mod_init(void) |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 203 | { |
| 204 | if (!(elf_hwcap & HWCAP_NEON)) |
| 205 | return -ENODEV; |
| 206 | |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 207 | return crypto_register_algs(algs, ARRAY_SIZE(algs)); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 208 | } |
| 209 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 210 | static void __exit chacha_simd_mod_fini(void) |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 211 | { |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 212 | crypto_unregister_algs(algs, ARRAY_SIZE(algs)); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 213 | } |
| 214 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 215 | module_init(chacha_simd_mod_init); |
| 216 | module_exit(chacha_simd_mod_fini); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 217 | |
Eric Biggers | 54a345a | 2018-11-16 17:26:25 -0800 | [diff] [blame] | 218 | MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)"); |
Ard Biesheuvel | dc6ff202 | 2016-12-08 14:28:59 +0000 | [diff] [blame] | 219 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); |
| 220 | MODULE_LICENSE("GPL v2"); |
| 221 | MODULE_ALIAS_CRYPTO("chacha20"); |
Eric Biggers | 0b8e72b | 2018-11-16 17:26:24 -0800 | [diff] [blame] | 222 | MODULE_ALIAS_CRYPTO("chacha20-neon"); |
| 223 | MODULE_ALIAS_CRYPTO("xchacha20"); |
| 224 | MODULE_ALIAS_CRYPTO("xchacha20-neon"); |
Eric Biggers | 38e3f25 | 2018-11-16 17:26:26 -0800 | [diff] [blame] | 225 | MODULE_ALIAS_CRYPTO("xchacha12"); |
| 226 | MODULE_ALIAS_CRYPTO("xchacha12-neon"); |