Sami Tolvanen | 7c20af0 | 2017-08-10 09:42:10 -0700 | [diff] [blame] | 1 | /* |
| 2 | * aes-ce-cipher-core.c - core AES cipher using ARMv8 Crypto Extensions |
| 3 | * |
| 4 | * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | |
| 11 | #include <asm/neon.h> |
| 12 | #include <crypto/aes.h> |
| 13 | #include <linux/crypto.h> |
| 14 | |
| 15 | #include "aes-ce-setkey.h" |
| 16 | |
| 17 | struct aes_block { |
| 18 | u8 b[AES_BLOCK_SIZE]; |
| 19 | }; |
| 20 | |
| 21 | static int num_rounds(struct crypto_aes_ctx *ctx) |
| 22 | { |
| 23 | /* |
| 24 | * # of rounds specified by AES: |
| 25 | * 128 bit key 10 rounds |
| 26 | * 192 bit key 12 rounds |
| 27 | * 256 bit key 14 rounds |
| 28 | * => n byte key => 6 + (n/4) rounds |
| 29 | */ |
| 30 | return 6 + ctx->key_length / 4; |
| 31 | } |
| 32 | |
| 33 | void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) |
| 34 | { |
| 35 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); |
| 36 | struct aes_block *out = (struct aes_block *)dst; |
| 37 | struct aes_block const *in = (struct aes_block *)src; |
| 38 | void *dummy0; |
| 39 | int dummy1; |
| 40 | |
| 41 | kernel_neon_begin_partial(4); |
| 42 | |
| 43 | __asm__(" ld1 {v0.16b}, %[in] ;" |
| 44 | " ld1 {v1.16b}, [%[key]], #16 ;" |
| 45 | " cmp %w[rounds], #10 ;" |
| 46 | " bmi 0f ;" |
| 47 | " bne 3f ;" |
| 48 | " mov v3.16b, v1.16b ;" |
| 49 | " b 2f ;" |
| 50 | "0: mov v2.16b, v1.16b ;" |
| 51 | " ld1 {v3.16b}, [%[key]], #16 ;" |
| 52 | "1: aese v0.16b, v2.16b ;" |
| 53 | " aesmc v0.16b, v0.16b ;" |
| 54 | "2: ld1 {v1.16b}, [%[key]], #16 ;" |
| 55 | " aese v0.16b, v3.16b ;" |
| 56 | " aesmc v0.16b, v0.16b ;" |
| 57 | "3: ld1 {v2.16b}, [%[key]], #16 ;" |
| 58 | " subs %w[rounds], %w[rounds], #3 ;" |
| 59 | " aese v0.16b, v1.16b ;" |
| 60 | " aesmc v0.16b, v0.16b ;" |
| 61 | " ld1 {v3.16b}, [%[key]], #16 ;" |
| 62 | " bpl 1b ;" |
| 63 | " aese v0.16b, v2.16b ;" |
| 64 | " eor v0.16b, v0.16b, v3.16b ;" |
| 65 | " st1 {v0.16b}, %[out] ;" |
| 66 | |
| 67 | : [out] "=Q"(*out), |
| 68 | [key] "=r"(dummy0), |
| 69 | [rounds] "=r"(dummy1) |
| 70 | : [in] "Q"(*in), |
| 71 | "1"(ctx->key_enc), |
| 72 | "2"(num_rounds(ctx) - 2) |
| 73 | : "cc"); |
| 74 | |
| 75 | kernel_neon_end(); |
| 76 | } |
| 77 | |
| 78 | void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) |
| 79 | { |
| 80 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); |
| 81 | struct aes_block *out = (struct aes_block *)dst; |
| 82 | struct aes_block const *in = (struct aes_block *)src; |
| 83 | void *dummy0; |
| 84 | int dummy1; |
| 85 | |
| 86 | kernel_neon_begin_partial(4); |
| 87 | |
| 88 | __asm__(" ld1 {v0.16b}, %[in] ;" |
| 89 | " ld1 {v1.16b}, [%[key]], #16 ;" |
| 90 | " cmp %w[rounds], #10 ;" |
| 91 | " bmi 0f ;" |
| 92 | " bne 3f ;" |
| 93 | " mov v3.16b, v1.16b ;" |
| 94 | " b 2f ;" |
| 95 | "0: mov v2.16b, v1.16b ;" |
| 96 | " ld1 {v3.16b}, [%[key]], #16 ;" |
| 97 | "1: aesd v0.16b, v2.16b ;" |
| 98 | " aesimc v0.16b, v0.16b ;" |
| 99 | "2: ld1 {v1.16b}, [%[key]], #16 ;" |
| 100 | " aesd v0.16b, v3.16b ;" |
| 101 | " aesimc v0.16b, v0.16b ;" |
| 102 | "3: ld1 {v2.16b}, [%[key]], #16 ;" |
| 103 | " subs %w[rounds], %w[rounds], #3 ;" |
| 104 | " aesd v0.16b, v1.16b ;" |
| 105 | " aesimc v0.16b, v0.16b ;" |
| 106 | " ld1 {v3.16b}, [%[key]], #16 ;" |
| 107 | " bpl 1b ;" |
| 108 | " aesd v0.16b, v2.16b ;" |
| 109 | " eor v0.16b, v0.16b, v3.16b ;" |
| 110 | " st1 {v0.16b}, %[out] ;" |
| 111 | |
| 112 | : [out] "=Q"(*out), |
| 113 | [key] "=r"(dummy0), |
| 114 | [rounds] "=r"(dummy1) |
| 115 | : [in] "Q"(*in), |
| 116 | "1"(ctx->key_dec), |
| 117 | "2"(num_rounds(ctx) - 2) |
| 118 | : "cc"); |
| 119 | |
| 120 | kernel_neon_end(); |
| 121 | } |
| 122 | |
| 123 | /* |
| 124 | * aes_sub() - use the aese instruction to perform the AES sbox substitution |
| 125 | * on each byte in 'input' |
| 126 | */ |
| 127 | static u32 aes_sub(u32 input) |
| 128 | { |
| 129 | u32 ret; |
| 130 | |
| 131 | __asm__("dup v1.4s, %w[in] ;" |
| 132 | "movi v0.16b, #0 ;" |
| 133 | "aese v0.16b, v1.16b ;" |
| 134 | "umov %w[out], v0.4s[0] ;" |
| 135 | |
| 136 | : [out] "=r"(ret) |
| 137 | : [in] "r"(input) |
| 138 | : "v0","v1"); |
| 139 | |
| 140 | return ret; |
| 141 | } |
| 142 | |
| 143 | int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, |
| 144 | unsigned int key_len) |
| 145 | { |
| 146 | /* |
| 147 | * The AES key schedule round constants |
| 148 | */ |
| 149 | static u8 const rcon[] = { |
| 150 | 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, |
| 151 | }; |
| 152 | |
| 153 | u32 kwords = key_len / sizeof(u32); |
| 154 | struct aes_block *key_enc, *key_dec; |
| 155 | int i, j; |
| 156 | |
| 157 | if (key_len != AES_KEYSIZE_128 && |
| 158 | key_len != AES_KEYSIZE_192 && |
| 159 | key_len != AES_KEYSIZE_256) |
| 160 | return -EINVAL; |
| 161 | |
| 162 | memcpy(ctx->key_enc, in_key, key_len); |
| 163 | ctx->key_length = key_len; |
| 164 | |
| 165 | kernel_neon_begin_partial(2); |
| 166 | for (i = 0; i < sizeof(rcon); i++) { |
| 167 | u32 *rki = ctx->key_enc + (i * kwords); |
| 168 | u32 *rko = rki + kwords; |
| 169 | |
| 170 | #ifndef CONFIG_CPU_BIG_ENDIAN |
| 171 | rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; |
| 172 | #else |
| 173 | rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^ |
| 174 | rki[0]; |
| 175 | #endif |
| 176 | rko[1] = rko[0] ^ rki[1]; |
| 177 | rko[2] = rko[1] ^ rki[2]; |
| 178 | rko[3] = rko[2] ^ rki[3]; |
| 179 | |
| 180 | if (key_len == AES_KEYSIZE_192) { |
| 181 | if (i >= 7) |
| 182 | break; |
| 183 | rko[4] = rko[3] ^ rki[4]; |
| 184 | rko[5] = rko[4] ^ rki[5]; |
| 185 | } else if (key_len == AES_KEYSIZE_256) { |
| 186 | if (i >= 6) |
| 187 | break; |
| 188 | rko[4] = aes_sub(rko[3]) ^ rki[4]; |
| 189 | rko[5] = rko[4] ^ rki[5]; |
| 190 | rko[6] = rko[5] ^ rki[6]; |
| 191 | rko[7] = rko[6] ^ rki[7]; |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | /* |
| 196 | * Generate the decryption keys for the Equivalent Inverse Cipher. |
| 197 | * This involves reversing the order of the round keys, and applying |
| 198 | * the Inverse Mix Columns transformation on all but the first and |
| 199 | * the last one. |
| 200 | */ |
| 201 | key_enc = (struct aes_block *)ctx->key_enc; |
| 202 | key_dec = (struct aes_block *)ctx->key_dec; |
| 203 | j = num_rounds(ctx); |
| 204 | |
| 205 | key_dec[0] = key_enc[j]; |
| 206 | for (i = 1, j--; j > 0; i++, j--) |
| 207 | __asm__("ld1 {v0.16b}, %[in] ;" |
| 208 | "aesimc v1.16b, v0.16b ;" |
| 209 | "st1 {v1.16b}, %[out] ;" |
| 210 | |
| 211 | : [out] "=Q"(key_dec[i]) |
| 212 | : [in] "Q"(key_enc[j]) |
| 213 | : "v0","v1"); |
| 214 | key_dec[i] = key_enc[0]; |
| 215 | |
| 216 | kernel_neon_end(); |
| 217 | return 0; |
| 218 | } |
| 219 | EXPORT_SYMBOL(ce_aes_expandkey); |