Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 1 | /* ==================================================================== |
| 2 | * Copyright (c) 2008 The OpenSSL Project. All rights reserved. |
| 3 | * |
| 4 | * Redistribution and use in source and binary forms, with or without |
| 5 | * modification, are permitted provided that the following conditions |
| 6 | * are met: |
| 7 | * |
| 8 | * 1. Redistributions of source code must retain the above copyright |
| 9 | * notice, this list of conditions and the following disclaimer. |
| 10 | * |
| 11 | * 2. Redistributions in binary form must reproduce the above copyright |
| 12 | * notice, this list of conditions and the following disclaimer in |
| 13 | * the documentation and/or other materials provided with the |
| 14 | * distribution. |
| 15 | * |
| 16 | * 3. All advertising materials mentioning features or use of this |
| 17 | * software must display the following acknowledgment: |
| 18 | * "This product includes software developed by the OpenSSL Project |
| 19 | * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" |
| 20 | * |
| 21 | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to |
| 22 | * endorse or promote products derived from this software without |
| 23 | * prior written permission. For written permission, please contact |
| 24 | * openssl-core@openssl.org. |
| 25 | * |
| 26 | * 5. Products derived from this software may not be called "OpenSSL" |
| 27 | * nor may "OpenSSL" appear in their names without prior written |
| 28 | * permission of the OpenSSL Project. |
| 29 | * |
| 30 | * 6. Redistributions of any form whatsoever must retain the following |
| 31 | * acknowledgment: |
| 32 | * "This product includes software developed by the OpenSSL Project |
| 33 | * for use in the OpenSSL Toolkit (http://www.openssl.org/)" |
| 34 | * |
| 35 | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY |
| 36 | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 37 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 38 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR |
| 39 | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 40 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| 41 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 42 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 43 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| 44 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 45 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED |
| 46 | * OF THE POSSIBILITY OF SUCH DAMAGE. |
| 47 | * ==================================================================== */ |
| 48 | |
| 49 | #ifndef OPENSSL_HEADER_MODES_INTERNAL_H |
| 50 | #define OPENSSL_HEADER_MODES_INTERNAL_H |
| 51 | |
| 52 | #include <openssl/base.h> |
| 53 | |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 54 | #include <openssl/aes.h> |
| 55 | #include <openssl/cpu.h> |
| 56 | |
| 57 | #include <stdlib.h> |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 58 | #include <string.h> |
| 59 | |
Robert Sloan | 9254e68 | 2017-04-24 09:42:06 -0700 | [diff] [blame] | 60 | #include "../../internal.h" |
Robert Sloan | 69939df | 2017-01-09 10:53:07 -0800 | [diff] [blame] | 61 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 62 | #if defined(__cplusplus) |
| 63 | extern "C" { |
| 64 | #endif |
| 65 | |
| 66 | |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 67 | static inline uint32_t GETU32(const void *in) { |
| 68 | uint32_t v; |
Robert Sloan | 69939df | 2017-01-09 10:53:07 -0800 | [diff] [blame] | 69 | OPENSSL_memcpy(&v, in, sizeof(v)); |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 70 | return CRYPTO_bswap4(v); |
| 71 | } |
| 72 | |
| 73 | static inline void PUTU32(void *out, uint32_t v) { |
| 74 | v = CRYPTO_bswap4(v); |
Robert Sloan | 69939df | 2017-01-09 10:53:07 -0800 | [diff] [blame] | 75 | OPENSSL_memcpy(out, &v, sizeof(v)); |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 76 | } |
| 77 | |
Robert Sloan | d5c2215 | 2017-11-13 09:22:12 -0800 | [diff] [blame] | 78 | static inline size_t load_word_le(const void *in) { |
| 79 | size_t v; |
| 80 | OPENSSL_memcpy(&v, in, sizeof(v)); |
| 81 | return v; |
| 82 | } |
| 83 | |
| 84 | static inline void store_word_le(void *out, size_t v) { |
| 85 | OPENSSL_memcpy(out, &v, sizeof(v)); |
| 86 | } |
| 87 | |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 88 | // block128_f is the type of an AES block cipher implementation. |
| 89 | // |
| 90 | // Unlike upstream OpenSSL, it and the other functions in this file hard-code |
| 91 | // |AES_KEY|. It is undefined in C to call a function pointer with anything |
| 92 | // other than the original type. Thus we either must match |block128_f| to the |
| 93 | // type signature of |AES_encrypt| and friends or pass in |void*| wrapper |
| 94 | // functions. |
| 95 | // |
| 96 | // These functions are called exclusively with AES, so we use the former. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 97 | typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16], |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 98 | const AES_KEY *key); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 99 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 100 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 101 | // CTR. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 102 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 103 | // ctr128_f is the type of a function that performs CTR-mode encryption. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 104 | typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 105 | const AES_KEY *key, const uint8_t ivec[16]); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 106 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 107 | // CRYPTO_ctr128_encrypt encrypts (or decrypts, it's the same in CTR mode) |
| 108 | // |len| bytes from |in| to |out| using |block| in counter mode. There's no |
| 109 | // requirement that |len| be a multiple of any value and any partial blocks are |
| 110 | // stored in |ecount_buf| and |*num|, which must be zeroed before the initial |
| 111 | // call. The counter is a 128-bit, big-endian value in |ivec| and is |
| 112 | // incremented by this function. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 113 | void CRYPTO_ctr128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 114 | const AES_KEY *key, uint8_t ivec[16], |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 115 | uint8_t ecount_buf[16], unsigned *num, |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 116 | block128_f block); |
| 117 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 118 | // CRYPTO_ctr128_encrypt_ctr32 acts like |CRYPTO_ctr128_encrypt| but takes |
| 119 | // |ctr|, a function that performs CTR mode but only deals with the lower 32 |
| 120 | // bits of the counter. This is useful when |ctr| can be an optimised |
| 121 | // function. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 122 | void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 123 | const AES_KEY *key, uint8_t ivec[16], |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 124 | uint8_t ecount_buf[16], unsigned *num, |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 125 | ctr128_f ctr); |
| 126 | |
| 127 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 128 | // GCM. |
| 129 | // |
| 130 | // This API differs from the upstream API slightly. The |GCM128_CONTEXT| does |
| 131 | // not have a |key| pointer that points to the key as upstream's version does. |
| 132 | // Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT| |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 133 | // can be safely copied. Additionally, |gcm_key| is split into a separate |
| 134 | // struct. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 135 | |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 136 | typedef struct { uint64_t hi,lo; } u128; |
| 137 | |
| 138 | // gmult_func multiplies |Xi| by the GCM key and writes the result back to |
| 139 | // |Xi|. |
| 140 | typedef void (*gmult_func)(uint64_t Xi[2], const u128 Htable[16]); |
| 141 | |
| 142 | // ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from |
| 143 | // |inp|. The result is written back to |Xi| and the |len| argument must be a |
| 144 | // multiple of 16. |
| 145 | typedef void (*ghash_func)(uint64_t Xi[2], const u128 Htable[16], |
| 146 | const uint8_t *inp, size_t len); |
| 147 | |
| 148 | typedef struct gcm128_key_st { |
| 149 | // Note the MOVBE-based, x86-64, GHASH assembly requires |H| and |Htable| to |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 150 | // be the first two elements of this struct. Additionally, some assembly |
| 151 | // routines require a 16-byte-aligned |Htable| when hashing data, but not |
| 152 | // initialization. |GCM128_KEY| is not itself aligned to simplify embedding in |
| 153 | // |EVP_AEAD_CTX|, but |Htable|'s offset must be a multiple of 16. |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 154 | u128 H; |
| 155 | u128 Htable[16]; |
| 156 | gmult_func gmult; |
| 157 | ghash_func ghash; |
| 158 | |
| 159 | block128_f block; |
| 160 | |
| 161 | // use_aesni_gcm_crypt is true if this context should use the assembly |
| 162 | // functions |aesni_gcm_encrypt| and |aesni_gcm_decrypt| to process data. |
| 163 | unsigned use_aesni_gcm_crypt:1; |
| 164 | } GCM128_KEY; |
| 165 | |
| 166 | // GCM128_CONTEXT contains state for a single GCM operation. The structure |
| 167 | // should be zero-initialized before use. |
| 168 | typedef struct { |
| 169 | // The following 5 names follow names in GCM specification |
| 170 | union { |
| 171 | uint64_t u[2]; |
| 172 | uint32_t d[4]; |
| 173 | uint8_t c[16]; |
| 174 | size_t t[16 / sizeof(size_t)]; |
| 175 | } Yi, EKi, EK0, len, Xi; |
| 176 | |
| 177 | // Note that the order of |Xi| and |gcm_key| is fixed by the MOVBE-based, |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 178 | // x86-64, GHASH assembly. Additionally, some assembly routines require |
| 179 | // |gcm_key| to be 16-byte aligned. |GCM128_KEY| is not itself aligned to |
| 180 | // simplify embedding in |EVP_AEAD_CTX|. |
| 181 | alignas(16) GCM128_KEY gcm_key; |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 182 | |
| 183 | unsigned mres, ares; |
| 184 | } GCM128_CONTEXT; |
| 185 | |
| 186 | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
| 187 | // crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is |
| 188 | // used. |
| 189 | int crypto_gcm_clmul_enabled(void); |
| 190 | #endif |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 191 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 192 | // CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to |
| 193 | // |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware |
| 194 | // accelerated) functions for performing operations in the GHASH field. If the |
| 195 | // AVX implementation was used |*out_is_avx| will be true. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 196 | void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash, |
Robert Sloan | 9254e68 | 2017-04-24 09:42:06 -0700 | [diff] [blame] | 197 | u128 *out_key, u128 out_table[16], int *out_is_avx, |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 198 | const uint8_t gcm_key[16]); |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 199 | |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 200 | // CRYPTO_gcm128_init_key initialises |gcm_key| to use |block| (typically AES) |
| 201 | // with the given key. |block_is_hwaes| is one if |block| is |aes_hw_encrypt|. |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 202 | OPENSSL_EXPORT void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key, |
| 203 | const AES_KEY *key, block128_f block, |
Robert Sloan | f573be7 | 2018-09-17 15:29:11 -0700 | [diff] [blame] | 204 | int block_is_hwaes); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 205 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 206 | // CRYPTO_gcm128_setiv sets the IV (nonce) for |ctx|. The |key| must be the |
| 207 | // same key that was passed to |CRYPTO_gcm128_init|. |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 208 | OPENSSL_EXPORT void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key, |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 209 | const uint8_t *iv, size_t iv_len); |
| 210 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 211 | // CRYPTO_gcm128_aad sets the authenticated data for an instance of GCM. |
| 212 | // This must be called before and data is encrypted. It returns one on success |
| 213 | // and zero otherwise. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 214 | OPENSSL_EXPORT int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, |
| 215 | size_t len); |
| 216 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 217 | // CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. The |key| |
| 218 | // must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one |
| 219 | // on success and zero otherwise. |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 220 | OPENSSL_EXPORT int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, |
| 221 | const AES_KEY *key, const uint8_t *in, |
| 222 | uint8_t *out, size_t len); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 223 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 224 | // CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. The |key| |
| 225 | // must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one |
| 226 | // on success and zero otherwise. |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 227 | OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, |
| 228 | const AES_KEY *key, const uint8_t *in, |
| 229 | uint8_t *out, size_t len); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 230 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 231 | // CRYPTO_gcm128_encrypt_ctr32 encrypts |len| bytes from |in| to |out| using |
| 232 | // a CTR function that only handles the bottom 32 bits of the nonce, like |
| 233 | // |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was |
| 234 | // passed to |CRYPTO_gcm128_init|. It returns one on success and zero |
| 235 | // otherwise. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 236 | OPENSSL_EXPORT int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 237 | const AES_KEY *key, |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 238 | const uint8_t *in, uint8_t *out, |
| 239 | size_t len, ctr128_f stream); |
| 240 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 241 | // CRYPTO_gcm128_decrypt_ctr32 decrypts |len| bytes from |in| to |out| using |
| 242 | // a CTR function that only handles the bottom 32 bits of the nonce, like |
| 243 | // |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was |
| 244 | // passed to |CRYPTO_gcm128_init|. It returns one on success and zero |
| 245 | // otherwise. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 246 | OPENSSL_EXPORT int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 247 | const AES_KEY *key, |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 248 | const uint8_t *in, uint8_t *out, |
| 249 | size_t len, ctr128_f stream); |
| 250 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 251 | // CRYPTO_gcm128_finish calculates the authenticator and compares it against |
| 252 | // |len| bytes of |tag|. It returns one on success and zero otherwise. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 253 | OPENSSL_EXPORT int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, |
| 254 | size_t len); |
| 255 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 256 | // CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|. |
| 257 | // The minimum of |len| and 16 bytes are copied into |tag|. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 258 | OPENSSL_EXPORT void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, uint8_t *tag, |
| 259 | size_t len); |
| 260 | |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 261 | |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 262 | // GCM assembly. |
| 263 | |
| 264 | #if !defined(OPENSSL_NO_ASM) && \ |
| 265 | (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \ |
| 266 | defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) || \ |
| 267 | defined(OPENSSL_PPC64LE)) |
| 268 | #define GHASH_ASM |
| 269 | #endif |
| 270 | |
| 271 | void gcm_init_4bit(u128 Htable[16], const uint64_t H[2]); |
| 272 | void gcm_gmult_4bit(uint64_t Xi[2], const u128 Htable[16]); |
| 273 | void gcm_ghash_4bit(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 274 | size_t len); |
| 275 | |
| 276 | #if defined(GHASH_ASM) |
| 277 | |
| 278 | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
| 279 | #define GCM_FUNCREF_4BIT |
| 280 | void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]); |
| 281 | void gcm_gmult_clmul(uint64_t Xi[2], const u128 Htable[16]); |
| 282 | void gcm_ghash_clmul(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 283 | size_t len); |
| 284 | |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 285 | OPENSSL_INLINE char gcm_ssse3_capable(void) { |
| 286 | return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0; |
| 287 | } |
| 288 | |
| 289 | // |gcm_gmult_ssse3| and |gcm_ghash_ssse3| require |Htable| to be |
| 290 | // 16-byte-aligned, but |gcm_init_ssse3| does not. |
| 291 | void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]); |
| 292 | void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]); |
| 293 | void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, |
| 294 | size_t len); |
| 295 | |
Robert Sloan | 8967815 | 2019-03-12 14:24:00 -0700 | [diff] [blame] | 296 | #if defined(OPENSSL_X86_64) |
| 297 | #define GHASH_ASM_X86_64 |
| 298 | void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]); |
| 299 | void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]); |
| 300 | void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, |
| 301 | size_t len); |
| 302 | |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 303 | #define AESNI_GCM |
| 304 | size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
| 305 | const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi); |
| 306 | size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
| 307 | const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi); |
| 308 | #endif // OPENSSL_X86_64 |
| 309 | |
| 310 | #if defined(OPENSSL_X86) |
| 311 | #define GHASH_ASM_X86 |
| 312 | void gcm_gmult_4bit_mmx(uint64_t Xi[2], const u128 Htable[16]); |
| 313 | void gcm_ghash_4bit_mmx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 314 | size_t len); |
| 315 | #endif // OPENSSL_X86 |
| 316 | |
| 317 | #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) |
| 318 | #define GHASH_ASM_ARM |
| 319 | #define GCM_FUNCREF_4BIT |
| 320 | |
| 321 | OPENSSL_INLINE int gcm_pmull_capable(void) { |
| 322 | return CRYPTO_is_ARMv8_PMULL_capable(); |
| 323 | } |
| 324 | |
| 325 | void gcm_init_v8(u128 Htable[16], const uint64_t Xi[2]); |
| 326 | void gcm_gmult_v8(uint64_t Xi[2], const u128 Htable[16]); |
| 327 | void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 328 | size_t len); |
| 329 | |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 330 | OPENSSL_INLINE int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); } |
| 331 | |
| 332 | void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]); |
| 333 | void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]); |
| 334 | void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 335 | size_t len); |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 336 | |
| 337 | #elif defined(OPENSSL_PPC64LE) |
| 338 | #define GHASH_ASM_PPC64LE |
| 339 | #define GCM_FUNCREF_4BIT |
| 340 | void gcm_init_p8(u128 Htable[16], const uint64_t Xi[2]); |
| 341 | void gcm_gmult_p8(uint64_t Xi[2], const u128 Htable[16]); |
| 342 | void gcm_ghash_p8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, |
| 343 | size_t len); |
| 344 | #endif |
| 345 | #endif // GHASH_ASM |
| 346 | |
| 347 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 348 | // CBC. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 349 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 350 | // cbc128_f is the type of a function that performs CBC-mode encryption. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 351 | typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 352 | const AES_KEY *key, uint8_t ivec[16], int enc); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 353 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 354 | // CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the |
| 355 | // given IV and block cipher in CBC mode. The input need not be a multiple of |
| 356 | // 128 bits long, but the output will round up to the nearest 128 bit multiple, |
| 357 | // zero padding the input if needed. The IV will be updated on return. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 358 | void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 359 | const AES_KEY *key, uint8_t ivec[16], |
| 360 | block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 361 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 362 | // CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the |
| 363 | // given IV and block cipher in CBC mode. If |len| is not a multiple of 128 |
| 364 | // bits then only that many bytes will be written, but a multiple of 128 bits |
| 365 | // is always read from |in|. The IV will be updated on return. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 366 | void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 367 | const AES_KEY *key, uint8_t ivec[16], |
| 368 | block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 369 | |
| 370 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 371 | // OFB. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 372 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 373 | // CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode) |
| 374 | // |len| bytes from |in| to |out| using |block| in OFB mode. There's no |
| 375 | // requirement that |len| be a multiple of any value and any partial blocks are |
| 376 | // stored in |ivec| and |*num|, the latter must be zero before the initial |
| 377 | // call. |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 378 | void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
| 379 | const AES_KEY *key, uint8_t ivec[16], unsigned *num, |
| 380 | block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 381 | |
| 382 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 383 | // CFB. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 384 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 385 | // CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
| 386 | // from |in| to |out| using |block| in CFB mode. There's no requirement that |
| 387 | // |len| be a multiple of any value and any partial blocks are stored in |ivec| |
| 388 | // and |*num|, the latter must be zero before the initial call. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 389 | void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 390 | const AES_KEY *key, uint8_t ivec[16], unsigned *num, |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 391 | int enc, block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 392 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 393 | // CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
| 394 | // from |in| to |out| using |block| in CFB-8 mode. Prior to the first call |
| 395 | // |num| should be set to zero. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 396 | void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 397 | const AES_KEY *key, uint8_t ivec[16], |
| 398 | unsigned *num, int enc, block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 399 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 400 | // CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
| 401 | // from |in| to |out| using |block| in CFB-1 mode. Prior to the first call |
| 402 | // |num| should be set to zero. |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 403 | void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 404 | const AES_KEY *key, uint8_t ivec[16], |
| 405 | unsigned *num, int enc, block128_f block); |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 406 | |
| 407 | size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len, |
Robert Sloan | f068def | 2018-10-10 18:45:40 -0700 | [diff] [blame] | 408 | const AES_KEY *key, uint8_t ivec[16], |
Kenny Root | e99801b | 2015-11-06 15:31:15 -0800 | [diff] [blame] | 409 | block128_f block); |
| 410 | |
| 411 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 412 | // POLYVAL. |
| 413 | // |
| 414 | // POLYVAL is a polynomial authenticator that operates over a field very |
| 415 | // similar to the one that GHASH uses. See |
| 416 | // https://tools.ietf.org/html/draft-irtf-cfrg-gcmsiv-02#section-3. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 417 | |
| 418 | typedef union { |
| 419 | uint64_t u[2]; |
| 420 | uint8_t c[16]; |
| 421 | } polyval_block; |
| 422 | |
| 423 | struct polyval_ctx { |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 424 | // Note that the order of |S|, |H| and |Htable| is fixed by the MOVBE-based, |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 425 | // x86-64, GHASH assembly. Additionally, some assembly routines require |
| 426 | // |Htable| to be 16-byte aligned. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 427 | polyval_block S; |
Steven Valdez | b0b45c6 | 2017-01-17 16:23:54 -0500 | [diff] [blame] | 428 | u128 H; |
Robert Sloan | 4c22c5f | 2019-03-01 15:53:37 -0800 | [diff] [blame] | 429 | alignas(16) u128 Htable[16]; |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 430 | gmult_func gmult; |
| 431 | ghash_func ghash; |
| 432 | }; |
| 433 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 434 | // CRYPTO_POLYVAL_init initialises |ctx| using |key|. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 435 | void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]); |
| 436 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 437 | // CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the |
| 438 | // blocks from |in|. Only a whole number of blocks can be processed so |in_len| |
| 439 | // must be a multiple of 16. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 440 | void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in, |
| 441 | size_t in_len); |
| 442 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 443 | // CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|. |
David Benjamin | 1b24967 | 2016-12-06 18:25:50 -0500 | [diff] [blame] | 444 | void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]); |
| 445 | |
David Benjamin | 4969cc9 | 2016-04-22 15:02:23 -0400 | [diff] [blame] | 446 | |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 447 | #if defined(__cplusplus) |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 448 | } // extern C |
Adam Langley | d9e397b | 2015-01-22 14:27:53 -0800 | [diff] [blame] | 449 | #endif |
| 450 | |
Robert Sloan | 8f860b1 | 2017-08-28 07:37:06 -0700 | [diff] [blame] | 451 | #endif // OPENSSL_HEADER_MODES_INTERNAL_H |