blob: a2226f8419609c461cab422337fd604ded53c8ee [file] [log] [blame]
Ard Biesheuvel7918ece2016-11-20 11:42:01 +00001/*
2 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
3 *
4 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 */
12
13#include <asm/hwcap.h>
14#include <asm/neon.h>
15#include <asm/simd.h>
16#include <crypto/internal/hash.h>
17#include <crypto/sha.h>
18#include <crypto/sha256_base.h>
19#include <linux/cryptohash.h>
20#include <linux/types.h>
21#include <linux/string.h>
22
23MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
24MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
25MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
26MODULE_LICENSE("GPL v2");
27MODULE_ALIAS_CRYPTO("sha224");
28MODULE_ALIAS_CRYPTO("sha256");
29
30asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
31 unsigned int num_blks);
32
33asmlinkage void sha256_block_neon(u32 *digest, const void *data,
34 unsigned int num_blks);
35
36static int sha256_update(struct shash_desc *desc, const u8 *data,
37 unsigned int len)
38{
39 return sha256_base_do_update(desc, data, len,
40 (sha256_block_fn *)sha256_block_data_order);
41}
42
43static int sha256_finup(struct shash_desc *desc, const u8 *data,
44 unsigned int len, u8 *out)
45{
46 if (len)
47 sha256_base_do_update(desc, data, len,
48 (sha256_block_fn *)sha256_block_data_order);
49 sha256_base_do_finalize(desc,
50 (sha256_block_fn *)sha256_block_data_order);
51
52 return sha256_base_finish(desc, out);
53}
54
55static int sha256_final(struct shash_desc *desc, u8 *out)
56{
57 return sha256_finup(desc, NULL, 0, out);
58}
59
60static struct shash_alg algs[] = { {
61 .digestsize = SHA256_DIGEST_SIZE,
62 .init = sha256_base_init,
63 .update = sha256_update,
64 .final = sha256_final,
65 .finup = sha256_finup,
66 .descsize = sizeof(struct sha256_state),
67 .base.cra_name = "sha256",
68 .base.cra_driver_name = "sha256-arm64",
69 .base.cra_priority = 100,
70 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
71 .base.cra_blocksize = SHA256_BLOCK_SIZE,
72 .base.cra_module = THIS_MODULE,
73}, {
74 .digestsize = SHA224_DIGEST_SIZE,
75 .init = sha224_base_init,
76 .update = sha256_update,
77 .final = sha256_final,
78 .finup = sha256_finup,
79 .descsize = sizeof(struct sha256_state),
80 .base.cra_name = "sha224",
81 .base.cra_driver_name = "sha224-arm64",
82 .base.cra_priority = 100,
83 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
84 .base.cra_blocksize = SHA224_BLOCK_SIZE,
85 .base.cra_module = THIS_MODULE,
86} };
87
88static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
89 unsigned int len)
90{
91 /*
92 * Stacking and unstacking a substantial slice of the NEON register
93 * file may significantly affect performance for small updates when
94 * executing in interrupt context, so fall back to the scalar code
95 * in that case.
96 */
97 if (!may_use_simd())
98 return sha256_base_do_update(desc, data, len,
99 (sha256_block_fn *)sha256_block_data_order);
100
101 kernel_neon_begin();
102 sha256_base_do_update(desc, data, len,
103 (sha256_block_fn *)sha256_block_neon);
104 kernel_neon_end();
105
106 return 0;
107}
108
109static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
110 unsigned int len, u8 *out)
111{
112 if (!may_use_simd()) {
113 if (len)
114 sha256_base_do_update(desc, data, len,
115 (sha256_block_fn *)sha256_block_data_order);
116 sha256_base_do_finalize(desc,
117 (sha256_block_fn *)sha256_block_data_order);
118 } else {
119 kernel_neon_begin();
120 if (len)
121 sha256_base_do_update(desc, data, len,
122 (sha256_block_fn *)sha256_block_neon);
123 sha256_base_do_finalize(desc,
124 (sha256_block_fn *)sha256_block_neon);
125 kernel_neon_end();
126 }
127 return sha256_base_finish(desc, out);
128}
129
130static int sha256_final_neon(struct shash_desc *desc, u8 *out)
131{
132 return sha256_finup_neon(desc, NULL, 0, out);
133}
134
135static struct shash_alg neon_algs[] = { {
136 .digestsize = SHA256_DIGEST_SIZE,
137 .init = sha256_base_init,
138 .update = sha256_update_neon,
139 .final = sha256_final_neon,
140 .finup = sha256_finup_neon,
141 .descsize = sizeof(struct sha256_state),
142 .base.cra_name = "sha256",
143 .base.cra_driver_name = "sha256-arm64-neon",
144 .base.cra_priority = 150,
145 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
146 .base.cra_blocksize = SHA256_BLOCK_SIZE,
147 .base.cra_module = THIS_MODULE,
148}, {
149 .digestsize = SHA224_DIGEST_SIZE,
150 .init = sha224_base_init,
151 .update = sha256_update_neon,
152 .final = sha256_final_neon,
153 .finup = sha256_finup_neon,
154 .descsize = sizeof(struct sha256_state),
155 .base.cra_name = "sha224",
156 .base.cra_driver_name = "sha224-arm64-neon",
157 .base.cra_priority = 150,
158 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
159 .base.cra_blocksize = SHA224_BLOCK_SIZE,
160 .base.cra_module = THIS_MODULE,
161} };
162
163static int __init sha256_mod_init(void)
164{
165 int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
166 if (ret)
167 return ret;
168
169 if (elf_hwcap & HWCAP_ASIMD) {
170 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
171 if (ret)
172 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
173 }
174 return ret;
175}
176
177static void __exit sha256_mod_fini(void)
178{
179 if (elf_hwcap & HWCAP_ASIMD)
180 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
181 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
182}
183
184module_init(sha256_mod_init);
185module_exit(sha256_mod_fini);