blob: 868da54b18508c1827096bee736d4497dd79c92c [file] [log] [blame]
Michal Ludvig6c833272006-07-12 12:29:38 +10001/*
2 * Cryptographic API.
3 *
4 * Support for VIA PadLock hardware crypto engine.
5 *
6 * Copyright (c) 2006 Michal Ludvig <michal@logix.cz>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 */
14
Herbert Xu7d024602009-07-10 17:26:44 +080015#include <crypto/internal/hash.h>
Jan Glauber5265eeb2007-10-09 22:43:13 +080016#include <crypto/sha.h>
Herbert Xu60104392006-08-26 18:34:10 +100017#include <linux/err.h>
Michal Ludvig6c833272006-07-12 12:29:38 +100018#include <linux/module.h>
19#include <linux/init.h>
20#include <linux/errno.h>
Michal Ludvig6c833272006-07-12 12:29:38 +100021#include <linux/interrupt.h>
22#include <linux/kernel.h>
23#include <linux/scatterlist.h>
Suresh Siddhae4914012008-08-13 22:02:26 +100024#include <asm/i387.h>
Michal Ludvig6c833272006-07-12 12:29:38 +100025#include "padlock.h"
26
Michal Ludvig6c833272006-07-12 12:29:38 +100027struct padlock_sha_ctx {
28 char *data;
29 size_t used;
30 int bypass;
31 void (*f_sha_padlock)(const char *in, char *out, int count);
Herbert Xu7d024602009-07-10 17:26:44 +080032 struct shash_desc *fallback;
Michal Ludvig6c833272006-07-12 12:29:38 +100033};
34
35static inline struct padlock_sha_ctx *ctx(struct crypto_tfm *tfm)
36{
Herbert Xu60104392006-08-26 18:34:10 +100037 return crypto_tfm_ctx(tfm);
Michal Ludvig6c833272006-07-12 12:29:38 +100038}
39
40/* We'll need aligned address on the stack */
41#define NEAREST_ALIGNED(ptr) \
42 ((void *)ALIGN((size_t)(ptr), PADLOCK_ALIGNMENT))
43
44static struct crypto_alg sha1_alg, sha256_alg;
45
Herbert Xu7d024602009-07-10 17:26:44 +080046static int padlock_sha_bypass(struct crypto_tfm *tfm)
Michal Ludvig6c833272006-07-12 12:29:38 +100047{
Herbert Xu7d024602009-07-10 17:26:44 +080048 int err = 0;
49
Michal Ludvig6c833272006-07-12 12:29:38 +100050 if (ctx(tfm)->bypass)
Herbert Xu7d024602009-07-10 17:26:44 +080051 goto out;
Michal Ludvig6c833272006-07-12 12:29:38 +100052
Herbert Xu7d024602009-07-10 17:26:44 +080053 err = crypto_shash_init(ctx(tfm)->fallback);
54 if (err)
55 goto out;
Michal Ludvig6c833272006-07-12 12:29:38 +100056
Herbert Xu7d024602009-07-10 17:26:44 +080057 if (ctx(tfm)->data && ctx(tfm)->used)
58 err = crypto_shash_update(ctx(tfm)->fallback, ctx(tfm)->data,
59 ctx(tfm)->used);
Michal Ludvig6c833272006-07-12 12:29:38 +100060
61 ctx(tfm)->used = 0;
62 ctx(tfm)->bypass = 1;
Herbert Xu7d024602009-07-10 17:26:44 +080063
64out:
65 return err;
Michal Ludvig6c833272006-07-12 12:29:38 +100066}
67
68static void padlock_sha_init(struct crypto_tfm *tfm)
69{
70 ctx(tfm)->used = 0;
71 ctx(tfm)->bypass = 0;
72}
73
74static void padlock_sha_update(struct crypto_tfm *tfm,
75 const uint8_t *data, unsigned int length)
76{
Herbert Xu7d024602009-07-10 17:26:44 +080077 int err;
78
Michal Ludvig6c833272006-07-12 12:29:38 +100079 /* Our buffer is always one page. */
80 if (unlikely(!ctx(tfm)->bypass &&
Herbert Xu7d024602009-07-10 17:26:44 +080081 (ctx(tfm)->used + length > PAGE_SIZE))) {
82 err = padlock_sha_bypass(tfm);
83 BUG_ON(err);
84 }
Michal Ludvig6c833272006-07-12 12:29:38 +100085
86 if (unlikely(ctx(tfm)->bypass)) {
Herbert Xu7d024602009-07-10 17:26:44 +080087 err = crypto_shash_update(ctx(tfm)->fallback, data, length);
88 BUG_ON(err);
Michal Ludvig6c833272006-07-12 12:29:38 +100089 return;
90 }
91
92 memcpy(ctx(tfm)->data + ctx(tfm)->used, data, length);
93 ctx(tfm)->used += length;
94}
95
96static inline void padlock_output_block(uint32_t *src,
97 uint32_t *dst, size_t count)
98{
99 while (count--)
100 *dst++ = swab32(*src++);
101}
102
Adrian Bunkcb175302006-07-15 11:31:25 +1000103static void padlock_do_sha1(const char *in, char *out, int count)
Michal Ludvig6c833272006-07-12 12:29:38 +1000104{
105 /* We can't store directly to *out as it may be unaligned. */
106 /* BTW Don't reduce the buffer size below 128 Bytes!
107 * PadLock microcode needs it that big. */
108 char buf[128+16];
109 char *result = NEAREST_ALIGNED(buf);
Suresh Siddhae4914012008-08-13 22:02:26 +1000110 int ts_state;
Michal Ludvig6c833272006-07-12 12:29:38 +1000111
Jan Glauber5265eeb2007-10-09 22:43:13 +0800112 ((uint32_t *)result)[0] = SHA1_H0;
113 ((uint32_t *)result)[1] = SHA1_H1;
114 ((uint32_t *)result)[2] = SHA1_H2;
115 ((uint32_t *)result)[3] = SHA1_H3;
116 ((uint32_t *)result)[4] = SHA1_H4;
Michal Ludvig6c833272006-07-12 12:29:38 +1000117
Suresh Siddhae4914012008-08-13 22:02:26 +1000118 /* prevent taking the spurious DNA fault with padlock. */
119 ts_state = irq_ts_save();
Michal Ludvig6c833272006-07-12 12:29:38 +1000120 asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
121 : "+S"(in), "+D"(result)
122 : "c"(count), "a"(0));
Suresh Siddhae4914012008-08-13 22:02:26 +1000123 irq_ts_restore(ts_state);
Michal Ludvig6c833272006-07-12 12:29:38 +1000124
125 padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);
126}
127
Adrian Bunkcb175302006-07-15 11:31:25 +1000128static void padlock_do_sha256(const char *in, char *out, int count)
Michal Ludvig6c833272006-07-12 12:29:38 +1000129{
130 /* We can't store directly to *out as it may be unaligned. */
131 /* BTW Don't reduce the buffer size below 128 Bytes!
132 * PadLock microcode needs it that big. */
133 char buf[128+16];
134 char *result = NEAREST_ALIGNED(buf);
Suresh Siddhae4914012008-08-13 22:02:26 +1000135 int ts_state;
Michal Ludvig6c833272006-07-12 12:29:38 +1000136
Jan Glauber5265eeb2007-10-09 22:43:13 +0800137 ((uint32_t *)result)[0] = SHA256_H0;
138 ((uint32_t *)result)[1] = SHA256_H1;
139 ((uint32_t *)result)[2] = SHA256_H2;
140 ((uint32_t *)result)[3] = SHA256_H3;
141 ((uint32_t *)result)[4] = SHA256_H4;
142 ((uint32_t *)result)[5] = SHA256_H5;
143 ((uint32_t *)result)[6] = SHA256_H6;
144 ((uint32_t *)result)[7] = SHA256_H7;
Michal Ludvig6c833272006-07-12 12:29:38 +1000145
Suresh Siddhae4914012008-08-13 22:02:26 +1000146 /* prevent taking the spurious DNA fault with padlock. */
147 ts_state = irq_ts_save();
Michal Ludvig6c833272006-07-12 12:29:38 +1000148 asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
149 : "+S"(in), "+D"(result)
150 : "c"(count), "a"(0));
Suresh Siddhae4914012008-08-13 22:02:26 +1000151 irq_ts_restore(ts_state);
Michal Ludvig6c833272006-07-12 12:29:38 +1000152
153 padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);
154}
155
156static void padlock_sha_final(struct crypto_tfm *tfm, uint8_t *out)
157{
Herbert Xu7d024602009-07-10 17:26:44 +0800158 int err;
159
Michal Ludvig6c833272006-07-12 12:29:38 +1000160 if (unlikely(ctx(tfm)->bypass)) {
Herbert Xu7d024602009-07-10 17:26:44 +0800161 err = crypto_shash_final(ctx(tfm)->fallback, out);
162 BUG_ON(err);
Michal Ludvig6c833272006-07-12 12:29:38 +1000163 ctx(tfm)->bypass = 0;
164 return;
165 }
166
167 /* Pass the input buffer to PadLock microcode... */
168 ctx(tfm)->f_sha_padlock(ctx(tfm)->data, out, ctx(tfm)->used);
169
170 ctx(tfm)->used = 0;
171}
172
Herbert Xu60104392006-08-26 18:34:10 +1000173static int padlock_cra_init(struct crypto_tfm *tfm)
Michal Ludvig6c833272006-07-12 12:29:38 +1000174{
Herbert Xu60104392006-08-26 18:34:10 +1000175 const char *fallback_driver_name = tfm->__crt_alg->cra_name;
Herbert Xu7d024602009-07-10 17:26:44 +0800176 struct crypto_shash *fallback_tfm;
177 int err = -ENOMEM;
Herbert Xu60104392006-08-26 18:34:10 +1000178
Michal Ludvig6c833272006-07-12 12:29:38 +1000179 /* For now we'll allocate one page. This
180 * could eventually be configurable one day. */
181 ctx(tfm)->data = (char *)__get_free_page(GFP_KERNEL);
182 if (!ctx(tfm)->data)
Herbert Xu7d024602009-07-10 17:26:44 +0800183 goto out;
Michal Ludvig6c833272006-07-12 12:29:38 +1000184
185 /* Allocate a fallback and abort if it failed. */
Herbert Xu7d024602009-07-10 17:26:44 +0800186 fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
187 CRYPTO_ALG_NEED_FALLBACK);
Herbert Xu60104392006-08-26 18:34:10 +1000188 if (IS_ERR(fallback_tfm)) {
Michal Ludvig6c833272006-07-12 12:29:38 +1000189 printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
190 fallback_driver_name);
Herbert Xu7d024602009-07-10 17:26:44 +0800191 err = PTR_ERR(fallback_tfm);
192 goto out_free_page;
Michal Ludvig6c833272006-07-12 12:29:38 +1000193 }
194
Herbert Xu7d024602009-07-10 17:26:44 +0800195 ctx(tfm)->fallback = kmalloc(sizeof(struct shash_desc) +
196 crypto_shash_descsize(fallback_tfm),
197 GFP_KERNEL);
198 if (!ctx(tfm)->fallback)
199 goto out_free_tfm;
200
201 ctx(tfm)->fallback->tfm = fallback_tfm;
202 ctx(tfm)->fallback->flags = 0;
Michal Ludvig6c833272006-07-12 12:29:38 +1000203 return 0;
Herbert Xu7d024602009-07-10 17:26:44 +0800204
205out_free_tfm:
206 crypto_free_shash(fallback_tfm);
207out_free_page:
208 free_page((unsigned long)(ctx(tfm)->data));
209out:
210 return err;
Michal Ludvig6c833272006-07-12 12:29:38 +1000211}
212
213static int padlock_sha1_cra_init(struct crypto_tfm *tfm)
214{
215 ctx(tfm)->f_sha_padlock = padlock_do_sha1;
216
Herbert Xu60104392006-08-26 18:34:10 +1000217 return padlock_cra_init(tfm);
Michal Ludvig6c833272006-07-12 12:29:38 +1000218}
219
220static int padlock_sha256_cra_init(struct crypto_tfm *tfm)
221{
222 ctx(tfm)->f_sha_padlock = padlock_do_sha256;
223
Herbert Xu60104392006-08-26 18:34:10 +1000224 return padlock_cra_init(tfm);
Michal Ludvig6c833272006-07-12 12:29:38 +1000225}
226
227static void padlock_cra_exit(struct crypto_tfm *tfm)
228{
229 if (ctx(tfm)->data) {
230 free_page((unsigned long)(ctx(tfm)->data));
231 ctx(tfm)->data = NULL;
232 }
233
Herbert Xu7d024602009-07-10 17:26:44 +0800234 crypto_free_shash(ctx(tfm)->fallback->tfm);
235
236 kzfree(ctx(tfm)->fallback);
Michal Ludvig6c833272006-07-12 12:29:38 +1000237}
238
239static struct crypto_alg sha1_alg = {
240 .cra_name = "sha1",
241 .cra_driver_name = "sha1-padlock",
242 .cra_priority = PADLOCK_CRA_PRIORITY,
Herbert Xu60104392006-08-26 18:34:10 +1000243 .cra_flags = CRYPTO_ALG_TYPE_DIGEST |
244 CRYPTO_ALG_NEED_FALLBACK,
Jan Glauber5265eeb2007-10-09 22:43:13 +0800245 .cra_blocksize = SHA1_BLOCK_SIZE,
Michal Ludvig6c833272006-07-12 12:29:38 +1000246 .cra_ctxsize = sizeof(struct padlock_sha_ctx),
247 .cra_module = THIS_MODULE,
248 .cra_list = LIST_HEAD_INIT(sha1_alg.cra_list),
249 .cra_init = padlock_sha1_cra_init,
250 .cra_exit = padlock_cra_exit,
251 .cra_u = {
252 .digest = {
253 .dia_digestsize = SHA1_DIGEST_SIZE,
254 .dia_init = padlock_sha_init,
255 .dia_update = padlock_sha_update,
256 .dia_final = padlock_sha_final,
257 }
258 }
259};
260
261static struct crypto_alg sha256_alg = {
262 .cra_name = "sha256",
263 .cra_driver_name = "sha256-padlock",
264 .cra_priority = PADLOCK_CRA_PRIORITY,
Herbert Xu60104392006-08-26 18:34:10 +1000265 .cra_flags = CRYPTO_ALG_TYPE_DIGEST |
266 CRYPTO_ALG_NEED_FALLBACK,
Jan Glauber5265eeb2007-10-09 22:43:13 +0800267 .cra_blocksize = SHA256_BLOCK_SIZE,
Michal Ludvig6c833272006-07-12 12:29:38 +1000268 .cra_ctxsize = sizeof(struct padlock_sha_ctx),
269 .cra_module = THIS_MODULE,
270 .cra_list = LIST_HEAD_INIT(sha256_alg.cra_list),
271 .cra_init = padlock_sha256_cra_init,
272 .cra_exit = padlock_cra_exit,
273 .cra_u = {
274 .digest = {
275 .dia_digestsize = SHA256_DIGEST_SIZE,
276 .dia_init = padlock_sha_init,
277 .dia_update = padlock_sha_update,
278 .dia_final = padlock_sha_final,
279 }
280 }
281};
282
Michal Ludvig6c833272006-07-12 12:29:38 +1000283static int __init padlock_init(void)
284{
285 int rc = -ENODEV;
286
287 if (!cpu_has_phe) {
Jeremy Katzb43e7262008-07-03 19:03:31 +0800288 printk(KERN_NOTICE PFX "VIA PadLock Hash Engine not detected.\n");
Michal Ludvig6c833272006-07-12 12:29:38 +1000289 return -ENODEV;
290 }
291
292 if (!cpu_has_phe_enabled) {
Jeremy Katzb43e7262008-07-03 19:03:31 +0800293 printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
Michal Ludvig6c833272006-07-12 12:29:38 +1000294 return -ENODEV;
295 }
296
Michal Ludvig6c833272006-07-12 12:29:38 +1000297 rc = crypto_register_alg(&sha1_alg);
298 if (rc)
299 goto out;
300
301 rc = crypto_register_alg(&sha256_alg);
302 if (rc)
303 goto out_unreg1;
304
305 printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
306
307 return 0;
308
309out_unreg1:
310 crypto_unregister_alg(&sha1_alg);
311out:
312 printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
313 return rc;
314}
315
316static void __exit padlock_fini(void)
317{
318 crypto_unregister_alg(&sha1_alg);
319 crypto_unregister_alg(&sha256_alg);
320}
321
322module_init(padlock_init);
323module_exit(padlock_fini);
324
325MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
326MODULE_LICENSE("GPL");
327MODULE_AUTHOR("Michal Ludvig");
328
Herbert Xua760a662009-02-26 14:06:31 +0800329MODULE_ALIAS("sha1-all");
330MODULE_ALIAS("sha256-all");
Michal Ludvig6c833272006-07-12 12:29:38 +1000331MODULE_ALIAS("sha1-padlock");
332MODULE_ALIAS("sha256-padlock");