Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 1 | /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc. |
| 2 | * |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License as published by |
| 5 | * the Free Software Foundation; either version 2 of the License, or |
| 6 | * (at your option) any later version. |
| 7 | */ |
| 8 | |
| 9 | #include <linux/module.h> |
| 10 | #include <linux/kernel.h> |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 11 | #include <linux/pci.h> |
| 12 | #include <linux/pci_ids.h> |
| 13 | #include <linux/crypto.h> |
| 14 | #include <linux/spinlock.h> |
| 15 | #include <crypto/algapi.h> |
Sebastian Siewior | 89e1265 | 2007-10-17 23:18:57 +0800 | [diff] [blame] | 16 | #include <crypto/aes.h> |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 17 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 18 | #include <linux/io.h> |
| 19 | #include <linux/delay.h> |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 20 | |
| 21 | #include "geode-aes.h" |
| 22 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 23 | /* Static structures */ |
| 24 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 25 | static void __iomem *_iobase; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 26 | static spinlock_t lock; |
| 27 | |
| 28 | /* Write a 128 bit field (either a writable key or IV) */ |
| 29 | static inline void |
| 30 | _writefield(u32 offset, void *value) |
| 31 | { |
| 32 | int i; |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 33 | for (i = 0; i < 4; i++) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 34 | iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); |
| 35 | } |
| 36 | |
| 37 | /* Read a 128 bit field (either a writable key or IV) */ |
| 38 | static inline void |
| 39 | _readfield(u32 offset, void *value) |
| 40 | { |
| 41 | int i; |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 42 | for (i = 0; i < 4; i++) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 43 | ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); |
| 44 | } |
| 45 | |
| 46 | static int |
| 47 | do_crypt(void *src, void *dst, int len, u32 flags) |
| 48 | { |
| 49 | u32 status; |
| 50 | u32 counter = AES_OP_TIMEOUT; |
| 51 | |
| 52 | iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG); |
| 53 | iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG); |
| 54 | iowrite32(len, _iobase + AES_LENA_REG); |
| 55 | |
| 56 | /* Start the operation */ |
| 57 | iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG); |
| 58 | |
Sebastian Siewior | 1f4e477 | 2007-10-21 16:18:12 +0800 | [diff] [blame] | 59 | do { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 60 | status = ioread32(_iobase + AES_INTR_REG); |
Sebastian Siewior | 1f4e477 | 2007-10-21 16:18:12 +0800 | [diff] [blame] | 61 | cpu_relax(); |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 62 | } while (!(status & AES_INTRA_PENDING) && --counter); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 63 | |
| 64 | /* Clear the event */ |
| 65 | iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG); |
| 66 | return counter ? 0 : 1; |
| 67 | } |
| 68 | |
Adrian Bunk | ab78270 | 2006-11-17 13:43:55 +1100 | [diff] [blame] | 69 | static unsigned int |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 70 | geode_aes_crypt(struct geode_aes_op *op) |
| 71 | { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 72 | u32 flags = 0; |
Alexey Dobriyan | 5efee17 | 2007-03-06 01:42:13 -0800 | [diff] [blame] | 73 | unsigned long iflags; |
Sebastian Siewior | 1f4e477 | 2007-10-21 16:18:12 +0800 | [diff] [blame] | 74 | int ret; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 75 | |
Jordan Crouse | 761e784 | 2007-05-24 21:23:24 +1000 | [diff] [blame] | 76 | if (op->len == 0) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 77 | return 0; |
| 78 | |
Jordan Crouse | 761e784 | 2007-05-24 21:23:24 +1000 | [diff] [blame] | 79 | /* If the source and destination is the same, then |
| 80 | * we need to turn on the coherent flags, otherwise |
| 81 | * we don't need to worry |
| 82 | */ |
| 83 | |
Sebastian Siewior | 2e21630 | 2007-11-10 19:37:49 +0800 | [diff] [blame] | 84 | flags |= (AES_CTRL_DCA | AES_CTRL_SCA); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 85 | |
| 86 | if (op->dir == AES_DIR_ENCRYPT) |
| 87 | flags |= AES_CTRL_ENCRYPT; |
| 88 | |
| 89 | /* Start the critical section */ |
| 90 | |
| 91 | spin_lock_irqsave(&lock, iflags); |
| 92 | |
| 93 | if (op->mode == AES_MODE_CBC) { |
| 94 | flags |= AES_CTRL_CBC; |
| 95 | _writefield(AES_WRITEIV0_REG, op->iv); |
| 96 | } |
| 97 | |
Jordan Crouse | 761e784 | 2007-05-24 21:23:24 +1000 | [diff] [blame] | 98 | if (!(op->flags & AES_FLAGS_HIDDENKEY)) { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 99 | flags |= AES_CTRL_WRKEY; |
| 100 | _writefield(AES_WRITEKEY0_REG, op->key); |
| 101 | } |
| 102 | |
Sebastian Siewior | 1f4e477 | 2007-10-21 16:18:12 +0800 | [diff] [blame] | 103 | ret = do_crypt(op->src, op->dst, op->len, flags); |
| 104 | BUG_ON(ret); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 105 | |
| 106 | if (op->mode == AES_MODE_CBC) |
| 107 | _readfield(AES_WRITEIV0_REG, op->iv); |
| 108 | |
| 109 | spin_unlock_irqrestore(&lock, iflags); |
| 110 | |
| 111 | return op->len; |
| 112 | } |
| 113 | |
| 114 | /* CRYPTO-API Functions */ |
| 115 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 116 | static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key, |
| 117 | unsigned int len) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 118 | { |
| 119 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 120 | unsigned int ret; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 121 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 122 | op->keylen = len; |
| 123 | |
| 124 | if (len == AES_KEYSIZE_128) { |
| 125 | memcpy(op->key, key, len); |
| 126 | return 0; |
| 127 | } |
| 128 | |
| 129 | if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { |
| 130 | /* not supported at all */ |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 131 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; |
| 132 | return -EINVAL; |
| 133 | } |
| 134 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 135 | /* |
| 136 | * The requested key size is not supported by HW, do a fallback |
| 137 | */ |
Roel Kluin | faad98f | 2010-01-08 14:19:21 +1100 | [diff] [blame] | 138 | op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; |
| 139 | op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 140 | |
| 141 | ret = crypto_cipher_setkey(op->fallback.cip, key, len); |
| 142 | if (ret) { |
| 143 | tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; |
Roel Kluin | e054f16 | 2010-02-04 11:39:13 +1100 | [diff] [blame] | 144 | tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 145 | } |
| 146 | return ret; |
| 147 | } |
| 148 | |
| 149 | static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key, |
| 150 | unsigned int len) |
| 151 | { |
| 152 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 153 | unsigned int ret; |
| 154 | |
| 155 | op->keylen = len; |
| 156 | |
| 157 | if (len == AES_KEYSIZE_128) { |
| 158 | memcpy(op->key, key, len); |
| 159 | return 0; |
| 160 | } |
| 161 | |
| 162 | if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { |
| 163 | /* not supported at all */ |
| 164 | tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; |
| 165 | return -EINVAL; |
| 166 | } |
| 167 | |
| 168 | /* |
| 169 | * The requested key size is not supported by HW, do a fallback |
| 170 | */ |
| 171 | op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; |
| 172 | op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); |
| 173 | |
| 174 | ret = crypto_blkcipher_setkey(op->fallback.blk, key, len); |
| 175 | if (ret) { |
| 176 | tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; |
| 177 | tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK); |
| 178 | } |
| 179 | return ret; |
| 180 | } |
| 181 | |
| 182 | static int fallback_blk_dec(struct blkcipher_desc *desc, |
| 183 | struct scatterlist *dst, struct scatterlist *src, |
| 184 | unsigned int nbytes) |
| 185 | { |
| 186 | unsigned int ret; |
| 187 | struct crypto_blkcipher *tfm; |
| 188 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 189 | |
| 190 | tfm = desc->tfm; |
| 191 | desc->tfm = op->fallback.blk; |
| 192 | |
Sebastian Siewior | fdc520a | 2007-12-10 15:48:17 +0800 | [diff] [blame] | 193 | ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 194 | |
| 195 | desc->tfm = tfm; |
| 196 | return ret; |
| 197 | } |
| 198 | static int fallback_blk_enc(struct blkcipher_desc *desc, |
| 199 | struct scatterlist *dst, struct scatterlist *src, |
| 200 | unsigned int nbytes) |
| 201 | { |
| 202 | unsigned int ret; |
| 203 | struct crypto_blkcipher *tfm; |
| 204 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 205 | |
| 206 | tfm = desc->tfm; |
| 207 | desc->tfm = op->fallback.blk; |
| 208 | |
Sebastian Siewior | fdc520a | 2007-12-10 15:48:17 +0800 | [diff] [blame] | 209 | ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 210 | |
| 211 | desc->tfm = tfm; |
| 212 | return ret; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 213 | } |
| 214 | |
| 215 | static void |
| 216 | geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) |
| 217 | { |
| 218 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 219 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 220 | if (unlikely(op->keylen != AES_KEYSIZE_128)) { |
| 221 | crypto_cipher_encrypt_one(op->fallback.cip, out, in); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 222 | return; |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 223 | } |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 224 | |
| 225 | op->src = (void *) in; |
| 226 | op->dst = (void *) out; |
| 227 | op->mode = AES_MODE_ECB; |
| 228 | op->flags = 0; |
| 229 | op->len = AES_MIN_BLOCK_SIZE; |
| 230 | op->dir = AES_DIR_ENCRYPT; |
| 231 | |
| 232 | geode_aes_crypt(op); |
| 233 | } |
| 234 | |
| 235 | |
| 236 | static void |
| 237 | geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) |
| 238 | { |
| 239 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 240 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 241 | if (unlikely(op->keylen != AES_KEYSIZE_128)) { |
| 242 | crypto_cipher_decrypt_one(op->fallback.cip, out, in); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 243 | return; |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 244 | } |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 245 | |
| 246 | op->src = (void *) in; |
| 247 | op->dst = (void *) out; |
| 248 | op->mode = AES_MODE_ECB; |
| 249 | op->flags = 0; |
| 250 | op->len = AES_MIN_BLOCK_SIZE; |
| 251 | op->dir = AES_DIR_DECRYPT; |
| 252 | |
| 253 | geode_aes_crypt(op); |
| 254 | } |
| 255 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 256 | static int fallback_init_cip(struct crypto_tfm *tfm) |
| 257 | { |
| 258 | const char *name = tfm->__crt_alg->cra_name; |
| 259 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 260 | |
| 261 | op->fallback.cip = crypto_alloc_cipher(name, 0, |
| 262 | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); |
| 263 | |
| 264 | if (IS_ERR(op->fallback.cip)) { |
| 265 | printk(KERN_ERR "Error allocating fallback algo %s\n", name); |
Roel Kluin | faad98f | 2010-01-08 14:19:21 +1100 | [diff] [blame] | 266 | return PTR_ERR(op->fallback.cip); |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 267 | } |
| 268 | |
| 269 | return 0; |
| 270 | } |
| 271 | |
| 272 | static void fallback_exit_cip(struct crypto_tfm *tfm) |
| 273 | { |
| 274 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 275 | |
| 276 | crypto_free_cipher(op->fallback.cip); |
| 277 | op->fallback.cip = NULL; |
| 278 | } |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 279 | |
| 280 | static struct crypto_alg geode_alg = { |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 281 | .cra_name = "aes", |
| 282 | .cra_driver_name = "geode-aes", |
| 283 | .cra_priority = 300, |
| 284 | .cra_alignmask = 15, |
| 285 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER | |
| 286 | CRYPTO_ALG_NEED_FALLBACK, |
| 287 | .cra_init = fallback_init_cip, |
| 288 | .cra_exit = fallback_exit_cip, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 289 | .cra_blocksize = AES_MIN_BLOCK_SIZE, |
| 290 | .cra_ctxsize = sizeof(struct geode_aes_op), |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 291 | .cra_module = THIS_MODULE, |
| 292 | .cra_list = LIST_HEAD_INIT(geode_alg.cra_list), |
| 293 | .cra_u = { |
| 294 | .cipher = { |
| 295 | .cia_min_keysize = AES_MIN_KEY_SIZE, |
| 296 | .cia_max_keysize = AES_MAX_KEY_SIZE, |
| 297 | .cia_setkey = geode_setkey_cip, |
| 298 | .cia_encrypt = geode_encrypt, |
| 299 | .cia_decrypt = geode_decrypt |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 300 | } |
| 301 | } |
| 302 | }; |
| 303 | |
| 304 | static int |
| 305 | geode_cbc_decrypt(struct blkcipher_desc *desc, |
| 306 | struct scatterlist *dst, struct scatterlist *src, |
| 307 | unsigned int nbytes) |
| 308 | { |
| 309 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 310 | struct blkcipher_walk walk; |
| 311 | int err, ret; |
| 312 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 313 | if (unlikely(op->keylen != AES_KEYSIZE_128)) |
| 314 | return fallback_blk_dec(desc, dst, src, nbytes); |
| 315 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 316 | blkcipher_walk_init(&walk, dst, src, nbytes); |
| 317 | err = blkcipher_walk_virt(desc, &walk); |
Sebastian Siewior | d2456c6 | 2007-11-30 16:36:57 +1100 | [diff] [blame] | 318 | op->iv = walk.iv; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 319 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 320 | while ((nbytes = walk.nbytes)) { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 321 | op->src = walk.src.virt.addr, |
| 322 | op->dst = walk.dst.virt.addr; |
| 323 | op->mode = AES_MODE_CBC; |
| 324 | op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); |
| 325 | op->dir = AES_DIR_DECRYPT; |
| 326 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 327 | ret = geode_aes_crypt(op); |
| 328 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 329 | nbytes -= ret; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 330 | err = blkcipher_walk_done(desc, &walk, nbytes); |
| 331 | } |
| 332 | |
| 333 | return err; |
| 334 | } |
| 335 | |
| 336 | static int |
| 337 | geode_cbc_encrypt(struct blkcipher_desc *desc, |
| 338 | struct scatterlist *dst, struct scatterlist *src, |
| 339 | unsigned int nbytes) |
| 340 | { |
| 341 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 342 | struct blkcipher_walk walk; |
| 343 | int err, ret; |
| 344 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 345 | if (unlikely(op->keylen != AES_KEYSIZE_128)) |
| 346 | return fallback_blk_enc(desc, dst, src, nbytes); |
| 347 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 348 | blkcipher_walk_init(&walk, dst, src, nbytes); |
| 349 | err = blkcipher_walk_virt(desc, &walk); |
Sebastian Siewior | d2456c6 | 2007-11-30 16:36:57 +1100 | [diff] [blame] | 350 | op->iv = walk.iv; |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 351 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 352 | while ((nbytes = walk.nbytes)) { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 353 | op->src = walk.src.virt.addr, |
| 354 | op->dst = walk.dst.virt.addr; |
| 355 | op->mode = AES_MODE_CBC; |
| 356 | op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); |
| 357 | op->dir = AES_DIR_ENCRYPT; |
| 358 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 359 | ret = geode_aes_crypt(op); |
| 360 | nbytes -= ret; |
| 361 | err = blkcipher_walk_done(desc, &walk, nbytes); |
| 362 | } |
| 363 | |
| 364 | return err; |
| 365 | } |
| 366 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 367 | static int fallback_init_blk(struct crypto_tfm *tfm) |
| 368 | { |
| 369 | const char *name = tfm->__crt_alg->cra_name; |
| 370 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 371 | |
| 372 | op->fallback.blk = crypto_alloc_blkcipher(name, 0, |
| 373 | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); |
| 374 | |
| 375 | if (IS_ERR(op->fallback.blk)) { |
| 376 | printk(KERN_ERR "Error allocating fallback algo %s\n", name); |
| 377 | return PTR_ERR(op->fallback.blk); |
| 378 | } |
| 379 | |
| 380 | return 0; |
| 381 | } |
| 382 | |
| 383 | static void fallback_exit_blk(struct crypto_tfm *tfm) |
| 384 | { |
| 385 | struct geode_aes_op *op = crypto_tfm_ctx(tfm); |
| 386 | |
| 387 | crypto_free_blkcipher(op->fallback.blk); |
| 388 | op->fallback.blk = NULL; |
| 389 | } |
| 390 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 391 | static struct crypto_alg geode_cbc_alg = { |
| 392 | .cra_name = "cbc(aes)", |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 393 | .cra_driver_name = "cbc-aes-geode", |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 394 | .cra_priority = 400, |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 395 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
| 396 | CRYPTO_ALG_NEED_FALLBACK, |
| 397 | .cra_init = fallback_init_blk, |
| 398 | .cra_exit = fallback_exit_blk, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 399 | .cra_blocksize = AES_MIN_BLOCK_SIZE, |
| 400 | .cra_ctxsize = sizeof(struct geode_aes_op), |
| 401 | .cra_alignmask = 15, |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 402 | .cra_type = &crypto_blkcipher_type, |
| 403 | .cra_module = THIS_MODULE, |
| 404 | .cra_list = LIST_HEAD_INIT(geode_cbc_alg.cra_list), |
| 405 | .cra_u = { |
| 406 | .blkcipher = { |
| 407 | .min_keysize = AES_MIN_KEY_SIZE, |
| 408 | .max_keysize = AES_MAX_KEY_SIZE, |
| 409 | .setkey = geode_setkey_blk, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 410 | .encrypt = geode_cbc_encrypt, |
| 411 | .decrypt = geode_cbc_decrypt, |
Jordan Crouse | 761e784 | 2007-05-24 21:23:24 +1000 | [diff] [blame] | 412 | .ivsize = AES_IV_LENGTH, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 413 | } |
| 414 | } |
| 415 | }; |
| 416 | |
| 417 | static int |
| 418 | geode_ecb_decrypt(struct blkcipher_desc *desc, |
| 419 | struct scatterlist *dst, struct scatterlist *src, |
| 420 | unsigned int nbytes) |
| 421 | { |
| 422 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 423 | struct blkcipher_walk walk; |
| 424 | int err, ret; |
| 425 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 426 | if (unlikely(op->keylen != AES_KEYSIZE_128)) |
| 427 | return fallback_blk_dec(desc, dst, src, nbytes); |
| 428 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 429 | blkcipher_walk_init(&walk, dst, src, nbytes); |
| 430 | err = blkcipher_walk_virt(desc, &walk); |
| 431 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 432 | while ((nbytes = walk.nbytes)) { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 433 | op->src = walk.src.virt.addr, |
| 434 | op->dst = walk.dst.virt.addr; |
| 435 | op->mode = AES_MODE_ECB; |
| 436 | op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); |
| 437 | op->dir = AES_DIR_DECRYPT; |
| 438 | |
| 439 | ret = geode_aes_crypt(op); |
| 440 | nbytes -= ret; |
| 441 | err = blkcipher_walk_done(desc, &walk, nbytes); |
| 442 | } |
| 443 | |
| 444 | return err; |
| 445 | } |
| 446 | |
| 447 | static int |
| 448 | geode_ecb_encrypt(struct blkcipher_desc *desc, |
| 449 | struct scatterlist *dst, struct scatterlist *src, |
| 450 | unsigned int nbytes) |
| 451 | { |
| 452 | struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); |
| 453 | struct blkcipher_walk walk; |
| 454 | int err, ret; |
| 455 | |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 456 | if (unlikely(op->keylen != AES_KEYSIZE_128)) |
| 457 | return fallback_blk_enc(desc, dst, src, nbytes); |
| 458 | |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 459 | blkcipher_walk_init(&walk, dst, src, nbytes); |
| 460 | err = blkcipher_walk_virt(desc, &walk); |
| 461 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 462 | while ((nbytes = walk.nbytes)) { |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 463 | op->src = walk.src.virt.addr, |
| 464 | op->dst = walk.dst.virt.addr; |
| 465 | op->mode = AES_MODE_ECB; |
| 466 | op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); |
| 467 | op->dir = AES_DIR_ENCRYPT; |
| 468 | |
| 469 | ret = geode_aes_crypt(op); |
| 470 | nbytes -= ret; |
| 471 | ret = blkcipher_walk_done(desc, &walk, nbytes); |
| 472 | } |
| 473 | |
| 474 | return err; |
| 475 | } |
| 476 | |
| 477 | static struct crypto_alg geode_ecb_alg = { |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 478 | .cra_name = "ecb(aes)", |
| 479 | .cra_driver_name = "ecb-aes-geode", |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 480 | .cra_priority = 400, |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 481 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
| 482 | CRYPTO_ALG_NEED_FALLBACK, |
| 483 | .cra_init = fallback_init_blk, |
| 484 | .cra_exit = fallback_exit_blk, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 485 | .cra_blocksize = AES_MIN_BLOCK_SIZE, |
| 486 | .cra_ctxsize = sizeof(struct geode_aes_op), |
| 487 | .cra_alignmask = 15, |
Sebastian Siewior | cd7c3bf | 2007-11-10 19:29:33 +0800 | [diff] [blame] | 488 | .cra_type = &crypto_blkcipher_type, |
| 489 | .cra_module = THIS_MODULE, |
| 490 | .cra_list = LIST_HEAD_INIT(geode_ecb_alg.cra_list), |
| 491 | .cra_u = { |
| 492 | .blkcipher = { |
| 493 | .min_keysize = AES_MIN_KEY_SIZE, |
| 494 | .max_keysize = AES_MAX_KEY_SIZE, |
| 495 | .setkey = geode_setkey_blk, |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 496 | .encrypt = geode_ecb_encrypt, |
| 497 | .decrypt = geode_ecb_decrypt, |
| 498 | } |
| 499 | } |
| 500 | }; |
| 501 | |
Adrian Bunk | f17922b | 2007-12-12 10:42:41 +0800 | [diff] [blame] | 502 | static void __devexit |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 503 | geode_aes_remove(struct pci_dev *dev) |
| 504 | { |
| 505 | crypto_unregister_alg(&geode_alg); |
| 506 | crypto_unregister_alg(&geode_ecb_alg); |
| 507 | crypto_unregister_alg(&geode_cbc_alg); |
| 508 | |
| 509 | pci_iounmap(dev, _iobase); |
| 510 | _iobase = NULL; |
| 511 | |
| 512 | pci_release_regions(dev); |
| 513 | pci_disable_device(dev); |
| 514 | } |
| 515 | |
| 516 | |
Adrian Bunk | f17922b | 2007-12-12 10:42:41 +0800 | [diff] [blame] | 517 | static int __devinit |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 518 | geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) |
| 519 | { |
| 520 | int ret; |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 521 | ret = pci_enable_device(dev); |
| 522 | if (ret) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 523 | return ret; |
| 524 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 525 | ret = pci_request_regions(dev, "geode-aes"); |
| 526 | if (ret) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 527 | goto eenable; |
| 528 | |
| 529 | _iobase = pci_iomap(dev, 0, 0); |
| 530 | |
| 531 | if (_iobase == NULL) { |
| 532 | ret = -ENOMEM; |
| 533 | goto erequest; |
| 534 | } |
| 535 | |
| 536 | spin_lock_init(&lock); |
| 537 | |
| 538 | /* Clear any pending activity */ |
| 539 | iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG); |
| 540 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 541 | ret = crypto_register_alg(&geode_alg); |
| 542 | if (ret) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 543 | goto eiomap; |
| 544 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 545 | ret = crypto_register_alg(&geode_ecb_alg); |
| 546 | if (ret) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 547 | goto ealg; |
| 548 | |
Chihau Chau | 9970071 | 2010-04-19 21:02:41 +0800 | [diff] [blame] | 549 | ret = crypto_register_alg(&geode_cbc_alg); |
| 550 | if (ret) |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 551 | goto eecb; |
| 552 | |
| 553 | printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n"); |
| 554 | return 0; |
| 555 | |
| 556 | eecb: |
| 557 | crypto_unregister_alg(&geode_ecb_alg); |
| 558 | |
| 559 | ealg: |
| 560 | crypto_unregister_alg(&geode_alg); |
| 561 | |
| 562 | eiomap: |
| 563 | pci_iounmap(dev, _iobase); |
| 564 | |
| 565 | erequest: |
| 566 | pci_release_regions(dev); |
| 567 | |
| 568 | eenable: |
| 569 | pci_disable_device(dev); |
| 570 | |
| 571 | printk(KERN_ERR "geode-aes: GEODE AES initialization failed.\n"); |
| 572 | return ret; |
| 573 | } |
| 574 | |
| 575 | static struct pci_device_id geode_aes_tbl[] = { |
| 576 | { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} , |
| 577 | { 0, } |
| 578 | }; |
| 579 | |
| 580 | MODULE_DEVICE_TABLE(pci, geode_aes_tbl); |
| 581 | |
| 582 | static struct pci_driver geode_aes_driver = { |
| 583 | .name = "Geode LX AES", |
| 584 | .id_table = geode_aes_tbl, |
| 585 | .probe = geode_aes_probe, |
| 586 | .remove = __devexit_p(geode_aes_remove) |
| 587 | }; |
| 588 | |
| 589 | static int __init |
| 590 | geode_aes_init(void) |
| 591 | { |
Richard Knutsson | 09cb914 | 2007-01-24 21:39:34 +1100 | [diff] [blame] | 592 | return pci_register_driver(&geode_aes_driver); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 593 | } |
| 594 | |
| 595 | static void __exit |
| 596 | geode_aes_exit(void) |
| 597 | { |
| 598 | pci_unregister_driver(&geode_aes_driver); |
| 599 | } |
| 600 | |
| 601 | MODULE_AUTHOR("Advanced Micro Devices, Inc."); |
| 602 | MODULE_DESCRIPTION("Geode LX Hardware AES driver"); |
| 603 | MODULE_LICENSE("GPL"); |
Jordan Crouse | 9fe757b | 2006-10-04 18:48:57 +1000 | [diff] [blame] | 604 | |
| 605 | module_init(geode_aes_init); |
| 606 | module_exit(geode_aes_exit); |