blob: 4e054bd91664a8047968d51ec313958013b4a8af [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
Milan Broz542da312009-12-10 23:51:57 +00004 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * This file is released under the GPL.
7 */
8
Milan Broz43d69032008-02-08 02:11:09 +00009#include <linux/completion.h>
Herbert Xud1806f62006-08-22 20:29:17 +100010#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/module.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/bio.h>
15#include <linux/blkdev.h>
16#include <linux/mempool.h>
17#include <linux/slab.h>
18#include <linux/crypto.h>
19#include <linux/workqueue.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070020#include <linux/backing-dev.h>
Andi Kleenc0297722011-01-13 19:59:53 +000021#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <asm/atomic.h>
David Hardeman378f0582005-09-17 17:55:31 +100023#include <linux/scatterlist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <asm/page.h>
Rik Snel48527fa2006-09-03 08:56:39 +100025#include <asm/unaligned.h>
Milan Broz34745782011-01-13 19:59:55 +000026#include <crypto/hash.h>
27#include <crypto/md5.h>
28#include <crypto/algapi.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Mikulas Patocka586e80e2008-10-21 17:44:59 +010030#include <linux/device-mapper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
Alasdair G Kergon72d94862006-06-26 00:27:35 -070032#define DM_MSG_PREFIX "crypt"
Milan Broze48d4bb2006-10-03 01:15:37 -070033#define MESG_STR(x) x, sizeof(x)
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
35/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * context holding the current state of a multi-part conversion
37 */
38struct convert_context {
Milan Broz43d69032008-02-08 02:11:09 +000039 struct completion restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 struct bio *bio_in;
41 struct bio *bio_out;
42 unsigned int offset_in;
43 unsigned int offset_out;
44 unsigned int idx_in;
45 unsigned int idx_out;
46 sector_t sector;
Milan Broz43d69032008-02-08 02:11:09 +000047 atomic_t pending;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048};
49
Milan Broz53017032008-02-08 02:10:38 +000050/*
51 * per bio private data
52 */
53struct dm_crypt_io {
54 struct dm_target *target;
55 struct bio *base_bio;
56 struct work_struct work;
57
58 struct convert_context ctx;
59
60 atomic_t pending;
61 int error;
Milan Broz0c395b02008-02-08 02:10:54 +000062 sector_t sector;
Milan Broz393b47e2008-10-21 17:45:02 +010063 struct dm_crypt_io *base_io;
Milan Broz53017032008-02-08 02:10:38 +000064};
65
Milan Broz01482b72008-02-08 02:11:04 +000066struct dm_crypt_request {
Huang Yingb2174ee2009-03-16 17:44:33 +000067 struct convert_context *ctx;
Milan Broz01482b72008-02-08 02:11:04 +000068 struct scatterlist sg_in;
69 struct scatterlist sg_out;
Milan Broz2dc53272011-01-13 19:59:54 +000070 sector_t iv_sector;
Milan Broz01482b72008-02-08 02:11:04 +000071};
72
Linus Torvalds1da177e2005-04-16 15:20:36 -070073struct crypt_config;
74
75struct crypt_iv_operations {
76 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +010077 const char *opts);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 void (*dtr)(struct crypt_config *cc);
Milan Brozb95bf2d2009-12-10 23:51:56 +000079 int (*init)(struct crypt_config *cc);
Milan Broz542da312009-12-10 23:51:57 +000080 int (*wipe)(struct crypt_config *cc);
Milan Broz2dc53272011-01-13 19:59:54 +000081 int (*generator)(struct crypt_config *cc, u8 *iv,
82 struct dm_crypt_request *dmreq);
83 int (*post)(struct crypt_config *cc, u8 *iv,
84 struct dm_crypt_request *dmreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085};
86
Milan Broz60473592009-12-10 23:51:55 +000087struct iv_essiv_private {
Milan Brozb95bf2d2009-12-10 23:51:56 +000088 struct crypto_hash *hash_tfm;
89 u8 *salt;
Milan Broz60473592009-12-10 23:51:55 +000090};
91
92struct iv_benbi_private {
93 int shift;
94};
95
Milan Broz34745782011-01-13 19:59:55 +000096#define LMK_SEED_SIZE 64 /* hash + 0 */
97struct iv_lmk_private {
98 struct crypto_shash *hash_tfm;
99 u8 *seed;
100};
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102/*
103 * Crypt: maps a linear range of a block device
104 * and encrypts / decrypts at the same time.
105 */
Milan Broze48d4bb2006-10-03 01:15:37 -0700106enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
Andi Kleenc0297722011-01-13 19:59:53 +0000107
108/*
109 * Duplicated per-CPU state for cipher.
110 */
111struct crypt_cpu {
112 struct ablkcipher_request *req;
Andi Kleenc0297722011-01-13 19:59:53 +0000113 /* ESSIV: struct crypto_cipher *essiv_tfm */
114 void *iv_private;
Milan Brozd1f96422011-01-13 19:59:54 +0000115 struct crypto_ablkcipher *tfms[0];
Andi Kleenc0297722011-01-13 19:59:53 +0000116};
117
118/*
119 * The fields in here must be read only after initialization,
120 * changing state should be in crypt_cpu.
121 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122struct crypt_config {
123 struct dm_dev *dev;
124 sector_t start;
125
126 /*
Milan Brozddd42ed2008-02-08 02:11:07 +0000127 * pool for per bio private data, crypto requests and
128 * encryption requeusts/buffer pages
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 */
130 mempool_t *io_pool;
Milan Brozddd42ed2008-02-08 02:11:07 +0000131 mempool_t *req_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 mempool_t *page_pool;
Milan Broz6a24c712006-10-03 01:15:40 -0700133 struct bio_set *bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
Milan Brozcabf08e2007-10-19 22:38:58 +0100135 struct workqueue_struct *io_queue;
136 struct workqueue_struct *crypt_queue;
Milan Broz3f1e9072008-03-28 14:16:07 -0700137
Milan Broz5ebaee62010-08-12 04:14:07 +0100138 char *cipher;
Milan Broz7dbcd132011-01-13 19:59:52 +0000139 char *cipher_string;
Milan Broz5ebaee62010-08-12 04:14:07 +0100140
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141 struct crypt_iv_operations *iv_gen_ops;
Herbert Xu79066ad2006-12-05 13:41:52 -0800142 union {
Milan Broz60473592009-12-10 23:51:55 +0000143 struct iv_essiv_private essiv;
144 struct iv_benbi_private benbi;
Milan Broz34745782011-01-13 19:59:55 +0000145 struct iv_lmk_private lmk;
Herbert Xu79066ad2006-12-05 13:41:52 -0800146 } iv_gen_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 sector_t iv_offset;
148 unsigned int iv_size;
149
Milan Brozddd42ed2008-02-08 02:11:07 +0000150 /*
Andi Kleenc0297722011-01-13 19:59:53 +0000151 * Duplicated per cpu state. Access through
152 * per_cpu_ptr() only.
153 */
154 struct crypt_cpu __percpu *cpu;
Milan Brozd1f96422011-01-13 19:59:54 +0000155 unsigned tfms_count;
Andi Kleenc0297722011-01-13 19:59:53 +0000156
157 /*
Milan Brozddd42ed2008-02-08 02:11:07 +0000158 * Layout of each crypto request:
159 *
160 * struct ablkcipher_request
161 * context
162 * padding
163 * struct dm_crypt_request
164 * padding
165 * IV
166 *
167 * The padding is added so that dm_crypt_request and the IV are
168 * correctly aligned.
169 */
170 unsigned int dmreq_start;
Milan Brozddd42ed2008-02-08 02:11:07 +0000171
Milan Broze48d4bb2006-10-03 01:15:37 -0700172 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 unsigned int key_size;
Milan Brozd1f96422011-01-13 19:59:54 +0000174 unsigned int key_parts;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 u8 key[0];
176};
177
Milan Broz6a24c712006-10-03 01:15:40 -0700178#define MIN_IOS 16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179#define MIN_POOL_PAGES 32
180#define MIN_BIO_PAGES 8
181
Christoph Lametere18b8902006-12-06 20:33:20 -0800182static struct kmem_cache *_crypt_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100184static void clone_init(struct dm_crypt_io *, struct bio *);
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000185static void kcryptd_queue_crypt(struct dm_crypt_io *io);
Milan Broz2dc53272011-01-13 19:59:54 +0000186static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
Olaf Kirch027581f2007-05-09 02:32:52 -0700187
Andi Kleenc0297722011-01-13 19:59:53 +0000188static struct crypt_cpu *this_crypt_config(struct crypt_config *cc)
189{
190 return this_cpu_ptr(cc->cpu);
191}
192
193/*
194 * Use this to access cipher attributes that are the same for each CPU.
195 */
196static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc)
197{
Milan Brozd1f96422011-01-13 19:59:54 +0000198 return __this_cpu_ptr(cc->cpu)->tfms[0];
Andi Kleenc0297722011-01-13 19:59:53 +0000199}
200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 * Different IV generation algorithms:
203 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000204 * plain: the initial vector is the 32-bit little-endian version of the sector
Robert P. J. Day3a4fa0a2007-10-19 23:10:43 +0200205 * number, padded with zeros if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206 *
Milan Broz61afef62009-12-10 23:52:25 +0000207 * plain64: the initial vector is the 64-bit little-endian version of the sector
208 * number, padded with zeros if necessary.
209 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000210 * essiv: "encrypted sector|salt initial vector", the sector number is
211 * encrypted with the bulk cipher using a salt as key. The salt
212 * should be derived from the bulk cipher's key via hashing.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 *
Rik Snel48527fa2006-09-03 08:56:39 +1000214 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
215 * (needed for LRW-32-AES and possible other narrow block modes)
216 *
Ludwig Nussel46b47732007-05-09 02:32:55 -0700217 * null: the initial vector is always zero. Provides compatibility with
218 * obsolete loop_fish2 devices. Do not use for new devices.
219 *
Milan Broz34745782011-01-13 19:59:55 +0000220 * lmk: Compatible implementation of the block chaining mode used
221 * by the Loop-AES block device encryption system
222 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
223 * It operates on full 512 byte sectors and uses CBC
224 * with an IV derived from the sector number, the data and
225 * optionally extra IV seed.
226 * This means that after decryption the first block
227 * of sector must be tweaked according to decrypted data.
228 * Loop-AES can use three encryption schemes:
229 * version 1: is plain aes-cbc mode
230 * version 2: uses 64 multikey scheme with lmk IV generator
231 * version 3: the same as version 2 with additional IV seed
232 * (it uses 65 keys, last key is used as IV seed)
233 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 * plumb: unimplemented, see:
235 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
236 */
237
Milan Broz2dc53272011-01-13 19:59:54 +0000238static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
239 struct dm_crypt_request *dmreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240{
241 memset(iv, 0, cc->iv_size);
Milan Broz2dc53272011-01-13 19:59:54 +0000242 *(u32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243
244 return 0;
245}
246
Milan Broz61afef62009-12-10 23:52:25 +0000247static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
Milan Broz2dc53272011-01-13 19:59:54 +0000248 struct dm_crypt_request *dmreq)
Milan Broz61afef62009-12-10 23:52:25 +0000249{
250 memset(iv, 0, cc->iv_size);
Milan Broz2dc53272011-01-13 19:59:54 +0000251 *(u64 *)iv = cpu_to_le64(dmreq->iv_sector);
Milan Broz61afef62009-12-10 23:52:25 +0000252
253 return 0;
254}
255
Milan Brozb95bf2d2009-12-10 23:51:56 +0000256/* Initialise ESSIV - compute salt but no local memory allocations */
257static int crypt_iv_essiv_init(struct crypt_config *cc)
258{
259 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
260 struct hash_desc desc;
261 struct scatterlist sg;
Andi Kleenc0297722011-01-13 19:59:53 +0000262 struct crypto_cipher *essiv_tfm;
263 int err, cpu;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000264
265 sg_init_one(&sg, cc->key, cc->key_size);
266 desc.tfm = essiv->hash_tfm;
267 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
268
269 err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt);
270 if (err)
271 return err;
272
Andi Kleenc0297722011-01-13 19:59:53 +0000273 for_each_possible_cpu(cpu) {
274 essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private,
275
276 err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
Milan Brozb95bf2d2009-12-10 23:51:56 +0000277 crypto_hash_digestsize(essiv->hash_tfm));
Andi Kleenc0297722011-01-13 19:59:53 +0000278 if (err)
279 return err;
280 }
281
282 return 0;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000283}
284
Milan Broz542da312009-12-10 23:51:57 +0000285/* Wipe salt and reset key derived from volume key */
286static int crypt_iv_essiv_wipe(struct crypt_config *cc)
287{
288 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
289 unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
Andi Kleenc0297722011-01-13 19:59:53 +0000290 struct crypto_cipher *essiv_tfm;
291 int cpu, r, err = 0;
Milan Broz542da312009-12-10 23:51:57 +0000292
293 memset(essiv->salt, 0, salt_size);
294
Andi Kleenc0297722011-01-13 19:59:53 +0000295 for_each_possible_cpu(cpu) {
296 essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private;
297 r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
298 if (r)
299 err = r;
300 }
301
302 return err;
303}
304
305/* Set up per cpu cipher state */
306static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
307 struct dm_target *ti,
308 u8 *salt, unsigned saltsize)
309{
310 struct crypto_cipher *essiv_tfm;
311 int err;
312
313 /* Setup the essiv_tfm with the given salt */
314 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
315 if (IS_ERR(essiv_tfm)) {
316 ti->error = "Error allocating crypto tfm for ESSIV";
317 return essiv_tfm;
318 }
319
320 if (crypto_cipher_blocksize(essiv_tfm) !=
321 crypto_ablkcipher_ivsize(any_tfm(cc))) {
322 ti->error = "Block size of ESSIV cipher does "
323 "not match IV size of block cipher";
324 crypto_free_cipher(essiv_tfm);
325 return ERR_PTR(-EINVAL);
326 }
327
328 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
329 if (err) {
330 ti->error = "Failed to set key for ESSIV cipher";
331 crypto_free_cipher(essiv_tfm);
332 return ERR_PTR(err);
333 }
334
335 return essiv_tfm;
Milan Broz542da312009-12-10 23:51:57 +0000336}
337
Milan Broz60473592009-12-10 23:51:55 +0000338static void crypt_iv_essiv_dtr(struct crypt_config *cc)
339{
Andi Kleenc0297722011-01-13 19:59:53 +0000340 int cpu;
341 struct crypt_cpu *cpu_cc;
342 struct crypto_cipher *essiv_tfm;
Milan Broz60473592009-12-10 23:51:55 +0000343 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
344
Milan Brozb95bf2d2009-12-10 23:51:56 +0000345 crypto_free_hash(essiv->hash_tfm);
346 essiv->hash_tfm = NULL;
347
348 kzfree(essiv->salt);
349 essiv->salt = NULL;
Andi Kleenc0297722011-01-13 19:59:53 +0000350
351 for_each_possible_cpu(cpu) {
352 cpu_cc = per_cpu_ptr(cc->cpu, cpu);
353 essiv_tfm = cpu_cc->iv_private;
354
355 if (essiv_tfm)
356 crypto_free_cipher(essiv_tfm);
357
358 cpu_cc->iv_private = NULL;
359 }
Milan Broz60473592009-12-10 23:51:55 +0000360}
361
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +0100363 const char *opts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364{
Milan Broz5861f1b2009-12-10 23:51:56 +0000365 struct crypto_cipher *essiv_tfm = NULL;
366 struct crypto_hash *hash_tfm = NULL;
Milan Broz5861f1b2009-12-10 23:51:56 +0000367 u8 *salt = NULL;
Andi Kleenc0297722011-01-13 19:59:53 +0000368 int err, cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369
Milan Broz5861f1b2009-12-10 23:51:56 +0000370 if (!opts) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700371 ti->error = "Digest algorithm missing for ESSIV mode";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 return -EINVAL;
373 }
374
Milan Brozb95bf2d2009-12-10 23:51:56 +0000375 /* Allocate hash algorithm */
Herbert Xu35058682006-08-24 19:10:20 +1000376 hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
377 if (IS_ERR(hash_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700378 ti->error = "Error initializing ESSIV hash";
Milan Broz5861f1b2009-12-10 23:51:56 +0000379 err = PTR_ERR(hash_tfm);
380 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 }
382
Milan Brozb95bf2d2009-12-10 23:51:56 +0000383 salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL);
Milan Broz5861f1b2009-12-10 23:51:56 +0000384 if (!salt) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700385 ti->error = "Error kmallocing salt storage in ESSIV";
Milan Broz5861f1b2009-12-10 23:51:56 +0000386 err = -ENOMEM;
387 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 }
389
Milan Brozb95bf2d2009-12-10 23:51:56 +0000390 cc->iv_gen_private.essiv.salt = salt;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000391 cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
392
Andi Kleenc0297722011-01-13 19:59:53 +0000393 for_each_possible_cpu(cpu) {
394 essiv_tfm = setup_essiv_cpu(cc, ti, salt,
395 crypto_hash_digestsize(hash_tfm));
396 if (IS_ERR(essiv_tfm)) {
397 crypt_iv_essiv_dtr(cc);
398 return PTR_ERR(essiv_tfm);
399 }
400 per_cpu_ptr(cc->cpu, cpu)->iv_private = essiv_tfm;
401 }
402
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403 return 0;
Milan Broz5861f1b2009-12-10 23:51:56 +0000404
405bad:
Milan Broz5861f1b2009-12-10 23:51:56 +0000406 if (hash_tfm && !IS_ERR(hash_tfm))
407 crypto_free_hash(hash_tfm);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000408 kfree(salt);
Milan Broz5861f1b2009-12-10 23:51:56 +0000409 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410}
411
Milan Broz2dc53272011-01-13 19:59:54 +0000412static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
413 struct dm_crypt_request *dmreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414{
Andi Kleenc0297722011-01-13 19:59:53 +0000415 struct crypto_cipher *essiv_tfm = this_crypt_config(cc)->iv_private;
416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 memset(iv, 0, cc->iv_size);
Milan Broz2dc53272011-01-13 19:59:54 +0000418 *(u64 *)iv = cpu_to_le64(dmreq->iv_sector);
Andi Kleenc0297722011-01-13 19:59:53 +0000419 crypto_cipher_encrypt_one(essiv_tfm, iv, iv);
420
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 return 0;
422}
423
Rik Snel48527fa2006-09-03 08:56:39 +1000424static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
425 const char *opts)
426{
Andi Kleenc0297722011-01-13 19:59:53 +0000427 unsigned bs = crypto_ablkcipher_blocksize(any_tfm(cc));
David Howellsf0d1b0b2006-12-08 02:37:49 -0800428 int log = ilog2(bs);
Rik Snel48527fa2006-09-03 08:56:39 +1000429
430 /* we need to calculate how far we must shift the sector count
431 * to get the cipher block count, we use this shift in _gen */
432
433 if (1 << log != bs) {
434 ti->error = "cypher blocksize is not a power of 2";
435 return -EINVAL;
436 }
437
438 if (log > 9) {
439 ti->error = "cypher blocksize is > 512";
440 return -EINVAL;
441 }
442
Milan Broz60473592009-12-10 23:51:55 +0000443 cc->iv_gen_private.benbi.shift = 9 - log;
Rik Snel48527fa2006-09-03 08:56:39 +1000444
445 return 0;
446}
447
448static void crypt_iv_benbi_dtr(struct crypt_config *cc)
449{
Rik Snel48527fa2006-09-03 08:56:39 +1000450}
451
Milan Broz2dc53272011-01-13 19:59:54 +0000452static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
453 struct dm_crypt_request *dmreq)
Rik Snel48527fa2006-09-03 08:56:39 +1000454{
Herbert Xu79066ad2006-12-05 13:41:52 -0800455 __be64 val;
456
Rik Snel48527fa2006-09-03 08:56:39 +1000457 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
Herbert Xu79066ad2006-12-05 13:41:52 -0800458
Milan Broz2dc53272011-01-13 19:59:54 +0000459 val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
Herbert Xu79066ad2006-12-05 13:41:52 -0800460 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
Rik Snel48527fa2006-09-03 08:56:39 +1000461
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 return 0;
463}
464
Milan Broz2dc53272011-01-13 19:59:54 +0000465static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
466 struct dm_crypt_request *dmreq)
Ludwig Nussel46b47732007-05-09 02:32:55 -0700467{
468 memset(iv, 0, cc->iv_size);
469
470 return 0;
471}
472
Milan Broz34745782011-01-13 19:59:55 +0000473static void crypt_iv_lmk_dtr(struct crypt_config *cc)
474{
475 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
476
477 if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
478 crypto_free_shash(lmk->hash_tfm);
479 lmk->hash_tfm = NULL;
480
481 kzfree(lmk->seed);
482 lmk->seed = NULL;
483}
484
485static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
486 const char *opts)
487{
488 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
489
490 lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0);
491 if (IS_ERR(lmk->hash_tfm)) {
492 ti->error = "Error initializing LMK hash";
493 return PTR_ERR(lmk->hash_tfm);
494 }
495
496 /* No seed in LMK version 2 */
497 if (cc->key_parts == cc->tfms_count) {
498 lmk->seed = NULL;
499 return 0;
500 }
501
502 lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
503 if (!lmk->seed) {
504 crypt_iv_lmk_dtr(cc);
505 ti->error = "Error kmallocing seed storage in LMK";
506 return -ENOMEM;
507 }
508
509 return 0;
510}
511
512static int crypt_iv_lmk_init(struct crypt_config *cc)
513{
514 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
515 int subkey_size = cc->key_size / cc->key_parts;
516
517 /* LMK seed is on the position of LMK_KEYS + 1 key */
518 if (lmk->seed)
519 memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
520 crypto_shash_digestsize(lmk->hash_tfm));
521
522 return 0;
523}
524
525static int crypt_iv_lmk_wipe(struct crypt_config *cc)
526{
527 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
528
529 if (lmk->seed)
530 memset(lmk->seed, 0, LMK_SEED_SIZE);
531
532 return 0;
533}
534
535static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
536 struct dm_crypt_request *dmreq,
537 u8 *data)
538{
539 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
540 struct {
541 struct shash_desc desc;
542 char ctx[crypto_shash_descsize(lmk->hash_tfm)];
543 } sdesc;
544 struct md5_state md5state;
545 u32 buf[4];
546 int i, r;
547
548 sdesc.desc.tfm = lmk->hash_tfm;
549 sdesc.desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
550
551 r = crypto_shash_init(&sdesc.desc);
552 if (r)
553 return r;
554
555 if (lmk->seed) {
556 r = crypto_shash_update(&sdesc.desc, lmk->seed, LMK_SEED_SIZE);
557 if (r)
558 return r;
559 }
560
561 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
562 r = crypto_shash_update(&sdesc.desc, data + 16, 16 * 31);
563 if (r)
564 return r;
565
566 /* Sector is cropped to 56 bits here */
567 buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
568 buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
569 buf[2] = cpu_to_le32(4024);
570 buf[3] = 0;
571 r = crypto_shash_update(&sdesc.desc, (u8 *)buf, sizeof(buf));
572 if (r)
573 return r;
574
575 /* No MD5 padding here */
576 r = crypto_shash_export(&sdesc.desc, &md5state);
577 if (r)
578 return r;
579
580 for (i = 0; i < MD5_HASH_WORDS; i++)
581 __cpu_to_le32s(&md5state.hash[i]);
582 memcpy(iv, &md5state.hash, cc->iv_size);
583
584 return 0;
585}
586
587static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
588 struct dm_crypt_request *dmreq)
589{
590 u8 *src;
591 int r = 0;
592
593 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
594 src = kmap_atomic(sg_page(&dmreq->sg_in), KM_USER0);
595 r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset);
596 kunmap_atomic(src, KM_USER0);
597 } else
598 memset(iv, 0, cc->iv_size);
599
600 return r;
601}
602
603static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
604 struct dm_crypt_request *dmreq)
605{
606 u8 *dst;
607 int r;
608
609 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
610 return 0;
611
612 dst = kmap_atomic(sg_page(&dmreq->sg_out), KM_USER0);
613 r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset);
614
615 /* Tweak the first block of plaintext sector */
616 if (!r)
617 crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size);
618
619 kunmap_atomic(dst, KM_USER0);
620 return r;
621}
622
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623static struct crypt_iv_operations crypt_iv_plain_ops = {
624 .generator = crypt_iv_plain_gen
625};
626
Milan Broz61afef62009-12-10 23:52:25 +0000627static struct crypt_iv_operations crypt_iv_plain64_ops = {
628 .generator = crypt_iv_plain64_gen
629};
630
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631static struct crypt_iv_operations crypt_iv_essiv_ops = {
632 .ctr = crypt_iv_essiv_ctr,
633 .dtr = crypt_iv_essiv_dtr,
Milan Brozb95bf2d2009-12-10 23:51:56 +0000634 .init = crypt_iv_essiv_init,
Milan Broz542da312009-12-10 23:51:57 +0000635 .wipe = crypt_iv_essiv_wipe,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 .generator = crypt_iv_essiv_gen
637};
638
Rik Snel48527fa2006-09-03 08:56:39 +1000639static struct crypt_iv_operations crypt_iv_benbi_ops = {
640 .ctr = crypt_iv_benbi_ctr,
641 .dtr = crypt_iv_benbi_dtr,
642 .generator = crypt_iv_benbi_gen
643};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Ludwig Nussel46b47732007-05-09 02:32:55 -0700645static struct crypt_iv_operations crypt_iv_null_ops = {
646 .generator = crypt_iv_null_gen
647};
648
Milan Broz34745782011-01-13 19:59:55 +0000649static struct crypt_iv_operations crypt_iv_lmk_ops = {
650 .ctr = crypt_iv_lmk_ctr,
651 .dtr = crypt_iv_lmk_dtr,
652 .init = crypt_iv_lmk_init,
653 .wipe = crypt_iv_lmk_wipe,
654 .generator = crypt_iv_lmk_gen,
655 .post = crypt_iv_lmk_post
656};
657
Milan Brozd469f842007-10-19 22:42:37 +0100658static void crypt_convert_init(struct crypt_config *cc,
659 struct convert_context *ctx,
660 struct bio *bio_out, struct bio *bio_in,
Milan Brozfcd369d2008-02-08 02:10:41 +0000661 sector_t sector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662{
663 ctx->bio_in = bio_in;
664 ctx->bio_out = bio_out;
665 ctx->offset_in = 0;
666 ctx->offset_out = 0;
667 ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
668 ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
669 ctx->sector = sector + cc->iv_offset;
Milan Broz43d69032008-02-08 02:11:09 +0000670 init_completion(&ctx->restart);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671}
672
Huang Yingb2174ee2009-03-16 17:44:33 +0000673static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
674 struct ablkcipher_request *req)
675{
676 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
677}
678
679static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc,
680 struct dm_crypt_request *dmreq)
681{
682 return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start);
683}
684
Milan Broz2dc53272011-01-13 19:59:54 +0000685static u8 *iv_of_dmreq(struct crypt_config *cc,
686 struct dm_crypt_request *dmreq)
687{
688 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
689 crypto_ablkcipher_alignmask(any_tfm(cc)) + 1);
690}
691
Milan Broz01482b72008-02-08 02:11:04 +0000692static int crypt_convert_block(struct crypt_config *cc,
Milan Broz3a7f6c92008-02-08 02:11:14 +0000693 struct convert_context *ctx,
694 struct ablkcipher_request *req)
Milan Broz01482b72008-02-08 02:11:04 +0000695{
696 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
697 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000698 struct dm_crypt_request *dmreq;
699 u8 *iv;
700 int r = 0;
Milan Broz01482b72008-02-08 02:11:04 +0000701
Huang Yingb2174ee2009-03-16 17:44:33 +0000702 dmreq = dmreq_of_req(cc, req);
Milan Broz2dc53272011-01-13 19:59:54 +0000703 iv = iv_of_dmreq(cc, dmreq);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000704
Milan Broz2dc53272011-01-13 19:59:54 +0000705 dmreq->iv_sector = ctx->sector;
Huang Yingb2174ee2009-03-16 17:44:33 +0000706 dmreq->ctx = ctx;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000707 sg_init_table(&dmreq->sg_in, 1);
708 sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000709 bv_in->bv_offset + ctx->offset_in);
710
Milan Broz3a7f6c92008-02-08 02:11:14 +0000711 sg_init_table(&dmreq->sg_out, 1);
712 sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000713 bv_out->bv_offset + ctx->offset_out);
714
715 ctx->offset_in += 1 << SECTOR_SHIFT;
716 if (ctx->offset_in >= bv_in->bv_len) {
717 ctx->offset_in = 0;
718 ctx->idx_in++;
719 }
720
721 ctx->offset_out += 1 << SECTOR_SHIFT;
722 if (ctx->offset_out >= bv_out->bv_len) {
723 ctx->offset_out = 0;
724 ctx->idx_out++;
725 }
726
Milan Broz3a7f6c92008-02-08 02:11:14 +0000727 if (cc->iv_gen_ops) {
Milan Broz2dc53272011-01-13 19:59:54 +0000728 r = cc->iv_gen_ops->generator(cc, iv, dmreq);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000729 if (r < 0)
730 return r;
731 }
732
733 ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
734 1 << SECTOR_SHIFT, iv);
735
736 if (bio_data_dir(ctx->bio_in) == WRITE)
737 r = crypto_ablkcipher_encrypt(req);
738 else
739 r = crypto_ablkcipher_decrypt(req);
740
Milan Broz2dc53272011-01-13 19:59:54 +0000741 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
742 r = cc->iv_gen_ops->post(cc, iv, dmreq);
743
Milan Broz3a7f6c92008-02-08 02:11:14 +0000744 return r;
Milan Broz01482b72008-02-08 02:11:04 +0000745}
746
Milan Broz95497a92008-02-08 02:11:12 +0000747static void kcryptd_async_done(struct crypto_async_request *async_req,
748 int error);
Andi Kleenc0297722011-01-13 19:59:53 +0000749
Milan Brozddd42ed2008-02-08 02:11:07 +0000750static void crypt_alloc_req(struct crypt_config *cc,
751 struct convert_context *ctx)
752{
Andi Kleenc0297722011-01-13 19:59:53 +0000753 struct crypt_cpu *this_cc = this_crypt_config(cc);
Milan Brozd1f96422011-01-13 19:59:54 +0000754 unsigned key_index = ctx->sector & (cc->tfms_count - 1);
Andi Kleenc0297722011-01-13 19:59:53 +0000755
756 if (!this_cc->req)
757 this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
758
Milan Brozd1f96422011-01-13 19:59:54 +0000759 ablkcipher_request_set_tfm(this_cc->req, this_cc->tfms[key_index]);
Andi Kleenc0297722011-01-13 19:59:53 +0000760 ablkcipher_request_set_callback(this_cc->req,
761 CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
762 kcryptd_async_done, dmreq_of_req(cc, this_cc->req));
Milan Brozddd42ed2008-02-08 02:11:07 +0000763}
764
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765/*
766 * Encrypt / decrypt data from one bio to another one (can be the same one)
767 */
768static int crypt_convert(struct crypt_config *cc,
Milan Brozd469f842007-10-19 22:42:37 +0100769 struct convert_context *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770{
Andi Kleenc0297722011-01-13 19:59:53 +0000771 struct crypt_cpu *this_cc = this_crypt_config(cc);
Milan Broz3f1e9072008-03-28 14:16:07 -0700772 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Milan Brozc8081612008-10-10 13:37:08 +0100774 atomic_set(&ctx->pending, 1);
775
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
777 ctx->idx_out < ctx->bio_out->bi_vcnt) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Milan Broz3a7f6c92008-02-08 02:11:14 +0000779 crypt_alloc_req(cc, ctx);
780
Milan Broz3f1e9072008-03-28 14:16:07 -0700781 atomic_inc(&ctx->pending);
782
Andi Kleenc0297722011-01-13 19:59:53 +0000783 r = crypt_convert_block(cc, ctx, this_cc->req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000784
785 switch (r) {
Milan Broz3f1e9072008-03-28 14:16:07 -0700786 /* async */
Milan Broz3a7f6c92008-02-08 02:11:14 +0000787 case -EBUSY:
788 wait_for_completion(&ctx->restart);
789 INIT_COMPLETION(ctx->restart);
790 /* fall through*/
791 case -EINPROGRESS:
Andi Kleenc0297722011-01-13 19:59:53 +0000792 this_cc->req = NULL;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000793 ctx->sector++;
794 continue;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000795
Milan Broz3f1e9072008-03-28 14:16:07 -0700796 /* sync */
797 case 0:
798 atomic_dec(&ctx->pending);
799 ctx->sector++;
Milan Brozc7f1b202008-07-02 09:34:28 +0100800 cond_resched();
Milan Broz3f1e9072008-03-28 14:16:07 -0700801 continue;
802
803 /* error */
804 default:
805 atomic_dec(&ctx->pending);
806 return r;
807 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 }
809
Milan Broz3f1e9072008-03-28 14:16:07 -0700810 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811}
812
Milan Brozd469f842007-10-19 22:42:37 +0100813static void dm_crypt_bio_destructor(struct bio *bio)
814{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100815 struct dm_crypt_io *io = bio->bi_private;
Milan Broz6a24c712006-10-03 01:15:40 -0700816 struct crypt_config *cc = io->target->private;
817
818 bio_free(bio, cc->bs);
Milan Brozd469f842007-10-19 22:42:37 +0100819}
Milan Broz6a24c712006-10-03 01:15:40 -0700820
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821/*
822 * Generate a new unfragmented bio with the given size
823 * This should never violate the device limitations
Milan Broz933f01d2008-10-10 13:37:08 +0100824 * May return a smaller bio when running out of pages, indicated by
825 * *out_of_pages set to 1.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 */
Milan Broz933f01d2008-10-10 13:37:08 +0100827static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
828 unsigned *out_of_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829{
Olaf Kirch027581f2007-05-09 02:32:52 -0700830 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -0700831 struct bio *clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Al Virob4e3ca12005-10-21 03:22:34 -0400833 gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
Milan Broz91e10622007-12-13 14:16:10 +0000834 unsigned i, len;
835 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836
Olaf Kirch2f9941b2007-05-09 02:32:53 -0700837 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
Milan Broz8b004452006-10-03 01:15:37 -0700838 if (!clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
Olaf Kirch027581f2007-05-09 02:32:52 -0700841 clone_init(io, clone);
Milan Broz933f01d2008-10-10 13:37:08 +0100842 *out_of_pages = 0;
Milan Broz6a24c712006-10-03 01:15:40 -0700843
Olaf Kirchf97380b2007-05-09 02:32:54 -0700844 for (i = 0; i < nr_iovecs; i++) {
Milan Broz91e10622007-12-13 14:16:10 +0000845 page = mempool_alloc(cc->page_pool, gfp_mask);
Milan Broz933f01d2008-10-10 13:37:08 +0100846 if (!page) {
847 *out_of_pages = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 break;
Milan Broz933f01d2008-10-10 13:37:08 +0100849 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850
851 /*
852 * if additional pages cannot be allocated without waiting,
853 * return a partially allocated bio, the caller will then try
854 * to allocate additional bios while submitting this partial bio
855 */
Olaf Kirchf97380b2007-05-09 02:32:54 -0700856 if (i == (MIN_BIO_PAGES - 1))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
858
Milan Broz91e10622007-12-13 14:16:10 +0000859 len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860
Milan Broz91e10622007-12-13 14:16:10 +0000861 if (!bio_add_page(clone, page, len, 0)) {
862 mempool_free(page, cc->page_pool);
863 break;
864 }
865
866 size -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867 }
868
Milan Broz8b004452006-10-03 01:15:37 -0700869 if (!clone->bi_size) {
870 bio_put(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 return NULL;
872 }
873
Milan Broz8b004452006-10-03 01:15:37 -0700874 return clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875}
876
Neil Brown644bd2f2007-10-16 13:48:46 +0200877static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878{
Neil Brown644bd2f2007-10-16 13:48:46 +0200879 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 struct bio_vec *bv;
881
Neil Brown644bd2f2007-10-16 13:48:46 +0200882 for (i = 0; i < clone->bi_vcnt; i++) {
Milan Broz8b004452006-10-03 01:15:37 -0700883 bv = bio_iovec_idx(clone, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 BUG_ON(!bv->bv_page);
885 mempool_free(bv->bv_page, cc->page_pool);
886 bv->bv_page = NULL;
887 }
888}
889
Milan Brozdc440d1e2008-10-10 13:37:03 +0100890static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti,
891 struct bio *bio, sector_t sector)
892{
893 struct crypt_config *cc = ti->private;
894 struct dm_crypt_io *io;
895
896 io = mempool_alloc(cc->io_pool, GFP_NOIO);
897 io->target = ti;
898 io->base_bio = bio;
899 io->sector = sector;
900 io->error = 0;
Milan Broz393b47e2008-10-21 17:45:02 +0100901 io->base_io = NULL;
Milan Brozdc440d1e2008-10-10 13:37:03 +0100902 atomic_set(&io->pending, 0);
903
904 return io;
905}
906
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100907static void crypt_inc_pending(struct dm_crypt_io *io)
908{
909 atomic_inc(&io->pending);
910}
911
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912/*
913 * One of the bios was finished. Check for completion of
914 * the whole request and correctly clean up the buffer.
Milan Broz393b47e2008-10-21 17:45:02 +0100915 * If base_io is set, wait for the last fragment to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 */
Milan Broz5742fd72008-02-08 02:10:43 +0000917static void crypt_dec_pending(struct dm_crypt_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918{
Milan Broz5742fd72008-02-08 02:10:43 +0000919 struct crypt_config *cc = io->target->private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000920 struct bio *base_bio = io->base_bio;
921 struct dm_crypt_io *base_io = io->base_io;
922 int error = io->error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923
924 if (!atomic_dec_and_test(&io->pending))
925 return;
926
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 mempool_free(io, cc->io_pool);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000928
929 if (likely(!base_io))
930 bio_endio(base_bio, error);
931 else {
932 if (error && !base_io->error)
933 base_io->error = error;
934 crypt_dec_pending(base_io);
935 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700936}
937
938/*
Milan Brozcabf08e2007-10-19 22:38:58 +0100939 * kcryptd/kcryptd_io:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700940 *
941 * Needed because it would be very unwise to do decryption in an
Milan Broz23541d22006-10-03 01:15:39 -0700942 * interrupt context.
Milan Brozcabf08e2007-10-19 22:38:58 +0100943 *
944 * kcryptd performs the actual encryption or decryption.
945 *
946 * kcryptd_io performs the IO submission.
947 *
948 * They must be separated as otherwise the final stages could be
949 * starved by new requests which can block in the first stages due
950 * to memory allocation.
Andi Kleenc0297722011-01-13 19:59:53 +0000951 *
952 * The work is done per CPU global for all dm-crypt instances.
953 * They should not depend on each other and do not block.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 */
NeilBrown6712ecf2007-09-27 12:47:43 +0200955static void crypt_endio(struct bio *clone, int error)
Milan Broz8b004452006-10-03 01:15:37 -0700956{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100957 struct dm_crypt_io *io = clone->bi_private;
Milan Broz8b004452006-10-03 01:15:37 -0700958 struct crypt_config *cc = io->target->private;
Milan Brozee7a4912008-02-08 02:10:46 +0000959 unsigned rw = bio_data_dir(clone);
Milan Broz8b004452006-10-03 01:15:37 -0700960
Milan Brozadfe4772007-12-13 14:15:51 +0000961 if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error))
962 error = -EIO;
963
Milan Broz8b004452006-10-03 01:15:37 -0700964 /*
NeilBrown6712ecf2007-09-27 12:47:43 +0200965 * free the processed pages
Milan Broz8b004452006-10-03 01:15:37 -0700966 */
Milan Brozee7a4912008-02-08 02:10:46 +0000967 if (rw == WRITE)
Neil Brown644bd2f2007-10-16 13:48:46 +0200968 crypt_free_buffer_pages(cc, clone);
Milan Brozee7a4912008-02-08 02:10:46 +0000969
970 bio_put(clone);
971
972 if (rw == READ && !error) {
973 kcryptd_queue_crypt(io);
974 return;
NeilBrown6712ecf2007-09-27 12:47:43 +0200975 }
Milan Broz8b004452006-10-03 01:15:37 -0700976
Milan Brozadfe4772007-12-13 14:15:51 +0000977 if (unlikely(error))
Milan Broz5742fd72008-02-08 02:10:43 +0000978 io->error = error;
979
980 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700981}
982
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100983static void clone_init(struct dm_crypt_io *io, struct bio *clone)
Milan Broz8b004452006-10-03 01:15:37 -0700984{
985 struct crypt_config *cc = io->target->private;
986
987 clone->bi_private = io;
988 clone->bi_end_io = crypt_endio;
989 clone->bi_bdev = cc->dev->bdev;
990 clone->bi_rw = io->base_bio->bi_rw;
Olaf Kirch027581f2007-05-09 02:32:52 -0700991 clone->bi_destructor = dm_crypt_bio_destructor;
Milan Broz8b004452006-10-03 01:15:37 -0700992}
993
Milan Broz20c82532011-01-13 19:59:53 +0000994static void kcryptd_unplug(struct crypt_config *cc)
995{
996 blk_unplug(bdev_get_queue(cc->dev->bdev));
997}
998
999static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
Milan Broz8b004452006-10-03 01:15:37 -07001000{
1001 struct crypt_config *cc = io->target->private;
1002 struct bio *base_bio = io->base_bio;
1003 struct bio *clone;
Milan Broz93e605c2006-10-03 01:15:38 -07001004
Milan Broz8b004452006-10-03 01:15:37 -07001005 /*
1006 * The block layer might modify the bvec array, so always
1007 * copy the required bvecs because we need the original
1008 * one in order to decrypt the whole bio data *afterwards*.
1009 */
Milan Broz20c82532011-01-13 19:59:53 +00001010 clone = bio_alloc_bioset(gfp, bio_segments(base_bio), cc->bs);
1011 if (!clone) {
1012 kcryptd_unplug(cc);
1013 return 1;
Milan Broz93e605c2006-10-03 01:15:38 -07001014 }
Milan Broz8b004452006-10-03 01:15:37 -07001015
Milan Broz20c82532011-01-13 19:59:53 +00001016 crypt_inc_pending(io);
1017
Milan Broz8b004452006-10-03 01:15:37 -07001018 clone_init(io, clone);
1019 clone->bi_idx = 0;
1020 clone->bi_vcnt = bio_segments(base_bio);
1021 clone->bi_size = base_bio->bi_size;
Milan Broz0c395b02008-02-08 02:10:54 +00001022 clone->bi_sector = cc->start + io->sector;
Milan Broz8b004452006-10-03 01:15:37 -07001023 memcpy(clone->bi_io_vec, bio_iovec(base_bio),
1024 sizeof(struct bio_vec) * clone->bi_vcnt);
Milan Broz8b004452006-10-03 01:15:37 -07001025
Milan Broz93e605c2006-10-03 01:15:38 -07001026 generic_make_request(clone);
Milan Broz20c82532011-01-13 19:59:53 +00001027 return 0;
Milan Broz8b004452006-10-03 01:15:37 -07001028}
1029
Milan Broz4e4eef62008-02-08 02:10:49 +00001030static void kcryptd_io_write(struct dm_crypt_io *io)
1031{
Milan Broz95497a92008-02-08 02:11:12 +00001032 struct bio *clone = io->ctx.bio_out;
Milan Broz95497a92008-02-08 02:11:12 +00001033 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +00001034}
1035
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001036static void kcryptd_io(struct work_struct *work)
1037{
1038 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1039
Milan Broz20c82532011-01-13 19:59:53 +00001040 if (bio_data_dir(io->base_bio) == READ) {
1041 crypt_inc_pending(io);
1042 if (kcryptd_io_read(io, GFP_NOIO))
1043 io->error = -ENOMEM;
1044 crypt_dec_pending(io);
1045 } else
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001046 kcryptd_io_write(io);
1047}
1048
1049static void kcryptd_queue_io(struct dm_crypt_io *io)
1050{
1051 struct crypt_config *cc = io->target->private;
1052
1053 INIT_WORK(&io->work, kcryptd_io);
1054 queue_work(cc->io_queue, &io->work);
1055}
1056
Milan Broz95497a92008-02-08 02:11:12 +00001057static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io,
1058 int error, int async)
Milan Broz4e4eef62008-02-08 02:10:49 +00001059{
Milan Brozdec1ced2008-02-08 02:10:57 +00001060 struct bio *clone = io->ctx.bio_out;
1061 struct crypt_config *cc = io->target->private;
1062
1063 if (unlikely(error < 0)) {
1064 crypt_free_buffer_pages(cc, clone);
1065 bio_put(clone);
1066 io->error = -EIO;
Milan Broz6c031f42008-10-10 13:37:06 +01001067 crypt_dec_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +00001068 return;
1069 }
1070
1071 /* crypt_convert should have filled the clone bio */
1072 BUG_ON(io->ctx.idx_out < clone->bi_vcnt);
1073
1074 clone->bi_sector = cc->start + io->sector;
Milan Broz899c95d2008-02-08 02:11:02 +00001075
Milan Broz95497a92008-02-08 02:11:12 +00001076 if (async)
1077 kcryptd_queue_io(io);
Alasdair G Kergon1e37bb82008-10-10 13:37:05 +01001078 else
Milan Broz95497a92008-02-08 02:11:12 +00001079 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +00001080}
1081
Milan Brozfc5a5e92008-10-10 13:37:04 +01001082static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -07001083{
1084 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -07001085 struct bio *clone;
Milan Broz393b47e2008-10-21 17:45:02 +01001086 struct dm_crypt_io *new_io;
Milan Brozc8081612008-10-10 13:37:08 +01001087 int crypt_finished;
Milan Broz933f01d2008-10-10 13:37:08 +01001088 unsigned out_of_pages = 0;
Milan Brozdec1ced2008-02-08 02:10:57 +00001089 unsigned remaining = io->base_bio->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +01001090 sector_t sector = io->sector;
Milan Brozdec1ced2008-02-08 02:10:57 +00001091 int r;
Milan Broz8b004452006-10-03 01:15:37 -07001092
Milan Broz93e605c2006-10-03 01:15:38 -07001093 /*
Milan Brozfc5a5e92008-10-10 13:37:04 +01001094 * Prevent io from disappearing until this function completes.
1095 */
1096 crypt_inc_pending(io);
Milan Brozb635b002008-10-21 17:45:00 +01001097 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
Milan Brozfc5a5e92008-10-10 13:37:04 +01001098
1099 /*
Milan Broz93e605c2006-10-03 01:15:38 -07001100 * The allocated buffers can be smaller than the whole bio,
1101 * so repeat the whole process until all the data can be handled.
1102 */
1103 while (remaining) {
Milan Broz933f01d2008-10-10 13:37:08 +01001104 clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
Milan Broz23541d22006-10-03 01:15:39 -07001105 if (unlikely(!clone)) {
Milan Broz5742fd72008-02-08 02:10:43 +00001106 io->error = -ENOMEM;
Milan Brozfc5a5e92008-10-10 13:37:04 +01001107 break;
Milan Broz23541d22006-10-03 01:15:39 -07001108 }
Milan Broz93e605c2006-10-03 01:15:38 -07001109
Milan Broz53017032008-02-08 02:10:38 +00001110 io->ctx.bio_out = clone;
1111 io->ctx.idx_out = 0;
Milan Broz93e605c2006-10-03 01:15:38 -07001112
Milan Broz93e605c2006-10-03 01:15:38 -07001113 remaining -= clone->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +01001114 sector += bio_sectors(clone);
Milan Brozdec1ced2008-02-08 02:10:57 +00001115
Milan Broz4e594092008-10-10 13:37:07 +01001116 crypt_inc_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +00001117 r = crypt_convert(cc, &io->ctx);
Milan Brozc8081612008-10-10 13:37:08 +01001118 crypt_finished = atomic_dec_and_test(&io->ctx.pending);
Milan Brozdec1ced2008-02-08 02:10:57 +00001119
Milan Brozc8081612008-10-10 13:37:08 +01001120 /* Encryption was already finished, submit io now */
1121 if (crypt_finished) {
Milan Broz3a7f6c92008-02-08 02:11:14 +00001122 kcryptd_crypt_write_io_submit(io, r, 0);
Milan Brozc8081612008-10-10 13:37:08 +01001123
1124 /*
1125 * If there was an error, do not try next fragments.
1126 * For async, error is processed in async handler.
1127 */
Milan Broz6c031f42008-10-10 13:37:06 +01001128 if (unlikely(r < 0))
Milan Brozfc5a5e92008-10-10 13:37:04 +01001129 break;
Milan Brozb635b002008-10-21 17:45:00 +01001130
1131 io->sector = sector;
Milan Broz4e594092008-10-10 13:37:07 +01001132 }
Milan Broz93e605c2006-10-03 01:15:38 -07001133
Milan Broz933f01d2008-10-10 13:37:08 +01001134 /*
1135 * Out of memory -> run queues
1136 * But don't wait if split was due to the io size restriction
1137 */
1138 if (unlikely(out_of_pages))
Jens Axboe8aa7e842009-07-09 14:52:32 +02001139 congestion_wait(BLK_RW_ASYNC, HZ/100);
Milan Broz933f01d2008-10-10 13:37:08 +01001140
Milan Broz393b47e2008-10-21 17:45:02 +01001141 /*
1142 * With async crypto it is unsafe to share the crypto context
1143 * between fragments, so switch to a new dm_crypt_io structure.
1144 */
1145 if (unlikely(!crypt_finished && remaining)) {
1146 new_io = crypt_io_alloc(io->target, io->base_bio,
1147 sector);
1148 crypt_inc_pending(new_io);
1149 crypt_convert_init(cc, &new_io->ctx, NULL,
1150 io->base_bio, sector);
1151 new_io->ctx.idx_in = io->ctx.idx_in;
1152 new_io->ctx.offset_in = io->ctx.offset_in;
1153
1154 /*
1155 * Fragments after the first use the base_io
1156 * pending count.
1157 */
1158 if (!io->base_io)
1159 new_io->base_io = io;
1160 else {
1161 new_io->base_io = io->base_io;
1162 crypt_inc_pending(io->base_io);
1163 crypt_dec_pending(io);
1164 }
1165
1166 io = new_io;
1167 }
Milan Broz8b004452006-10-03 01:15:37 -07001168 }
Milan Broz899c95d2008-02-08 02:11:02 +00001169
1170 crypt_dec_pending(io);
Milan Broz84131db2008-02-08 02:10:59 +00001171}
1172
Milan Broz4e4eef62008-02-08 02:10:49 +00001173static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error)
Milan Broz5742fd72008-02-08 02:10:43 +00001174{
1175 if (unlikely(error < 0))
1176 io->error = -EIO;
1177
1178 crypt_dec_pending(io);
1179}
1180
Milan Broz4e4eef62008-02-08 02:10:49 +00001181static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -07001182{
1183 struct crypt_config *cc = io->target->private;
Milan Broz5742fd72008-02-08 02:10:43 +00001184 int r = 0;
Milan Broz8b004452006-10-03 01:15:37 -07001185
Milan Broz3e1a8bd2008-10-10 13:37:02 +01001186 crypt_inc_pending(io);
Milan Broz3a7f6c92008-02-08 02:11:14 +00001187
Milan Broz53017032008-02-08 02:10:38 +00001188 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
Milan Broz0c395b02008-02-08 02:10:54 +00001189 io->sector);
Milan Broz8b004452006-10-03 01:15:37 -07001190
Milan Broz5742fd72008-02-08 02:10:43 +00001191 r = crypt_convert(cc, &io->ctx);
1192
Milan Broz3f1e9072008-03-28 14:16:07 -07001193 if (atomic_dec_and_test(&io->ctx.pending))
Milan Broz3a7f6c92008-02-08 02:11:14 +00001194 kcryptd_crypt_read_done(io, r);
1195
1196 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -07001197}
1198
Milan Broz95497a92008-02-08 02:11:12 +00001199static void kcryptd_async_done(struct crypto_async_request *async_req,
1200 int error)
1201{
Huang Yingb2174ee2009-03-16 17:44:33 +00001202 struct dm_crypt_request *dmreq = async_req->data;
1203 struct convert_context *ctx = dmreq->ctx;
Milan Broz95497a92008-02-08 02:11:12 +00001204 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1205 struct crypt_config *cc = io->target->private;
1206
1207 if (error == -EINPROGRESS) {
1208 complete(&ctx->restart);
1209 return;
1210 }
1211
Milan Broz2dc53272011-01-13 19:59:54 +00001212 if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
1213 error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq);
1214
Huang Yingb2174ee2009-03-16 17:44:33 +00001215 mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool);
Milan Broz95497a92008-02-08 02:11:12 +00001216
1217 if (!atomic_dec_and_test(&ctx->pending))
1218 return;
1219
1220 if (bio_data_dir(io->base_bio) == READ)
1221 kcryptd_crypt_read_done(io, error);
1222 else
1223 kcryptd_crypt_write_io_submit(io, error, 1);
1224}
1225
Milan Broz4e4eef62008-02-08 02:10:49 +00001226static void kcryptd_crypt(struct work_struct *work)
1227{
1228 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1229
1230 if (bio_data_dir(io->base_bio) == READ)
1231 kcryptd_crypt_read_convert(io);
1232 else
1233 kcryptd_crypt_write_convert(io);
Milan Broz8b004452006-10-03 01:15:37 -07001234}
1235
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001236static void kcryptd_queue_crypt(struct dm_crypt_io *io)
1237{
1238 struct crypt_config *cc = io->target->private;
1239
1240 INIT_WORK(&io->work, kcryptd_crypt);
1241 queue_work(cc->crypt_queue, &io->work);
1242}
1243
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244/*
1245 * Decode key from its hex representation
1246 */
1247static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
1248{
1249 char buffer[3];
1250 char *endp;
1251 unsigned int i;
1252
1253 buffer[2] = '\0';
1254
Milan Broz8b004452006-10-03 01:15:37 -07001255 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 buffer[0] = *hex++;
1257 buffer[1] = *hex++;
1258
1259 key[i] = (u8)simple_strtoul(buffer, &endp, 16);
1260
1261 if (endp != &buffer[2])
1262 return -EINVAL;
1263 }
1264
1265 if (*hex != '\0')
1266 return -EINVAL;
1267
1268 return 0;
1269}
1270
1271/*
1272 * Encode key into its hex representation
1273 */
1274static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
1275{
1276 unsigned int i;
1277
Milan Broz8b004452006-10-03 01:15:37 -07001278 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001279 sprintf(hex, "%02x", *key);
1280 hex += 2;
1281 key++;
1282 }
1283}
1284
Milan Brozd1f96422011-01-13 19:59:54 +00001285static void crypt_free_tfms(struct crypt_config *cc, int cpu)
1286{
1287 struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);
1288 unsigned i;
1289
1290 for (i = 0; i < cc->tfms_count; i++)
1291 if (cpu_cc->tfms[i] && !IS_ERR(cpu_cc->tfms[i])) {
1292 crypto_free_ablkcipher(cpu_cc->tfms[i]);
1293 cpu_cc->tfms[i] = NULL;
1294 }
1295}
1296
1297static int crypt_alloc_tfms(struct crypt_config *cc, int cpu, char *ciphermode)
1298{
1299 struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);
1300 unsigned i;
1301 int err;
1302
1303 for (i = 0; i < cc->tfms_count; i++) {
1304 cpu_cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
1305 if (IS_ERR(cpu_cc->tfms[i])) {
1306 err = PTR_ERR(cpu_cc->tfms[i]);
1307 crypt_free_tfms(cc, cpu);
1308 return err;
1309 }
1310 }
1311
1312 return 0;
1313}
1314
Andi Kleenc0297722011-01-13 19:59:53 +00001315static int crypt_setkey_allcpus(struct crypt_config *cc)
1316{
Milan Brozd1f96422011-01-13 19:59:54 +00001317 unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count);
1318 int cpu, err = 0, i, r;
Andi Kleenc0297722011-01-13 19:59:53 +00001319
1320 for_each_possible_cpu(cpu) {
Milan Brozd1f96422011-01-13 19:59:54 +00001321 for (i = 0; i < cc->tfms_count; i++) {
1322 r = crypto_ablkcipher_setkey(per_cpu_ptr(cc->cpu, cpu)->tfms[i],
1323 cc->key + (i * subkey_size), subkey_size);
1324 if (r)
1325 err = r;
1326 }
Andi Kleenc0297722011-01-13 19:59:53 +00001327 }
1328
1329 return err;
1330}
1331
Milan Broze48d4bb2006-10-03 01:15:37 -07001332static int crypt_set_key(struct crypt_config *cc, char *key)
1333{
Milan Broz69a8cfc2011-01-13 19:59:49 +00001334 /* The key size may not be changed. */
1335 if (cc->key_size != (strlen(key) >> 1))
Milan Broze48d4bb2006-10-03 01:15:37 -07001336 return -EINVAL;
1337
Milan Broz69a8cfc2011-01-13 19:59:49 +00001338 /* Hyphen (which gives a key_size of zero) means there is no key. */
1339 if (!cc->key_size && strcmp(key, "-"))
1340 return -EINVAL;
Milan Broze48d4bb2006-10-03 01:15:37 -07001341
Milan Broz69a8cfc2011-01-13 19:59:49 +00001342 if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
Milan Broze48d4bb2006-10-03 01:15:37 -07001343 return -EINVAL;
1344
1345 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
1346
Andi Kleenc0297722011-01-13 19:59:53 +00001347 return crypt_setkey_allcpus(cc);
Milan Broze48d4bb2006-10-03 01:15:37 -07001348}
1349
1350static int crypt_wipe_key(struct crypt_config *cc)
1351{
1352 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
1353 memset(&cc->key, 0, cc->key_size * sizeof(u8));
Andi Kleenc0297722011-01-13 19:59:53 +00001354
1355 return crypt_setkey_allcpus(cc);
Milan Broze48d4bb2006-10-03 01:15:37 -07001356}
1357
Milan Broz28513fc2010-08-12 04:14:06 +01001358static void crypt_dtr(struct dm_target *ti)
1359{
1360 struct crypt_config *cc = ti->private;
Andi Kleenc0297722011-01-13 19:59:53 +00001361 struct crypt_cpu *cpu_cc;
1362 int cpu;
Milan Broz28513fc2010-08-12 04:14:06 +01001363
1364 ti->private = NULL;
1365
1366 if (!cc)
1367 return;
1368
1369 if (cc->io_queue)
1370 destroy_workqueue(cc->io_queue);
1371 if (cc->crypt_queue)
1372 destroy_workqueue(cc->crypt_queue);
1373
Andi Kleenc0297722011-01-13 19:59:53 +00001374 if (cc->cpu)
1375 for_each_possible_cpu(cpu) {
1376 cpu_cc = per_cpu_ptr(cc->cpu, cpu);
1377 if (cpu_cc->req)
1378 mempool_free(cpu_cc->req, cc->req_pool);
Milan Brozd1f96422011-01-13 19:59:54 +00001379 crypt_free_tfms(cc, cpu);
Andi Kleenc0297722011-01-13 19:59:53 +00001380 }
1381
Milan Broz28513fc2010-08-12 04:14:06 +01001382 if (cc->bs)
1383 bioset_free(cc->bs);
1384
1385 if (cc->page_pool)
1386 mempool_destroy(cc->page_pool);
1387 if (cc->req_pool)
1388 mempool_destroy(cc->req_pool);
1389 if (cc->io_pool)
1390 mempool_destroy(cc->io_pool);
1391
1392 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
1393 cc->iv_gen_ops->dtr(cc);
1394
Milan Broz28513fc2010-08-12 04:14:06 +01001395 if (cc->dev)
1396 dm_put_device(ti, cc->dev);
1397
Andi Kleenc0297722011-01-13 19:59:53 +00001398 if (cc->cpu)
1399 free_percpu(cc->cpu);
1400
Milan Broz5ebaee62010-08-12 04:14:07 +01001401 kzfree(cc->cipher);
Milan Broz7dbcd132011-01-13 19:59:52 +00001402 kzfree(cc->cipher_string);
Milan Broz28513fc2010-08-12 04:14:06 +01001403
1404 /* Must zero key material before freeing */
1405 kzfree(cc);
1406}
1407
Milan Broz5ebaee62010-08-12 04:14:07 +01001408static int crypt_ctr_cipher(struct dm_target *ti,
1409 char *cipher_in, char *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410{
Milan Broz5ebaee62010-08-12 04:14:07 +01001411 struct crypt_config *cc = ti->private;
Milan Brozd1f96422011-01-13 19:59:54 +00001412 char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
Milan Broz5ebaee62010-08-12 04:14:07 +01001413 char *cipher_api = NULL;
Andi Kleenc0297722011-01-13 19:59:53 +00001414 int cpu, ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415
Milan Broz5ebaee62010-08-12 04:14:07 +01001416 /* Convert to crypto api definition? */
1417 if (strchr(cipher_in, '(')) {
1418 ti->error = "Bad cipher specification";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 return -EINVAL;
1420 }
1421
Milan Broz7dbcd132011-01-13 19:59:52 +00001422 cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
1423 if (!cc->cipher_string)
1424 goto bad_mem;
1425
Milan Broz5ebaee62010-08-12 04:14:07 +01001426 /*
1427 * Legacy dm-crypt cipher specification
Milan Brozd1f96422011-01-13 19:59:54 +00001428 * cipher[:keycount]-mode-iv:ivopts
Milan Broz5ebaee62010-08-12 04:14:07 +01001429 */
1430 tmp = cipher_in;
Milan Brozd1f96422011-01-13 19:59:54 +00001431 keycount = strsep(&tmp, "-");
1432 cipher = strsep(&keycount, ":");
1433
1434 if (!keycount)
1435 cc->tfms_count = 1;
1436 else if (sscanf(keycount, "%u", &cc->tfms_count) != 1 ||
1437 !is_power_of_2(cc->tfms_count)) {
1438 ti->error = "Bad cipher key count specification";
1439 return -EINVAL;
1440 }
1441 cc->key_parts = cc->tfms_count;
Milan Broz5ebaee62010-08-12 04:14:07 +01001442
1443 cc->cipher = kstrdup(cipher, GFP_KERNEL);
1444 if (!cc->cipher)
1445 goto bad_mem;
1446
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447 chainmode = strsep(&tmp, "-");
1448 ivopts = strsep(&tmp, "-");
1449 ivmode = strsep(&ivopts, ":");
1450
1451 if (tmp)
Milan Broz5ebaee62010-08-12 04:14:07 +01001452 DMWARN("Ignoring unexpected additional cipher options");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453
Milan Brozd1f96422011-01-13 19:59:54 +00001454 cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)) +
1455 cc->tfms_count * sizeof(*(cc->cpu->tfms)),
1456 __alignof__(struct crypt_cpu));
Andi Kleenc0297722011-01-13 19:59:53 +00001457 if (!cc->cpu) {
1458 ti->error = "Cannot allocate per cpu state";
1459 goto bad_mem;
1460 }
1461
Milan Broz7dbcd132011-01-13 19:59:52 +00001462 /*
1463 * For compatibility with the original dm-crypt mapping format, if
1464 * only the cipher name is supplied, use cbc-plain.
1465 */
Milan Broz5ebaee62010-08-12 04:14:07 +01001466 if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467 chainmode = "cbc";
1468 ivmode = "plain";
1469 }
1470
Herbert Xud1806f62006-08-22 20:29:17 +10001471 if (strcmp(chainmode, "ecb") && !ivmode) {
Milan Broz5ebaee62010-08-12 04:14:07 +01001472 ti->error = "IV mechanism required";
1473 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 }
1475
Milan Broz5ebaee62010-08-12 04:14:07 +01001476 cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
1477 if (!cipher_api)
1478 goto bad_mem;
1479
1480 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
1481 "%s(%s)", chainmode, cipher);
1482 if (ret < 0) {
1483 kfree(cipher_api);
1484 goto bad_mem;
Herbert Xud1806f62006-08-22 20:29:17 +10001485 }
1486
Milan Broz5ebaee62010-08-12 04:14:07 +01001487 /* Allocate cipher */
Andi Kleenc0297722011-01-13 19:59:53 +00001488 for_each_possible_cpu(cpu) {
Milan Brozd1f96422011-01-13 19:59:54 +00001489 ret = crypt_alloc_tfms(cc, cpu, cipher_api);
1490 if (ret < 0) {
Andi Kleenc0297722011-01-13 19:59:53 +00001491 ti->error = "Error allocating crypto tfm";
1492 goto bad;
1493 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495
Milan Broz5ebaee62010-08-12 04:14:07 +01001496 /* Initialize and set key */
1497 ret = crypt_set_key(cc, key);
Milan Broz28513fc2010-08-12 04:14:06 +01001498 if (ret < 0) {
Milan Broz0b430952009-12-10 23:51:55 +00001499 ti->error = "Error decoding and setting key";
Milan Broz28513fc2010-08-12 04:14:06 +01001500 goto bad;
Milan Broz0b430952009-12-10 23:51:55 +00001501 }
1502
Milan Broz5ebaee62010-08-12 04:14:07 +01001503 /* Initialize IV */
Andi Kleenc0297722011-01-13 19:59:53 +00001504 cc->iv_size = crypto_ablkcipher_ivsize(any_tfm(cc));
Milan Broz5ebaee62010-08-12 04:14:07 +01001505 if (cc->iv_size)
1506 /* at least a 64 bit sector number should fit in our buffer */
1507 cc->iv_size = max(cc->iv_size,
1508 (unsigned int)(sizeof(u64) / sizeof(u8)));
1509 else if (ivmode) {
1510 DMWARN("Selected cipher does not support IVs");
1511 ivmode = NULL;
1512 }
1513
1514 /* Choose ivmode, see comments at iv code. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 if (ivmode == NULL)
1516 cc->iv_gen_ops = NULL;
1517 else if (strcmp(ivmode, "plain") == 0)
1518 cc->iv_gen_ops = &crypt_iv_plain_ops;
Milan Broz61afef62009-12-10 23:52:25 +00001519 else if (strcmp(ivmode, "plain64") == 0)
1520 cc->iv_gen_ops = &crypt_iv_plain64_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 else if (strcmp(ivmode, "essiv") == 0)
1522 cc->iv_gen_ops = &crypt_iv_essiv_ops;
Rik Snel48527fa2006-09-03 08:56:39 +10001523 else if (strcmp(ivmode, "benbi") == 0)
1524 cc->iv_gen_ops = &crypt_iv_benbi_ops;
Ludwig Nussel46b47732007-05-09 02:32:55 -07001525 else if (strcmp(ivmode, "null") == 0)
1526 cc->iv_gen_ops = &crypt_iv_null_ops;
Milan Broz34745782011-01-13 19:59:55 +00001527 else if (strcmp(ivmode, "lmk") == 0) {
1528 cc->iv_gen_ops = &crypt_iv_lmk_ops;
1529 /* Version 2 and 3 is recognised according
1530 * to length of provided multi-key string.
1531 * If present (version 3), last key is used as IV seed.
1532 */
1533 if (cc->key_size % cc->key_parts)
1534 cc->key_parts++;
1535 } else {
Milan Broz5ebaee62010-08-12 04:14:07 +01001536 ret = -EINVAL;
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001537 ti->error = "Invalid IV mode";
Milan Broz28513fc2010-08-12 04:14:06 +01001538 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 }
1540
Milan Broz28513fc2010-08-12 04:14:06 +01001541 /* Allocate IV */
1542 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
1543 ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
1544 if (ret < 0) {
1545 ti->error = "Error creating IV";
1546 goto bad;
1547 }
Milan Brozb95bf2d2009-12-10 23:51:56 +00001548 }
1549
Milan Broz28513fc2010-08-12 04:14:06 +01001550 /* Initialize IV (set keys for ESSIV etc) */
1551 if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
1552 ret = cc->iv_gen_ops->init(cc);
1553 if (ret < 0) {
1554 ti->error = "Error initialising IV";
1555 goto bad;
1556 }
1557 }
1558
Milan Broz5ebaee62010-08-12 04:14:07 +01001559 ret = 0;
1560bad:
1561 kfree(cipher_api);
1562 return ret;
1563
1564bad_mem:
1565 ti->error = "Cannot allocate cipher strings";
1566 return -ENOMEM;
1567}
1568
1569/*
1570 * Construct an encryption mapping:
1571 * <cipher> <key> <iv_offset> <dev_path> <start>
1572 */
1573static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1574{
1575 struct crypt_config *cc;
1576 unsigned int key_size;
1577 unsigned long long tmpll;
1578 int ret;
1579
1580 if (argc != 5) {
1581 ti->error = "Not enough arguments";
1582 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 }
1584
Milan Broz5ebaee62010-08-12 04:14:07 +01001585 key_size = strlen(argv[1]) >> 1;
1586
1587 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
1588 if (!cc) {
1589 ti->error = "Cannot allocate encryption context";
1590 return -ENOMEM;
1591 }
Milan Broz69a8cfc2011-01-13 19:59:49 +00001592 cc->key_size = key_size;
Milan Broz5ebaee62010-08-12 04:14:07 +01001593
1594 ti->private = cc;
1595 ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
1596 if (ret < 0)
1597 goto bad;
1598
Milan Broz28513fc2010-08-12 04:14:06 +01001599 ret = -ENOMEM;
Matthew Dobson93d23412006-03-26 01:37:50 -08001600 cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 if (!cc->io_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001602 ti->error = "Cannot allocate crypt io mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001603 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 }
1605
Milan Brozddd42ed2008-02-08 02:11:07 +00001606 cc->dmreq_start = sizeof(struct ablkcipher_request);
Andi Kleenc0297722011-01-13 19:59:53 +00001607 cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc));
Milan Brozddd42ed2008-02-08 02:11:07 +00001608 cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment());
Andi Kleenc0297722011-01-13 19:59:53 +00001609 cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) &
Milan Broz3a7f6c92008-02-08 02:11:14 +00001610 ~(crypto_tfm_ctx_alignment() - 1);
Milan Brozddd42ed2008-02-08 02:11:07 +00001611
1612 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
1613 sizeof(struct dm_crypt_request) + cc->iv_size);
1614 if (!cc->req_pool) {
1615 ti->error = "Cannot allocate crypt request mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001616 goto bad;
Milan Brozddd42ed2008-02-08 02:11:07 +00001617 }
Milan Brozddd42ed2008-02-08 02:11:07 +00001618
Matthew Dobsona19b27c2006-03-26 01:37:45 -08001619 cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620 if (!cc->page_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001621 ti->error = "Cannot allocate page mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001622 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623 }
1624
Jens Axboebb799ca2008-12-10 15:35:05 +01001625 cc->bs = bioset_create(MIN_IOS, 0);
Milan Broz6a24c712006-10-03 01:15:40 -07001626 if (!cc->bs) {
1627 ti->error = "Cannot allocate crypt bioset";
Milan Broz28513fc2010-08-12 04:14:06 +01001628 goto bad;
Milan Broz6a24c712006-10-03 01:15:40 -07001629 }
1630
Milan Broz28513fc2010-08-12 04:14:06 +01001631 ret = -EINVAL;
Andrew Morton4ee218c2006-03-27 01:17:48 -08001632 if (sscanf(argv[2], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001633 ti->error = "Invalid iv_offset sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001634 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001636 cc->iv_offset = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637
Milan Broz28513fc2010-08-12 04:14:06 +01001638 if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) {
1639 ti->error = "Device lookup failed";
1640 goto bad;
1641 }
1642
Andrew Morton4ee218c2006-03-27 01:17:48 -08001643 if (sscanf(argv[4], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001644 ti->error = "Invalid device sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001645 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001647 cc->start = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648
Milan Broz28513fc2010-08-12 04:14:06 +01001649 ret = -ENOMEM;
Andi Kleenc0297722011-01-13 19:59:53 +00001650 cc->io_queue = alloc_workqueue("kcryptd_io",
1651 WQ_NON_REENTRANT|
1652 WQ_MEM_RECLAIM,
1653 1);
Milan Brozcabf08e2007-10-19 22:38:58 +01001654 if (!cc->io_queue) {
1655 ti->error = "Couldn't create kcryptd io queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001656 goto bad;
Milan Brozcabf08e2007-10-19 22:38:58 +01001657 }
1658
Andi Kleenc0297722011-01-13 19:59:53 +00001659 cc->crypt_queue = alloc_workqueue("kcryptd",
1660 WQ_NON_REENTRANT|
1661 WQ_CPU_INTENSIVE|
1662 WQ_MEM_RECLAIM,
1663 1);
Milan Brozcabf08e2007-10-19 22:38:58 +01001664 if (!cc->crypt_queue) {
Milan Broz9934a8b2007-10-19 22:38:57 +01001665 ti->error = "Couldn't create kcryptd queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001666 goto bad;
Milan Broz9934a8b2007-10-19 22:38:57 +01001667 }
1668
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001669 ti->num_flush_requests = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 return 0;
1671
Milan Broz28513fc2010-08-12 04:14:06 +01001672bad:
1673 crypt_dtr(ti);
1674 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675}
1676
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677static int crypt_map(struct dm_target *ti, struct bio *bio,
1678 union map_info *map_context)
1679{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001680 struct dm_crypt_io *io;
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001681 struct crypt_config *cc;
1682
Tejun Heod87f4c12010-09-03 11:56:19 +02001683 if (bio->bi_rw & REQ_FLUSH) {
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001684 cc = ti->private;
1685 bio->bi_bdev = cc->dev->bdev;
1686 return DM_MAPIO_REMAPPED;
1687 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688
Alasdair G Kergonb441a2622010-08-12 04:14:11 +01001689 io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector));
Milan Brozcabf08e2007-10-19 22:38:58 +01001690
Milan Broz20c82532011-01-13 19:59:53 +00001691 if (bio_data_dir(io->base_bio) == READ) {
1692 if (kcryptd_io_read(io, GFP_NOWAIT))
1693 kcryptd_queue_io(io);
1694 } else
Milan Brozcabf08e2007-10-19 22:38:58 +01001695 kcryptd_queue_crypt(io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001697 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698}
1699
1700static int crypt_status(struct dm_target *ti, status_type_t type,
1701 char *result, unsigned int maxlen)
1702{
Milan Broz5ebaee62010-08-12 04:14:07 +01001703 struct crypt_config *cc = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 unsigned int sz = 0;
1705
1706 switch (type) {
1707 case STATUSTYPE_INFO:
1708 result[0] = '\0';
1709 break;
1710
1711 case STATUSTYPE_TABLE:
Milan Broz7dbcd132011-01-13 19:59:52 +00001712 DMEMIT("%s ", cc->cipher_string);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001713
1714 if (cc->key_size > 0) {
1715 if ((maxlen - sz) < ((cc->key_size << 1) + 1))
1716 return -ENOMEM;
1717
1718 crypt_encode_key(result + sz, cc->key, cc->key_size);
1719 sz += cc->key_size << 1;
1720 } else {
1721 if (sz >= maxlen)
1722 return -ENOMEM;
1723 result[sz++] = '-';
1724 }
1725
Andrew Morton4ee218c2006-03-27 01:17:48 -08001726 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
1727 cc->dev->name, (unsigned long long)cc->start);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728 break;
1729 }
1730 return 0;
1731}
1732
Milan Broze48d4bb2006-10-03 01:15:37 -07001733static void crypt_postsuspend(struct dm_target *ti)
1734{
1735 struct crypt_config *cc = ti->private;
1736
1737 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1738}
1739
1740static int crypt_preresume(struct dm_target *ti)
1741{
1742 struct crypt_config *cc = ti->private;
1743
1744 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
1745 DMERR("aborting resume - crypt key is not set.");
1746 return -EAGAIN;
1747 }
1748
1749 return 0;
1750}
1751
1752static void crypt_resume(struct dm_target *ti)
1753{
1754 struct crypt_config *cc = ti->private;
1755
1756 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1757}
1758
1759/* Message interface
1760 * key set <key>
1761 * key wipe
1762 */
1763static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
1764{
1765 struct crypt_config *cc = ti->private;
Milan Broz542da312009-12-10 23:51:57 +00001766 int ret = -EINVAL;
Milan Broze48d4bb2006-10-03 01:15:37 -07001767
1768 if (argc < 2)
1769 goto error;
1770
1771 if (!strnicmp(argv[0], MESG_STR("key"))) {
1772 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
1773 DMWARN("not suspended during key manipulation.");
1774 return -EINVAL;
1775 }
Milan Broz542da312009-12-10 23:51:57 +00001776 if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) {
1777 ret = crypt_set_key(cc, argv[2]);
1778 if (ret)
1779 return ret;
1780 if (cc->iv_gen_ops && cc->iv_gen_ops->init)
1781 ret = cc->iv_gen_ops->init(cc);
1782 return ret;
1783 }
1784 if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) {
1785 if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
1786 ret = cc->iv_gen_ops->wipe(cc);
1787 if (ret)
1788 return ret;
1789 }
Milan Broze48d4bb2006-10-03 01:15:37 -07001790 return crypt_wipe_key(cc);
Milan Broz542da312009-12-10 23:51:57 +00001791 }
Milan Broze48d4bb2006-10-03 01:15:37 -07001792 }
1793
1794error:
1795 DMWARN("unrecognised message received.");
1796 return -EINVAL;
1797}
1798
Milan Brozd41e26b2008-07-21 12:00:40 +01001799static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
1800 struct bio_vec *biovec, int max_size)
1801{
1802 struct crypt_config *cc = ti->private;
1803 struct request_queue *q = bdev_get_queue(cc->dev->bdev);
1804
1805 if (!q->merge_bvec_fn)
1806 return max_size;
1807
1808 bvm->bi_bdev = cc->dev->bdev;
Alasdair G Kergonb441a2622010-08-12 04:14:11 +01001809 bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector);
Milan Brozd41e26b2008-07-21 12:00:40 +01001810
1811 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
1812}
1813
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001814static int crypt_iterate_devices(struct dm_target *ti,
1815 iterate_devices_callout_fn fn, void *data)
1816{
1817 struct crypt_config *cc = ti->private;
1818
Mike Snitzer5dea2712009-07-23 20:30:42 +01001819 return fn(ti, cc->dev, cc->start, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001820}
1821
Linus Torvalds1da177e2005-04-16 15:20:36 -07001822static struct target_type crypt_target = {
1823 .name = "crypt",
Milan Brozd1f96422011-01-13 19:59:54 +00001824 .version = {1, 10, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825 .module = THIS_MODULE,
1826 .ctr = crypt_ctr,
1827 .dtr = crypt_dtr,
1828 .map = crypt_map,
1829 .status = crypt_status,
Milan Broze48d4bb2006-10-03 01:15:37 -07001830 .postsuspend = crypt_postsuspend,
1831 .preresume = crypt_preresume,
1832 .resume = crypt_resume,
1833 .message = crypt_message,
Milan Brozd41e26b2008-07-21 12:00:40 +01001834 .merge = crypt_merge,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001835 .iterate_devices = crypt_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836};
1837
1838static int __init dm_crypt_init(void)
1839{
1840 int r;
1841
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001842 _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 if (!_crypt_io_pool)
1844 return -ENOMEM;
1845
Linus Torvalds1da177e2005-04-16 15:20:36 -07001846 r = dm_register_target(&crypt_target);
1847 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001848 DMERR("register failed %d", r);
Milan Broz9934a8b2007-10-19 22:38:57 +01001849 kmem_cache_destroy(_crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850 }
1851
Linus Torvalds1da177e2005-04-16 15:20:36 -07001852 return r;
1853}
1854
1855static void __exit dm_crypt_exit(void)
1856{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001857 dm_unregister_target(&crypt_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858 kmem_cache_destroy(_crypt_io_pool);
1859}
1860
1861module_init(dm_crypt_init);
1862module_exit(dm_crypt_exit);
1863
1864MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1865MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
1866MODULE_LICENSE("GPL");