blob: ed1038164019a1d5935937037c6aae2bd429c666 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
Milan Broz3f1e9072008-03-28 14:16:07 -07004 * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * This file is released under the GPL.
7 */
8
Milan Broz43d69032008-02-08 02:11:09 +00009#include <linux/completion.h>
Herbert Xud1806f62006-08-22 20:29:17 +100010#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/module.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/bio.h>
15#include <linux/blkdev.h>
16#include <linux/mempool.h>
17#include <linux/slab.h>
18#include <linux/crypto.h>
19#include <linux/workqueue.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070020#include <linux/backing-dev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/atomic.h>
David Hardeman378f0582005-09-17 17:55:31 +100022#include <linux/scatterlist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <asm/page.h>
Rik Snel48527fa2006-09-03 08:56:39 +100024#include <asm/unaligned.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Mikulas Patocka586e80e2008-10-21 17:44:59 +010026#include <linux/device-mapper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Alasdair G Kergon72d94862006-06-26 00:27:35 -070028#define DM_MSG_PREFIX "crypt"
Milan Broze48d4bb2006-10-03 01:15:37 -070029#define MESG_STR(x) x, sizeof(x)
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070032 * context holding the current state of a multi-part conversion
33 */
34struct convert_context {
Milan Broz43d69032008-02-08 02:11:09 +000035 struct completion restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 struct bio *bio_in;
37 struct bio *bio_out;
38 unsigned int offset_in;
39 unsigned int offset_out;
40 unsigned int idx_in;
41 unsigned int idx_out;
42 sector_t sector;
Milan Broz43d69032008-02-08 02:11:09 +000043 atomic_t pending;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044};
45
Milan Broz53017032008-02-08 02:10:38 +000046/*
47 * per bio private data
48 */
49struct dm_crypt_io {
50 struct dm_target *target;
51 struct bio *base_bio;
52 struct work_struct work;
53
54 struct convert_context ctx;
55
56 atomic_t pending;
57 int error;
Milan Broz0c395b02008-02-08 02:10:54 +000058 sector_t sector;
Milan Broz393b47e2008-10-21 17:45:02 +010059 struct dm_crypt_io *base_io;
Milan Broz53017032008-02-08 02:10:38 +000060};
61
Milan Broz01482b72008-02-08 02:11:04 +000062struct dm_crypt_request {
Huang Yingb2174ee2009-03-16 17:44:33 +000063 struct convert_context *ctx;
Milan Broz01482b72008-02-08 02:11:04 +000064 struct scatterlist sg_in;
65 struct scatterlist sg_out;
66};
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068struct crypt_config;
69
70struct crypt_iv_operations {
71 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +010072 const char *opts);
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 void (*dtr)(struct crypt_config *cc);
74 const char *(*status)(struct crypt_config *cc);
75 int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
76};
77
78/*
79 * Crypt: maps a linear range of a block device
80 * and encrypts / decrypts at the same time.
81 */
Milan Broze48d4bb2006-10-03 01:15:37 -070082enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
Linus Torvalds1da177e2005-04-16 15:20:36 -070083struct crypt_config {
84 struct dm_dev *dev;
85 sector_t start;
86
87 /*
Milan Brozddd42ed2008-02-08 02:11:07 +000088 * pool for per bio private data, crypto requests and
89 * encryption requeusts/buffer pages
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 */
91 mempool_t *io_pool;
Milan Brozddd42ed2008-02-08 02:11:07 +000092 mempool_t *req_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 mempool_t *page_pool;
Milan Broz6a24c712006-10-03 01:15:40 -070094 struct bio_set *bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -070095
Milan Brozcabf08e2007-10-19 22:38:58 +010096 struct workqueue_struct *io_queue;
97 struct workqueue_struct *crypt_queue;
Milan Broz3f1e9072008-03-28 14:16:07 -070098
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 /*
100 * crypto related data
101 */
102 struct crypt_iv_operations *iv_gen_ops;
103 char *iv_mode;
Herbert Xu79066ad2006-12-05 13:41:52 -0800104 union {
105 struct crypto_cipher *essiv_tfm;
106 int benbi_shift;
107 } iv_gen_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 sector_t iv_offset;
109 unsigned int iv_size;
110
Milan Brozddd42ed2008-02-08 02:11:07 +0000111 /*
112 * Layout of each crypto request:
113 *
114 * struct ablkcipher_request
115 * context
116 * padding
117 * struct dm_crypt_request
118 * padding
119 * IV
120 *
121 * The padding is added so that dm_crypt_request and the IV are
122 * correctly aligned.
123 */
124 unsigned int dmreq_start;
125 struct ablkcipher_request *req;
126
Herbert Xud1806f62006-08-22 20:29:17 +1000127 char cipher[CRYPTO_MAX_ALG_NAME];
128 char chainmode[CRYPTO_MAX_ALG_NAME];
Milan Broz3a7f6c92008-02-08 02:11:14 +0000129 struct crypto_ablkcipher *tfm;
Milan Broze48d4bb2006-10-03 01:15:37 -0700130 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 unsigned int key_size;
132 u8 key[0];
133};
134
Milan Broz6a24c712006-10-03 01:15:40 -0700135#define MIN_IOS 16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136#define MIN_POOL_PAGES 32
137#define MIN_BIO_PAGES 8
138
Christoph Lametere18b8902006-12-06 20:33:20 -0800139static struct kmem_cache *_crypt_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100141static void clone_init(struct dm_crypt_io *, struct bio *);
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000142static void kcryptd_queue_crypt(struct dm_crypt_io *io);
Olaf Kirch027581f2007-05-09 02:32:52 -0700143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145 * Different IV generation algorithms:
146 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000147 * plain: the initial vector is the 32-bit little-endian version of the sector
Robert P. J. Day3a4fa0a2007-10-19 23:10:43 +0200148 * number, padded with zeros if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000150 * essiv: "encrypted sector|salt initial vector", the sector number is
151 * encrypted with the bulk cipher using a salt as key. The salt
152 * should be derived from the bulk cipher's key via hashing.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 *
Rik Snel48527fa2006-09-03 08:56:39 +1000154 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
155 * (needed for LRW-32-AES and possible other narrow block modes)
156 *
Ludwig Nussel46b47732007-05-09 02:32:55 -0700157 * null: the initial vector is always zero. Provides compatibility with
158 * obsolete loop_fish2 devices. Do not use for new devices.
159 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 * plumb: unimplemented, see:
161 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
162 */
163
164static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
165{
166 memset(iv, 0, cc->iv_size);
167 *(u32 *)iv = cpu_to_le32(sector & 0xffffffff);
168
169 return 0;
170}
171
172static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +0100173 const char *opts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174{
Herbert Xud1806f62006-08-22 20:29:17 +1000175 struct crypto_cipher *essiv_tfm;
Herbert Xu35058682006-08-24 19:10:20 +1000176 struct crypto_hash *hash_tfm;
177 struct hash_desc desc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 struct scatterlist sg;
179 unsigned int saltsize;
180 u8 *salt;
Herbert Xud1806f62006-08-22 20:29:17 +1000181 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182
183 if (opts == NULL) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700184 ti->error = "Digest algorithm missing for ESSIV mode";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 return -EINVAL;
186 }
187
188 /* Hash the cipher key with the given hash algorithm */
Herbert Xu35058682006-08-24 19:10:20 +1000189 hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
190 if (IS_ERR(hash_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700191 ti->error = "Error initializing ESSIV hash";
Herbert Xu35058682006-08-24 19:10:20 +1000192 return PTR_ERR(hash_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 }
194
Herbert Xu35058682006-08-24 19:10:20 +1000195 saltsize = crypto_hash_digestsize(hash_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 salt = kmalloc(saltsize, GFP_KERNEL);
197 if (salt == NULL) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700198 ti->error = "Error kmallocing salt storage in ESSIV";
Herbert Xu35058682006-08-24 19:10:20 +1000199 crypto_free_hash(hash_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 return -ENOMEM;
201 }
202
Herbert Xu68e3f5d2007-10-27 00:52:07 -0700203 sg_init_one(&sg, cc->key, cc->key_size);
Herbert Xu35058682006-08-24 19:10:20 +1000204 desc.tfm = hash_tfm;
205 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
206 err = crypto_hash_digest(&desc, &sg, cc->key_size, salt);
207 crypto_free_hash(hash_tfm);
208
209 if (err) {
210 ti->error = "Error calculating hash in ESSIV";
Dmitry Monakhov815f9e32007-10-19 22:38:38 +0100211 kfree(salt);
Herbert Xu35058682006-08-24 19:10:20 +1000212 return err;
213 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
215 /* Setup the essiv_tfm with the given salt */
Herbert Xud1806f62006-08-22 20:29:17 +1000216 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
217 if (IS_ERR(essiv_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700218 ti->error = "Error allocating crypto tfm for ESSIV";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 kfree(salt);
Herbert Xud1806f62006-08-22 20:29:17 +1000220 return PTR_ERR(essiv_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 }
Herbert Xud1806f62006-08-22 20:29:17 +1000222 if (crypto_cipher_blocksize(essiv_tfm) !=
Milan Broz3a7f6c92008-02-08 02:11:14 +0000223 crypto_ablkcipher_ivsize(cc->tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700224 ti->error = "Block size of ESSIV cipher does "
Milan Brozd469f842007-10-19 22:42:37 +0100225 "not match IV size of block cipher";
Herbert Xud1806f62006-08-22 20:29:17 +1000226 crypto_free_cipher(essiv_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 kfree(salt);
228 return -EINVAL;
229 }
Herbert Xud1806f62006-08-22 20:29:17 +1000230 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
231 if (err) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700232 ti->error = "Failed to set key for ESSIV cipher";
Herbert Xud1806f62006-08-22 20:29:17 +1000233 crypto_free_cipher(essiv_tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 kfree(salt);
Herbert Xud1806f62006-08-22 20:29:17 +1000235 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 }
237 kfree(salt);
238
Herbert Xu79066ad2006-12-05 13:41:52 -0800239 cc->iv_gen_private.essiv_tfm = essiv_tfm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 return 0;
241}
242
243static void crypt_iv_essiv_dtr(struct crypt_config *cc)
244{
Herbert Xu79066ad2006-12-05 13:41:52 -0800245 crypto_free_cipher(cc->iv_gen_private.essiv_tfm);
246 cc->iv_gen_private.essiv_tfm = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247}
248
249static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
250{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 memset(iv, 0, cc->iv_size);
252 *(u64 *)iv = cpu_to_le64(sector);
Herbert Xu79066ad2006-12-05 13:41:52 -0800253 crypto_cipher_encrypt_one(cc->iv_gen_private.essiv_tfm, iv, iv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 return 0;
255}
256
Rik Snel48527fa2006-09-03 08:56:39 +1000257static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
258 const char *opts)
259{
Milan Broz3a7f6c92008-02-08 02:11:14 +0000260 unsigned bs = crypto_ablkcipher_blocksize(cc->tfm);
David Howellsf0d1b0b2006-12-08 02:37:49 -0800261 int log = ilog2(bs);
Rik Snel48527fa2006-09-03 08:56:39 +1000262
263 /* we need to calculate how far we must shift the sector count
264 * to get the cipher block count, we use this shift in _gen */
265
266 if (1 << log != bs) {
267 ti->error = "cypher blocksize is not a power of 2";
268 return -EINVAL;
269 }
270
271 if (log > 9) {
272 ti->error = "cypher blocksize is > 512";
273 return -EINVAL;
274 }
275
Herbert Xu79066ad2006-12-05 13:41:52 -0800276 cc->iv_gen_private.benbi_shift = 9 - log;
Rik Snel48527fa2006-09-03 08:56:39 +1000277
278 return 0;
279}
280
281static void crypt_iv_benbi_dtr(struct crypt_config *cc)
282{
Rik Snel48527fa2006-09-03 08:56:39 +1000283}
284
285static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
286{
Herbert Xu79066ad2006-12-05 13:41:52 -0800287 __be64 val;
288
Rik Snel48527fa2006-09-03 08:56:39 +1000289 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
Herbert Xu79066ad2006-12-05 13:41:52 -0800290
291 val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi_shift) + 1);
292 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
Rik Snel48527fa2006-09-03 08:56:39 +1000293
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 return 0;
295}
296
Ludwig Nussel46b47732007-05-09 02:32:55 -0700297static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
298{
299 memset(iv, 0, cc->iv_size);
300
301 return 0;
302}
303
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304static struct crypt_iv_operations crypt_iv_plain_ops = {
305 .generator = crypt_iv_plain_gen
306};
307
308static struct crypt_iv_operations crypt_iv_essiv_ops = {
309 .ctr = crypt_iv_essiv_ctr,
310 .dtr = crypt_iv_essiv_dtr,
311 .generator = crypt_iv_essiv_gen
312};
313
Rik Snel48527fa2006-09-03 08:56:39 +1000314static struct crypt_iv_operations crypt_iv_benbi_ops = {
315 .ctr = crypt_iv_benbi_ctr,
316 .dtr = crypt_iv_benbi_dtr,
317 .generator = crypt_iv_benbi_gen
318};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Ludwig Nussel46b47732007-05-09 02:32:55 -0700320static struct crypt_iv_operations crypt_iv_null_ops = {
321 .generator = crypt_iv_null_gen
322};
323
Milan Brozd469f842007-10-19 22:42:37 +0100324static void crypt_convert_init(struct crypt_config *cc,
325 struct convert_context *ctx,
326 struct bio *bio_out, struct bio *bio_in,
Milan Brozfcd369d2008-02-08 02:10:41 +0000327 sector_t sector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328{
329 ctx->bio_in = bio_in;
330 ctx->bio_out = bio_out;
331 ctx->offset_in = 0;
332 ctx->offset_out = 0;
333 ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
334 ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
335 ctx->sector = sector + cc->iv_offset;
Milan Broz43d69032008-02-08 02:11:09 +0000336 init_completion(&ctx->restart);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337}
338
Huang Yingb2174ee2009-03-16 17:44:33 +0000339static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
340 struct ablkcipher_request *req)
341{
342 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
343}
344
345static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc,
346 struct dm_crypt_request *dmreq)
347{
348 return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start);
349}
350
Milan Broz01482b72008-02-08 02:11:04 +0000351static int crypt_convert_block(struct crypt_config *cc,
Milan Broz3a7f6c92008-02-08 02:11:14 +0000352 struct convert_context *ctx,
353 struct ablkcipher_request *req)
Milan Broz01482b72008-02-08 02:11:04 +0000354{
355 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
356 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000357 struct dm_crypt_request *dmreq;
358 u8 *iv;
359 int r = 0;
Milan Broz01482b72008-02-08 02:11:04 +0000360
Huang Yingb2174ee2009-03-16 17:44:33 +0000361 dmreq = dmreq_of_req(cc, req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000362 iv = (u8 *)ALIGN((unsigned long)(dmreq + 1),
363 crypto_ablkcipher_alignmask(cc->tfm) + 1);
364
Huang Yingb2174ee2009-03-16 17:44:33 +0000365 dmreq->ctx = ctx;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000366 sg_init_table(&dmreq->sg_in, 1);
367 sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000368 bv_in->bv_offset + ctx->offset_in);
369
Milan Broz3a7f6c92008-02-08 02:11:14 +0000370 sg_init_table(&dmreq->sg_out, 1);
371 sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000372 bv_out->bv_offset + ctx->offset_out);
373
374 ctx->offset_in += 1 << SECTOR_SHIFT;
375 if (ctx->offset_in >= bv_in->bv_len) {
376 ctx->offset_in = 0;
377 ctx->idx_in++;
378 }
379
380 ctx->offset_out += 1 << SECTOR_SHIFT;
381 if (ctx->offset_out >= bv_out->bv_len) {
382 ctx->offset_out = 0;
383 ctx->idx_out++;
384 }
385
Milan Broz3a7f6c92008-02-08 02:11:14 +0000386 if (cc->iv_gen_ops) {
387 r = cc->iv_gen_ops->generator(cc, iv, ctx->sector);
388 if (r < 0)
389 return r;
390 }
391
392 ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
393 1 << SECTOR_SHIFT, iv);
394
395 if (bio_data_dir(ctx->bio_in) == WRITE)
396 r = crypto_ablkcipher_encrypt(req);
397 else
398 r = crypto_ablkcipher_decrypt(req);
399
400 return r;
Milan Broz01482b72008-02-08 02:11:04 +0000401}
402
Milan Broz95497a92008-02-08 02:11:12 +0000403static void kcryptd_async_done(struct crypto_async_request *async_req,
404 int error);
Milan Brozddd42ed2008-02-08 02:11:07 +0000405static void crypt_alloc_req(struct crypt_config *cc,
406 struct convert_context *ctx)
407{
408 if (!cc->req)
409 cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
Milan Broz95497a92008-02-08 02:11:12 +0000410 ablkcipher_request_set_tfm(cc->req, cc->tfm);
411 ablkcipher_request_set_callback(cc->req, CRYPTO_TFM_REQ_MAY_BACKLOG |
Huang Yingb2174ee2009-03-16 17:44:33 +0000412 CRYPTO_TFM_REQ_MAY_SLEEP,
413 kcryptd_async_done,
414 dmreq_of_req(cc, cc->req));
Milan Brozddd42ed2008-02-08 02:11:07 +0000415}
416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417/*
418 * Encrypt / decrypt data from one bio to another one (can be the same one)
419 */
420static int crypt_convert(struct crypt_config *cc,
Milan Brozd469f842007-10-19 22:42:37 +0100421 struct convert_context *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422{
Milan Broz3f1e9072008-03-28 14:16:07 -0700423 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424
Milan Brozc8081612008-10-10 13:37:08 +0100425 atomic_set(&ctx->pending, 1);
426
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
428 ctx->idx_out < ctx->bio_out->bi_vcnt) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429
Milan Broz3a7f6c92008-02-08 02:11:14 +0000430 crypt_alloc_req(cc, ctx);
431
Milan Broz3f1e9072008-03-28 14:16:07 -0700432 atomic_inc(&ctx->pending);
433
Milan Broz3a7f6c92008-02-08 02:11:14 +0000434 r = crypt_convert_block(cc, ctx, cc->req);
435
436 switch (r) {
Milan Broz3f1e9072008-03-28 14:16:07 -0700437 /* async */
Milan Broz3a7f6c92008-02-08 02:11:14 +0000438 case -EBUSY:
439 wait_for_completion(&ctx->restart);
440 INIT_COMPLETION(ctx->restart);
441 /* fall through*/
442 case -EINPROGRESS:
Milan Broz3a7f6c92008-02-08 02:11:14 +0000443 cc->req = NULL;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000444 ctx->sector++;
445 continue;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000446
Milan Broz3f1e9072008-03-28 14:16:07 -0700447 /* sync */
448 case 0:
449 atomic_dec(&ctx->pending);
450 ctx->sector++;
Milan Brozc7f1b202008-07-02 09:34:28 +0100451 cond_resched();
Milan Broz3f1e9072008-03-28 14:16:07 -0700452 continue;
453
454 /* error */
455 default:
456 atomic_dec(&ctx->pending);
457 return r;
458 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 }
460
Milan Broz3f1e9072008-03-28 14:16:07 -0700461 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462}
463
Milan Brozd469f842007-10-19 22:42:37 +0100464static void dm_crypt_bio_destructor(struct bio *bio)
465{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100466 struct dm_crypt_io *io = bio->bi_private;
Milan Broz6a24c712006-10-03 01:15:40 -0700467 struct crypt_config *cc = io->target->private;
468
469 bio_free(bio, cc->bs);
Milan Brozd469f842007-10-19 22:42:37 +0100470}
Milan Broz6a24c712006-10-03 01:15:40 -0700471
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472/*
473 * Generate a new unfragmented bio with the given size
474 * This should never violate the device limitations
Milan Broz933f01d2008-10-10 13:37:08 +0100475 * May return a smaller bio when running out of pages, indicated by
476 * *out_of_pages set to 1.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 */
Milan Broz933f01d2008-10-10 13:37:08 +0100478static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
479 unsigned *out_of_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480{
Olaf Kirch027581f2007-05-09 02:32:52 -0700481 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -0700482 struct bio *clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Al Virob4e3ca12005-10-21 03:22:34 -0400484 gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
Milan Broz91e10622007-12-13 14:16:10 +0000485 unsigned i, len;
486 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
Olaf Kirch2f9941b2007-05-09 02:32:53 -0700488 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
Milan Broz8b004452006-10-03 01:15:37 -0700489 if (!clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491
Olaf Kirch027581f2007-05-09 02:32:52 -0700492 clone_init(io, clone);
Milan Broz933f01d2008-10-10 13:37:08 +0100493 *out_of_pages = 0;
Milan Broz6a24c712006-10-03 01:15:40 -0700494
Olaf Kirchf97380b2007-05-09 02:32:54 -0700495 for (i = 0; i < nr_iovecs; i++) {
Milan Broz91e10622007-12-13 14:16:10 +0000496 page = mempool_alloc(cc->page_pool, gfp_mask);
Milan Broz933f01d2008-10-10 13:37:08 +0100497 if (!page) {
498 *out_of_pages = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 break;
Milan Broz933f01d2008-10-10 13:37:08 +0100500 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501
502 /*
503 * if additional pages cannot be allocated without waiting,
504 * return a partially allocated bio, the caller will then try
505 * to allocate additional bios while submitting this partial bio
506 */
Olaf Kirchf97380b2007-05-09 02:32:54 -0700507 if (i == (MIN_BIO_PAGES - 1))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
509
Milan Broz91e10622007-12-13 14:16:10 +0000510 len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511
Milan Broz91e10622007-12-13 14:16:10 +0000512 if (!bio_add_page(clone, page, len, 0)) {
513 mempool_free(page, cc->page_pool);
514 break;
515 }
516
517 size -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 }
519
Milan Broz8b004452006-10-03 01:15:37 -0700520 if (!clone->bi_size) {
521 bio_put(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 return NULL;
523 }
524
Milan Broz8b004452006-10-03 01:15:37 -0700525 return clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526}
527
Neil Brown644bd2f2007-10-16 13:48:46 +0200528static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529{
Neil Brown644bd2f2007-10-16 13:48:46 +0200530 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 struct bio_vec *bv;
532
Neil Brown644bd2f2007-10-16 13:48:46 +0200533 for (i = 0; i < clone->bi_vcnt; i++) {
Milan Broz8b004452006-10-03 01:15:37 -0700534 bv = bio_iovec_idx(clone, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 BUG_ON(!bv->bv_page);
536 mempool_free(bv->bv_page, cc->page_pool);
537 bv->bv_page = NULL;
538 }
539}
540
Milan Brozdc440d1e2008-10-10 13:37:03 +0100541static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti,
542 struct bio *bio, sector_t sector)
543{
544 struct crypt_config *cc = ti->private;
545 struct dm_crypt_io *io;
546
547 io = mempool_alloc(cc->io_pool, GFP_NOIO);
548 io->target = ti;
549 io->base_bio = bio;
550 io->sector = sector;
551 io->error = 0;
Milan Broz393b47e2008-10-21 17:45:02 +0100552 io->base_io = NULL;
Milan Brozdc440d1e2008-10-10 13:37:03 +0100553 atomic_set(&io->pending, 0);
554
555 return io;
556}
557
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100558static void crypt_inc_pending(struct dm_crypt_io *io)
559{
560 atomic_inc(&io->pending);
561}
562
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563/*
564 * One of the bios was finished. Check for completion of
565 * the whole request and correctly clean up the buffer.
Milan Broz393b47e2008-10-21 17:45:02 +0100566 * If base_io is set, wait for the last fragment to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 */
Milan Broz5742fd72008-02-08 02:10:43 +0000568static void crypt_dec_pending(struct dm_crypt_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569{
Milan Broz5742fd72008-02-08 02:10:43 +0000570 struct crypt_config *cc = io->target->private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000571 struct bio *base_bio = io->base_bio;
572 struct dm_crypt_io *base_io = io->base_io;
573 int error = io->error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574
575 if (!atomic_dec_and_test(&io->pending))
576 return;
577
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 mempool_free(io, cc->io_pool);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000579
580 if (likely(!base_io))
581 bio_endio(base_bio, error);
582 else {
583 if (error && !base_io->error)
584 base_io->error = error;
585 crypt_dec_pending(base_io);
586 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587}
588
589/*
Milan Brozcabf08e2007-10-19 22:38:58 +0100590 * kcryptd/kcryptd_io:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 *
592 * Needed because it would be very unwise to do decryption in an
Milan Broz23541d22006-10-03 01:15:39 -0700593 * interrupt context.
Milan Brozcabf08e2007-10-19 22:38:58 +0100594 *
595 * kcryptd performs the actual encryption or decryption.
596 *
597 * kcryptd_io performs the IO submission.
598 *
599 * They must be separated as otherwise the final stages could be
600 * starved by new requests which can block in the first stages due
601 * to memory allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 */
NeilBrown6712ecf2007-09-27 12:47:43 +0200603static void crypt_endio(struct bio *clone, int error)
Milan Broz8b004452006-10-03 01:15:37 -0700604{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100605 struct dm_crypt_io *io = clone->bi_private;
Milan Broz8b004452006-10-03 01:15:37 -0700606 struct crypt_config *cc = io->target->private;
Milan Brozee7a4912008-02-08 02:10:46 +0000607 unsigned rw = bio_data_dir(clone);
Milan Broz8b004452006-10-03 01:15:37 -0700608
Milan Brozadfe4772007-12-13 14:15:51 +0000609 if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error))
610 error = -EIO;
611
Milan Broz8b004452006-10-03 01:15:37 -0700612 /*
NeilBrown6712ecf2007-09-27 12:47:43 +0200613 * free the processed pages
Milan Broz8b004452006-10-03 01:15:37 -0700614 */
Milan Brozee7a4912008-02-08 02:10:46 +0000615 if (rw == WRITE)
Neil Brown644bd2f2007-10-16 13:48:46 +0200616 crypt_free_buffer_pages(cc, clone);
Milan Brozee7a4912008-02-08 02:10:46 +0000617
618 bio_put(clone);
619
620 if (rw == READ && !error) {
621 kcryptd_queue_crypt(io);
622 return;
NeilBrown6712ecf2007-09-27 12:47:43 +0200623 }
Milan Broz8b004452006-10-03 01:15:37 -0700624
Milan Brozadfe4772007-12-13 14:15:51 +0000625 if (unlikely(error))
Milan Broz5742fd72008-02-08 02:10:43 +0000626 io->error = error;
627
628 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700629}
630
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100631static void clone_init(struct dm_crypt_io *io, struct bio *clone)
Milan Broz8b004452006-10-03 01:15:37 -0700632{
633 struct crypt_config *cc = io->target->private;
634
635 clone->bi_private = io;
636 clone->bi_end_io = crypt_endio;
637 clone->bi_bdev = cc->dev->bdev;
638 clone->bi_rw = io->base_bio->bi_rw;
Olaf Kirch027581f2007-05-09 02:32:52 -0700639 clone->bi_destructor = dm_crypt_bio_destructor;
Milan Broz8b004452006-10-03 01:15:37 -0700640}
641
Milan Broz4e4eef62008-02-08 02:10:49 +0000642static void kcryptd_io_read(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700643{
644 struct crypt_config *cc = io->target->private;
645 struct bio *base_bio = io->base_bio;
646 struct bio *clone;
Milan Broz93e605c2006-10-03 01:15:38 -0700647
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100648 crypt_inc_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700649
650 /*
651 * The block layer might modify the bvec array, so always
652 * copy the required bvecs because we need the original
653 * one in order to decrypt the whole bio data *afterwards*.
654 */
Milan Broz6a24c712006-10-03 01:15:40 -0700655 clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs);
Milan Broz93e605c2006-10-03 01:15:38 -0700656 if (unlikely(!clone)) {
Milan Broz5742fd72008-02-08 02:10:43 +0000657 io->error = -ENOMEM;
658 crypt_dec_pending(io);
Milan Broz23541d22006-10-03 01:15:39 -0700659 return;
Milan Broz93e605c2006-10-03 01:15:38 -0700660 }
Milan Broz8b004452006-10-03 01:15:37 -0700661
662 clone_init(io, clone);
663 clone->bi_idx = 0;
664 clone->bi_vcnt = bio_segments(base_bio);
665 clone->bi_size = base_bio->bi_size;
Milan Broz0c395b02008-02-08 02:10:54 +0000666 clone->bi_sector = cc->start + io->sector;
Milan Broz8b004452006-10-03 01:15:37 -0700667 memcpy(clone->bi_io_vec, bio_iovec(base_bio),
668 sizeof(struct bio_vec) * clone->bi_vcnt);
Milan Broz8b004452006-10-03 01:15:37 -0700669
Milan Broz93e605c2006-10-03 01:15:38 -0700670 generic_make_request(clone);
Milan Broz8b004452006-10-03 01:15:37 -0700671}
672
Milan Broz4e4eef62008-02-08 02:10:49 +0000673static void kcryptd_io_write(struct dm_crypt_io *io)
674{
Milan Broz95497a92008-02-08 02:11:12 +0000675 struct bio *clone = io->ctx.bio_out;
Milan Broz95497a92008-02-08 02:11:12 +0000676 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +0000677}
678
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000679static void kcryptd_io(struct work_struct *work)
680{
681 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
682
683 if (bio_data_dir(io->base_bio) == READ)
684 kcryptd_io_read(io);
685 else
686 kcryptd_io_write(io);
687}
688
689static void kcryptd_queue_io(struct dm_crypt_io *io)
690{
691 struct crypt_config *cc = io->target->private;
692
693 INIT_WORK(&io->work, kcryptd_io);
694 queue_work(cc->io_queue, &io->work);
695}
696
Milan Broz95497a92008-02-08 02:11:12 +0000697static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io,
698 int error, int async)
Milan Broz4e4eef62008-02-08 02:10:49 +0000699{
Milan Brozdec1ced2008-02-08 02:10:57 +0000700 struct bio *clone = io->ctx.bio_out;
701 struct crypt_config *cc = io->target->private;
702
703 if (unlikely(error < 0)) {
704 crypt_free_buffer_pages(cc, clone);
705 bio_put(clone);
706 io->error = -EIO;
Milan Broz6c031f42008-10-10 13:37:06 +0100707 crypt_dec_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +0000708 return;
709 }
710
711 /* crypt_convert should have filled the clone bio */
712 BUG_ON(io->ctx.idx_out < clone->bi_vcnt);
713
714 clone->bi_sector = cc->start + io->sector;
Milan Broz899c95d2008-02-08 02:11:02 +0000715
Milan Broz95497a92008-02-08 02:11:12 +0000716 if (async)
717 kcryptd_queue_io(io);
Alasdair G Kergon1e37bb82008-10-10 13:37:05 +0100718 else
Milan Broz95497a92008-02-08 02:11:12 +0000719 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +0000720}
721
Milan Brozfc5a5e92008-10-10 13:37:04 +0100722static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700723{
724 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -0700725 struct bio *clone;
Milan Broz393b47e2008-10-21 17:45:02 +0100726 struct dm_crypt_io *new_io;
Milan Brozc8081612008-10-10 13:37:08 +0100727 int crypt_finished;
Milan Broz933f01d2008-10-10 13:37:08 +0100728 unsigned out_of_pages = 0;
Milan Brozdec1ced2008-02-08 02:10:57 +0000729 unsigned remaining = io->base_bio->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +0100730 sector_t sector = io->sector;
Milan Brozdec1ced2008-02-08 02:10:57 +0000731 int r;
Milan Broz8b004452006-10-03 01:15:37 -0700732
Milan Broz93e605c2006-10-03 01:15:38 -0700733 /*
Milan Brozfc5a5e92008-10-10 13:37:04 +0100734 * Prevent io from disappearing until this function completes.
735 */
736 crypt_inc_pending(io);
Milan Brozb635b002008-10-21 17:45:00 +0100737 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
Milan Brozfc5a5e92008-10-10 13:37:04 +0100738
739 /*
Milan Broz93e605c2006-10-03 01:15:38 -0700740 * The allocated buffers can be smaller than the whole bio,
741 * so repeat the whole process until all the data can be handled.
742 */
743 while (remaining) {
Milan Broz933f01d2008-10-10 13:37:08 +0100744 clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
Milan Broz23541d22006-10-03 01:15:39 -0700745 if (unlikely(!clone)) {
Milan Broz5742fd72008-02-08 02:10:43 +0000746 io->error = -ENOMEM;
Milan Brozfc5a5e92008-10-10 13:37:04 +0100747 break;
Milan Broz23541d22006-10-03 01:15:39 -0700748 }
Milan Broz93e605c2006-10-03 01:15:38 -0700749
Milan Broz53017032008-02-08 02:10:38 +0000750 io->ctx.bio_out = clone;
751 io->ctx.idx_out = 0;
Milan Broz93e605c2006-10-03 01:15:38 -0700752
Milan Broz93e605c2006-10-03 01:15:38 -0700753 remaining -= clone->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +0100754 sector += bio_sectors(clone);
Milan Brozdec1ced2008-02-08 02:10:57 +0000755
Milan Broz4e594092008-10-10 13:37:07 +0100756 crypt_inc_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +0000757 r = crypt_convert(cc, &io->ctx);
Milan Brozc8081612008-10-10 13:37:08 +0100758 crypt_finished = atomic_dec_and_test(&io->ctx.pending);
Milan Brozdec1ced2008-02-08 02:10:57 +0000759
Milan Brozc8081612008-10-10 13:37:08 +0100760 /* Encryption was already finished, submit io now */
761 if (crypt_finished) {
Milan Broz3a7f6c92008-02-08 02:11:14 +0000762 kcryptd_crypt_write_io_submit(io, r, 0);
Milan Brozc8081612008-10-10 13:37:08 +0100763
764 /*
765 * If there was an error, do not try next fragments.
766 * For async, error is processed in async handler.
767 */
Milan Broz6c031f42008-10-10 13:37:06 +0100768 if (unlikely(r < 0))
Milan Brozfc5a5e92008-10-10 13:37:04 +0100769 break;
Milan Brozb635b002008-10-21 17:45:00 +0100770
771 io->sector = sector;
Milan Broz4e594092008-10-10 13:37:07 +0100772 }
Milan Broz93e605c2006-10-03 01:15:38 -0700773
Milan Broz933f01d2008-10-10 13:37:08 +0100774 /*
775 * Out of memory -> run queues
776 * But don't wait if split was due to the io size restriction
777 */
778 if (unlikely(out_of_pages))
Jens Axboe8aa7e842009-07-09 14:52:32 +0200779 congestion_wait(BLK_RW_ASYNC, HZ/100);
Milan Broz933f01d2008-10-10 13:37:08 +0100780
Milan Broz393b47e2008-10-21 17:45:02 +0100781 /*
782 * With async crypto it is unsafe to share the crypto context
783 * between fragments, so switch to a new dm_crypt_io structure.
784 */
785 if (unlikely(!crypt_finished && remaining)) {
786 new_io = crypt_io_alloc(io->target, io->base_bio,
787 sector);
788 crypt_inc_pending(new_io);
789 crypt_convert_init(cc, &new_io->ctx, NULL,
790 io->base_bio, sector);
791 new_io->ctx.idx_in = io->ctx.idx_in;
792 new_io->ctx.offset_in = io->ctx.offset_in;
793
794 /*
795 * Fragments after the first use the base_io
796 * pending count.
797 */
798 if (!io->base_io)
799 new_io->base_io = io;
800 else {
801 new_io->base_io = io->base_io;
802 crypt_inc_pending(io->base_io);
803 crypt_dec_pending(io);
804 }
805
806 io = new_io;
807 }
Milan Broz8b004452006-10-03 01:15:37 -0700808 }
Milan Broz899c95d2008-02-08 02:11:02 +0000809
810 crypt_dec_pending(io);
Milan Broz84131db2008-02-08 02:10:59 +0000811}
812
Milan Broz4e4eef62008-02-08 02:10:49 +0000813static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error)
Milan Broz5742fd72008-02-08 02:10:43 +0000814{
815 if (unlikely(error < 0))
816 io->error = -EIO;
817
818 crypt_dec_pending(io);
819}
820
Milan Broz4e4eef62008-02-08 02:10:49 +0000821static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700822{
823 struct crypt_config *cc = io->target->private;
Milan Broz5742fd72008-02-08 02:10:43 +0000824 int r = 0;
Milan Broz8b004452006-10-03 01:15:37 -0700825
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100826 crypt_inc_pending(io);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000827
Milan Broz53017032008-02-08 02:10:38 +0000828 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
Milan Broz0c395b02008-02-08 02:10:54 +0000829 io->sector);
Milan Broz8b004452006-10-03 01:15:37 -0700830
Milan Broz5742fd72008-02-08 02:10:43 +0000831 r = crypt_convert(cc, &io->ctx);
832
Milan Broz3f1e9072008-03-28 14:16:07 -0700833 if (atomic_dec_and_test(&io->ctx.pending))
Milan Broz3a7f6c92008-02-08 02:11:14 +0000834 kcryptd_crypt_read_done(io, r);
835
836 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700837}
838
Milan Broz95497a92008-02-08 02:11:12 +0000839static void kcryptd_async_done(struct crypto_async_request *async_req,
840 int error)
841{
Huang Yingb2174ee2009-03-16 17:44:33 +0000842 struct dm_crypt_request *dmreq = async_req->data;
843 struct convert_context *ctx = dmreq->ctx;
Milan Broz95497a92008-02-08 02:11:12 +0000844 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
845 struct crypt_config *cc = io->target->private;
846
847 if (error == -EINPROGRESS) {
848 complete(&ctx->restart);
849 return;
850 }
851
Huang Yingb2174ee2009-03-16 17:44:33 +0000852 mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool);
Milan Broz95497a92008-02-08 02:11:12 +0000853
854 if (!atomic_dec_and_test(&ctx->pending))
855 return;
856
857 if (bio_data_dir(io->base_bio) == READ)
858 kcryptd_crypt_read_done(io, error);
859 else
860 kcryptd_crypt_write_io_submit(io, error, 1);
861}
862
Milan Broz4e4eef62008-02-08 02:10:49 +0000863static void kcryptd_crypt(struct work_struct *work)
864{
865 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
866
867 if (bio_data_dir(io->base_bio) == READ)
868 kcryptd_crypt_read_convert(io);
869 else
870 kcryptd_crypt_write_convert(io);
Milan Broz8b004452006-10-03 01:15:37 -0700871}
872
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000873static void kcryptd_queue_crypt(struct dm_crypt_io *io)
874{
875 struct crypt_config *cc = io->target->private;
876
877 INIT_WORK(&io->work, kcryptd_crypt);
878 queue_work(cc->crypt_queue, &io->work);
879}
880
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881/*
882 * Decode key from its hex representation
883 */
884static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
885{
886 char buffer[3];
887 char *endp;
888 unsigned int i;
889
890 buffer[2] = '\0';
891
Milan Broz8b004452006-10-03 01:15:37 -0700892 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 buffer[0] = *hex++;
894 buffer[1] = *hex++;
895
896 key[i] = (u8)simple_strtoul(buffer, &endp, 16);
897
898 if (endp != &buffer[2])
899 return -EINVAL;
900 }
901
902 if (*hex != '\0')
903 return -EINVAL;
904
905 return 0;
906}
907
908/*
909 * Encode key into its hex representation
910 */
911static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
912{
913 unsigned int i;
914
Milan Broz8b004452006-10-03 01:15:37 -0700915 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 sprintf(hex, "%02x", *key);
917 hex += 2;
918 key++;
919 }
920}
921
Milan Broze48d4bb2006-10-03 01:15:37 -0700922static int crypt_set_key(struct crypt_config *cc, char *key)
923{
924 unsigned key_size = strlen(key) >> 1;
925
926 if (cc->key_size && cc->key_size != key_size)
927 return -EINVAL;
928
929 cc->key_size = key_size; /* initial settings */
930
931 if ((!key_size && strcmp(key, "-")) ||
Milan Brozd469f842007-10-19 22:42:37 +0100932 (key_size && crypt_decode_key(cc->key, key, key_size) < 0))
Milan Broze48d4bb2006-10-03 01:15:37 -0700933 return -EINVAL;
934
935 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
936
937 return 0;
938}
939
940static int crypt_wipe_key(struct crypt_config *cc)
941{
942 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
943 memset(&cc->key, 0, cc->key_size * sizeof(u8));
944 return 0;
945}
946
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947/*
948 * Construct an encryption mapping:
949 * <cipher> <key> <iv_offset> <dev_path> <start>
950 */
951static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
952{
953 struct crypt_config *cc;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000954 struct crypto_ablkcipher *tfm;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 char *tmp;
956 char *cipher;
957 char *chainmode;
958 char *ivmode;
959 char *ivopts;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 unsigned int key_size;
Andrew Morton4ee218c2006-03-27 01:17:48 -0800961 unsigned long long tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
963 if (argc != 5) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700964 ti->error = "Not enough arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 return -EINVAL;
966 }
967
968 tmp = argv[0];
969 cipher = strsep(&tmp, "-");
970 chainmode = strsep(&tmp, "-");
971 ivopts = strsep(&tmp, "-");
972 ivmode = strsep(&ivopts, ":");
973
974 if (tmp)
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700975 DMWARN("Unexpected additional cipher options");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976
977 key_size = strlen(argv[1]) >> 1;
978
Milan Broze48d4bb2006-10-03 01:15:37 -0700979 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 if (cc == NULL) {
981 ti->error =
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700982 "Cannot allocate transparent encryption context";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 return -ENOMEM;
984 }
985
Milan Broze48d4bb2006-10-03 01:15:37 -0700986 if (crypt_set_key(cc, argv[1])) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700987 ti->error = "Error decoding key";
Milan Broz636d5782007-10-19 22:47:52 +0100988 goto bad_cipher;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 }
990
991 /* Compatiblity mode for old dm-crypt cipher strings */
992 if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) {
993 chainmode = "cbc";
994 ivmode = "plain";
995 }
996
Herbert Xud1806f62006-08-22 20:29:17 +1000997 if (strcmp(chainmode, "ecb") && !ivmode) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700998 ti->error = "This chaining mode requires an IV mechanism";
Milan Broz636d5782007-10-19 22:47:52 +0100999 goto bad_cipher;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 }
1001
Milan Brozd469f842007-10-19 22:42:37 +01001002 if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)",
1003 chainmode, cipher) >= CRYPTO_MAX_ALG_NAME) {
Herbert Xud1806f62006-08-22 20:29:17 +10001004 ti->error = "Chain mode + cipher name is too long";
Milan Broz636d5782007-10-19 22:47:52 +01001005 goto bad_cipher;
Herbert Xud1806f62006-08-22 20:29:17 +10001006 }
1007
Milan Broz3a7f6c92008-02-08 02:11:14 +00001008 tfm = crypto_alloc_ablkcipher(cc->cipher, 0, 0);
Herbert Xud1806f62006-08-22 20:29:17 +10001009 if (IS_ERR(tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001010 ti->error = "Error allocating crypto tfm";
Milan Broz636d5782007-10-19 22:47:52 +01001011 goto bad_cipher;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013
Herbert Xud1806f62006-08-22 20:29:17 +10001014 strcpy(cc->cipher, cipher);
1015 strcpy(cc->chainmode, chainmode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 cc->tfm = tfm;
1017
1018 /*
Rik Snel48527fa2006-09-03 08:56:39 +10001019 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi".
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 * See comments at iv code
1021 */
1022
1023 if (ivmode == NULL)
1024 cc->iv_gen_ops = NULL;
1025 else if (strcmp(ivmode, "plain") == 0)
1026 cc->iv_gen_ops = &crypt_iv_plain_ops;
1027 else if (strcmp(ivmode, "essiv") == 0)
1028 cc->iv_gen_ops = &crypt_iv_essiv_ops;
Rik Snel48527fa2006-09-03 08:56:39 +10001029 else if (strcmp(ivmode, "benbi") == 0)
1030 cc->iv_gen_ops = &crypt_iv_benbi_ops;
Ludwig Nussel46b47732007-05-09 02:32:55 -07001031 else if (strcmp(ivmode, "null") == 0)
1032 cc->iv_gen_ops = &crypt_iv_null_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 else {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001034 ti->error = "Invalid IV mode";
Milan Broz636d5782007-10-19 22:47:52 +01001035 goto bad_ivmode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 }
1037
1038 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
1039 cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
Milan Broz636d5782007-10-19 22:47:52 +01001040 goto bad_ivmode;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041
Milan Broz3a7f6c92008-02-08 02:11:14 +00001042 cc->iv_size = crypto_ablkcipher_ivsize(tfm);
Herbert Xud1806f62006-08-22 20:29:17 +10001043 if (cc->iv_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 /* at least a 64 bit sector number should fit in our buffer */
Herbert Xud1806f62006-08-22 20:29:17 +10001045 cc->iv_size = max(cc->iv_size,
Milan Brozd469f842007-10-19 22:42:37 +01001046 (unsigned int)(sizeof(u64) / sizeof(u8)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 if (cc->iv_gen_ops) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001049 DMWARN("Selected cipher does not support IVs");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050 if (cc->iv_gen_ops->dtr)
1051 cc->iv_gen_ops->dtr(cc);
1052 cc->iv_gen_ops = NULL;
1053 }
1054 }
1055
Matthew Dobson93d23412006-03-26 01:37:50 -08001056 cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 if (!cc->io_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001058 ti->error = "Cannot allocate crypt io mempool";
Milan Broz636d5782007-10-19 22:47:52 +01001059 goto bad_slab_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 }
1061
Milan Brozddd42ed2008-02-08 02:11:07 +00001062 cc->dmreq_start = sizeof(struct ablkcipher_request);
Milan Broz3a7f6c92008-02-08 02:11:14 +00001063 cc->dmreq_start += crypto_ablkcipher_reqsize(tfm);
Milan Brozddd42ed2008-02-08 02:11:07 +00001064 cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment());
Milan Broz3a7f6c92008-02-08 02:11:14 +00001065 cc->dmreq_start += crypto_ablkcipher_alignmask(tfm) &
1066 ~(crypto_tfm_ctx_alignment() - 1);
Milan Brozddd42ed2008-02-08 02:11:07 +00001067
1068 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
1069 sizeof(struct dm_crypt_request) + cc->iv_size);
1070 if (!cc->req_pool) {
1071 ti->error = "Cannot allocate crypt request mempool";
1072 goto bad_req_pool;
1073 }
1074 cc->req = NULL;
1075
Matthew Dobsona19b27c2006-03-26 01:37:45 -08001076 cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 if (!cc->page_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001078 ti->error = "Cannot allocate page mempool";
Milan Broz636d5782007-10-19 22:47:52 +01001079 goto bad_page_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 }
1081
Jens Axboebb799ca2008-12-10 15:35:05 +01001082 cc->bs = bioset_create(MIN_IOS, 0);
Milan Broz6a24c712006-10-03 01:15:40 -07001083 if (!cc->bs) {
1084 ti->error = "Cannot allocate crypt bioset";
1085 goto bad_bs;
1086 }
1087
Milan Broz3a7f6c92008-02-08 02:11:14 +00001088 if (crypto_ablkcipher_setkey(tfm, cc->key, key_size) < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001089 ti->error = "Error setting key";
Milan Broz636d5782007-10-19 22:47:52 +01001090 goto bad_device;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091 }
1092
Andrew Morton4ee218c2006-03-27 01:17:48 -08001093 if (sscanf(argv[2], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001094 ti->error = "Invalid iv_offset sector";
Milan Broz636d5782007-10-19 22:47:52 +01001095 goto bad_device;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001097 cc->iv_offset = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098
Andrew Morton4ee218c2006-03-27 01:17:48 -08001099 if (sscanf(argv[4], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001100 ti->error = "Invalid device sector";
Milan Broz636d5782007-10-19 22:47:52 +01001101 goto bad_device;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001103 cc->start = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
1105 if (dm_get_device(ti, argv[3], cc->start, ti->len,
Milan Brozd469f842007-10-19 22:42:37 +01001106 dm_table_get_mode(ti->table), &cc->dev)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001107 ti->error = "Device lookup failed";
Milan Broz636d5782007-10-19 22:47:52 +01001108 goto bad_device;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 }
1110
1111 if (ivmode && cc->iv_gen_ops) {
1112 if (ivopts)
1113 *(ivopts - 1) = ':';
1114 cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
1115 if (!cc->iv_mode) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001116 ti->error = "Error kmallocing iv_mode string";
Milan Broz636d5782007-10-19 22:47:52 +01001117 goto bad_ivmode_string;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 }
1119 strcpy(cc->iv_mode, ivmode);
1120 } else
1121 cc->iv_mode = NULL;
1122
Milan Brozcabf08e2007-10-19 22:38:58 +01001123 cc->io_queue = create_singlethread_workqueue("kcryptd_io");
1124 if (!cc->io_queue) {
1125 ti->error = "Couldn't create kcryptd io queue";
1126 goto bad_io_queue;
1127 }
1128
1129 cc->crypt_queue = create_singlethread_workqueue("kcryptd");
1130 if (!cc->crypt_queue) {
Milan Broz9934a8b2007-10-19 22:38:57 +01001131 ti->error = "Couldn't create kcryptd queue";
Milan Brozcabf08e2007-10-19 22:38:58 +01001132 goto bad_crypt_queue;
Milan Broz9934a8b2007-10-19 22:38:57 +01001133 }
1134
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001135 ti->num_flush_requests = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 ti->private = cc;
1137 return 0;
1138
Milan Brozcabf08e2007-10-19 22:38:58 +01001139bad_crypt_queue:
1140 destroy_workqueue(cc->io_queue);
1141bad_io_queue:
Milan Broz9934a8b2007-10-19 22:38:57 +01001142 kfree(cc->iv_mode);
Milan Broz636d5782007-10-19 22:47:52 +01001143bad_ivmode_string:
Dmitry Monakhov55b42c52007-10-19 22:38:37 +01001144 dm_put_device(ti, cc->dev);
Milan Broz636d5782007-10-19 22:47:52 +01001145bad_device:
Milan Broz6a24c712006-10-03 01:15:40 -07001146 bioset_free(cc->bs);
1147bad_bs:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 mempool_destroy(cc->page_pool);
Milan Broz636d5782007-10-19 22:47:52 +01001149bad_page_pool:
Milan Brozddd42ed2008-02-08 02:11:07 +00001150 mempool_destroy(cc->req_pool);
1151bad_req_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 mempool_destroy(cc->io_pool);
Milan Broz636d5782007-10-19 22:47:52 +01001153bad_slab_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
1155 cc->iv_gen_ops->dtr(cc);
Milan Broz636d5782007-10-19 22:47:52 +01001156bad_ivmode:
Milan Broz3a7f6c92008-02-08 02:11:14 +00001157 crypto_free_ablkcipher(tfm);
Milan Broz636d5782007-10-19 22:47:52 +01001158bad_cipher:
Stefan Rompf9d3520a2006-01-06 00:20:08 -08001159 /* Must zero key material before freeing */
Johannes Weinerb81d6cf2009-04-02 19:55:28 +01001160 kzfree(cc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 return -EINVAL;
1162}
1163
1164static void crypt_dtr(struct dm_target *ti)
1165{
1166 struct crypt_config *cc = (struct crypt_config *) ti->private;
1167
Milan Brozcabf08e2007-10-19 22:38:58 +01001168 destroy_workqueue(cc->io_queue);
1169 destroy_workqueue(cc->crypt_queue);
Milan Broz80b16c12007-07-21 04:37:27 -07001170
Milan Brozddd42ed2008-02-08 02:11:07 +00001171 if (cc->req)
1172 mempool_free(cc->req, cc->req_pool);
1173
Milan Broz6a24c712006-10-03 01:15:40 -07001174 bioset_free(cc->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 mempool_destroy(cc->page_pool);
Milan Brozddd42ed2008-02-08 02:11:07 +00001176 mempool_destroy(cc->req_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 mempool_destroy(cc->io_pool);
1178
Jesper Juhl990a8ba2005-06-21 17:17:30 -07001179 kfree(cc->iv_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
1181 cc->iv_gen_ops->dtr(cc);
Milan Broz3a7f6c92008-02-08 02:11:14 +00001182 crypto_free_ablkcipher(cc->tfm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183 dm_put_device(ti, cc->dev);
Stefan Rompf9d3520a2006-01-06 00:20:08 -08001184
1185 /* Must zero key material before freeing */
Johannes Weinerb81d6cf2009-04-02 19:55:28 +01001186 kzfree(cc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001187}
1188
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189static int crypt_map(struct dm_target *ti, struct bio *bio,
1190 union map_info *map_context)
1191{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001192 struct dm_crypt_io *io;
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001193 struct crypt_config *cc;
1194
1195 if (unlikely(bio_empty_barrier(bio))) {
1196 cc = ti->private;
1197 bio->bi_bdev = cc->dev->bdev;
1198 return DM_MAPIO_REMAPPED;
1199 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200
Milan Brozdc440d1e2008-10-10 13:37:03 +01001201 io = crypt_io_alloc(ti, bio, bio->bi_sector - ti->begin);
Milan Brozcabf08e2007-10-19 22:38:58 +01001202
1203 if (bio_data_dir(io->base_bio) == READ)
1204 kcryptd_queue_io(io);
1205 else
1206 kcryptd_queue_crypt(io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001208 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209}
1210
1211static int crypt_status(struct dm_target *ti, status_type_t type,
1212 char *result, unsigned int maxlen)
1213{
1214 struct crypt_config *cc = (struct crypt_config *) ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 unsigned int sz = 0;
1216
1217 switch (type) {
1218 case STATUSTYPE_INFO:
1219 result[0] = '\0';
1220 break;
1221
1222 case STATUSTYPE_TABLE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 if (cc->iv_mode)
Christophe Saout37af6562006-10-30 20:39:08 +01001224 DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode,
1225 cc->iv_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 else
Christophe Saout37af6562006-10-30 20:39:08 +01001227 DMEMIT("%s-%s ", cc->cipher, cc->chainmode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
1229 if (cc->key_size > 0) {
1230 if ((maxlen - sz) < ((cc->key_size << 1) + 1))
1231 return -ENOMEM;
1232
1233 crypt_encode_key(result + sz, cc->key, cc->key_size);
1234 sz += cc->key_size << 1;
1235 } else {
1236 if (sz >= maxlen)
1237 return -ENOMEM;
1238 result[sz++] = '-';
1239 }
1240
Andrew Morton4ee218c2006-03-27 01:17:48 -08001241 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
1242 cc->dev->name, (unsigned long long)cc->start);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 break;
1244 }
1245 return 0;
1246}
1247
Milan Broze48d4bb2006-10-03 01:15:37 -07001248static void crypt_postsuspend(struct dm_target *ti)
1249{
1250 struct crypt_config *cc = ti->private;
1251
1252 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1253}
1254
1255static int crypt_preresume(struct dm_target *ti)
1256{
1257 struct crypt_config *cc = ti->private;
1258
1259 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
1260 DMERR("aborting resume - crypt key is not set.");
1261 return -EAGAIN;
1262 }
1263
1264 return 0;
1265}
1266
1267static void crypt_resume(struct dm_target *ti)
1268{
1269 struct crypt_config *cc = ti->private;
1270
1271 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1272}
1273
1274/* Message interface
1275 * key set <key>
1276 * key wipe
1277 */
1278static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
1279{
1280 struct crypt_config *cc = ti->private;
1281
1282 if (argc < 2)
1283 goto error;
1284
1285 if (!strnicmp(argv[0], MESG_STR("key"))) {
1286 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
1287 DMWARN("not suspended during key manipulation.");
1288 return -EINVAL;
1289 }
1290 if (argc == 3 && !strnicmp(argv[1], MESG_STR("set")))
1291 return crypt_set_key(cc, argv[2]);
1292 if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe")))
1293 return crypt_wipe_key(cc);
1294 }
1295
1296error:
1297 DMWARN("unrecognised message received.");
1298 return -EINVAL;
1299}
1300
Milan Brozd41e26b2008-07-21 12:00:40 +01001301static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
1302 struct bio_vec *biovec, int max_size)
1303{
1304 struct crypt_config *cc = ti->private;
1305 struct request_queue *q = bdev_get_queue(cc->dev->bdev);
1306
1307 if (!q->merge_bvec_fn)
1308 return max_size;
1309
1310 bvm->bi_bdev = cc->dev->bdev;
1311 bvm->bi_sector = cc->start + bvm->bi_sector - ti->begin;
1312
1313 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
1314}
1315
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001316static int crypt_iterate_devices(struct dm_target *ti,
1317 iterate_devices_callout_fn fn, void *data)
1318{
1319 struct crypt_config *cc = ti->private;
1320
Mike Snitzer5dea2712009-07-23 20:30:42 +01001321 return fn(ti, cc->dev, cc->start, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001322}
1323
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324static struct target_type crypt_target = {
1325 .name = "crypt",
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001326 .version = {1, 7, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327 .module = THIS_MODULE,
1328 .ctr = crypt_ctr,
1329 .dtr = crypt_dtr,
1330 .map = crypt_map,
1331 .status = crypt_status,
Milan Broze48d4bb2006-10-03 01:15:37 -07001332 .postsuspend = crypt_postsuspend,
1333 .preresume = crypt_preresume,
1334 .resume = crypt_resume,
1335 .message = crypt_message,
Milan Brozd41e26b2008-07-21 12:00:40 +01001336 .merge = crypt_merge,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001337 .iterate_devices = crypt_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338};
1339
1340static int __init dm_crypt_init(void)
1341{
1342 int r;
1343
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001344 _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 if (!_crypt_io_pool)
1346 return -ENOMEM;
1347
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 r = dm_register_target(&crypt_target);
1349 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001350 DMERR("register failed %d", r);
Milan Broz9934a8b2007-10-19 22:38:57 +01001351 kmem_cache_destroy(_crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001352 }
1353
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 return r;
1355}
1356
1357static void __exit dm_crypt_exit(void)
1358{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001359 dm_unregister_target(&crypt_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 kmem_cache_destroy(_crypt_io_pool);
1361}
1362
1363module_init(dm_crypt_init);
1364module_exit(dm_crypt_exit);
1365
1366MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1367MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
1368MODULE_LICENSE("GPL");