blob: 368e8e98f7050e0fa5ddd7fa2220379681fdb04d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
Milan Broz542da312009-12-10 23:51:57 +00004 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * This file is released under the GPL.
7 */
8
Milan Broz43d69032008-02-08 02:11:09 +00009#include <linux/completion.h>
Herbert Xud1806f62006-08-22 20:29:17 +100010#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/module.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/bio.h>
15#include <linux/blkdev.h>
16#include <linux/mempool.h>
17#include <linux/slab.h>
18#include <linux/crypto.h>
19#include <linux/workqueue.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070020#include <linux/backing-dev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/atomic.h>
David Hardeman378f0582005-09-17 17:55:31 +100022#include <linux/scatterlist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <asm/page.h>
Rik Snel48527fa2006-09-03 08:56:39 +100024#include <asm/unaligned.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Mikulas Patocka586e80e2008-10-21 17:44:59 +010026#include <linux/device-mapper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Alasdair G Kergon72d94862006-06-26 00:27:35 -070028#define DM_MSG_PREFIX "crypt"
Milan Broze48d4bb2006-10-03 01:15:37 -070029#define MESG_STR(x) x, sizeof(x)
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070032 * context holding the current state of a multi-part conversion
33 */
34struct convert_context {
Milan Broz43d69032008-02-08 02:11:09 +000035 struct completion restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 struct bio *bio_in;
37 struct bio *bio_out;
38 unsigned int offset_in;
39 unsigned int offset_out;
40 unsigned int idx_in;
41 unsigned int idx_out;
42 sector_t sector;
Milan Broz43d69032008-02-08 02:11:09 +000043 atomic_t pending;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044};
45
Milan Broz53017032008-02-08 02:10:38 +000046/*
47 * per bio private data
48 */
49struct dm_crypt_io {
50 struct dm_target *target;
51 struct bio *base_bio;
52 struct work_struct work;
53
54 struct convert_context ctx;
55
56 atomic_t pending;
57 int error;
Milan Broz0c395b02008-02-08 02:10:54 +000058 sector_t sector;
Milan Broz393b47e2008-10-21 17:45:02 +010059 struct dm_crypt_io *base_io;
Milan Broz53017032008-02-08 02:10:38 +000060};
61
Milan Broz01482b72008-02-08 02:11:04 +000062struct dm_crypt_request {
Huang Yingb2174ee2009-03-16 17:44:33 +000063 struct convert_context *ctx;
Milan Broz01482b72008-02-08 02:11:04 +000064 struct scatterlist sg_in;
65 struct scatterlist sg_out;
66};
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068struct crypt_config;
69
70struct crypt_iv_operations {
71 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +010072 const char *opts);
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 void (*dtr)(struct crypt_config *cc);
Milan Brozb95bf2d2009-12-10 23:51:56 +000074 int (*init)(struct crypt_config *cc);
Milan Broz542da312009-12-10 23:51:57 +000075 int (*wipe)(struct crypt_config *cc);
Linus Torvalds1da177e2005-04-16 15:20:36 -070076 int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
77};
78
Milan Broz60473592009-12-10 23:51:55 +000079struct iv_essiv_private {
80 struct crypto_cipher *tfm;
Milan Brozb95bf2d2009-12-10 23:51:56 +000081 struct crypto_hash *hash_tfm;
82 u8 *salt;
Milan Broz60473592009-12-10 23:51:55 +000083};
84
85struct iv_benbi_private {
86 int shift;
87};
88
Linus Torvalds1da177e2005-04-16 15:20:36 -070089/*
90 * Crypt: maps a linear range of a block device
91 * and encrypts / decrypts at the same time.
92 */
Milan Broze48d4bb2006-10-03 01:15:37 -070093enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
Linus Torvalds1da177e2005-04-16 15:20:36 -070094struct crypt_config {
95 struct dm_dev *dev;
96 sector_t start;
97
98 /*
Milan Brozddd42ed2008-02-08 02:11:07 +000099 * pool for per bio private data, crypto requests and
100 * encryption requeusts/buffer pages
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 */
102 mempool_t *io_pool;
Milan Brozddd42ed2008-02-08 02:11:07 +0000103 mempool_t *req_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 mempool_t *page_pool;
Milan Broz6a24c712006-10-03 01:15:40 -0700105 struct bio_set *bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Milan Brozcabf08e2007-10-19 22:38:58 +0100107 struct workqueue_struct *io_queue;
108 struct workqueue_struct *crypt_queue;
Milan Broz3f1e9072008-03-28 14:16:07 -0700109
Milan Broz5ebaee62010-08-12 04:14:07 +0100110 char *cipher;
111 char *cipher_mode;
112
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 struct crypt_iv_operations *iv_gen_ops;
Herbert Xu79066ad2006-12-05 13:41:52 -0800114 union {
Milan Broz60473592009-12-10 23:51:55 +0000115 struct iv_essiv_private essiv;
116 struct iv_benbi_private benbi;
Herbert Xu79066ad2006-12-05 13:41:52 -0800117 } iv_gen_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 sector_t iv_offset;
119 unsigned int iv_size;
120
Milan Brozddd42ed2008-02-08 02:11:07 +0000121 /*
122 * Layout of each crypto request:
123 *
124 * struct ablkcipher_request
125 * context
126 * padding
127 * struct dm_crypt_request
128 * padding
129 * IV
130 *
131 * The padding is added so that dm_crypt_request and the IV are
132 * correctly aligned.
133 */
134 unsigned int dmreq_start;
135 struct ablkcipher_request *req;
136
Milan Broz3a7f6c92008-02-08 02:11:14 +0000137 struct crypto_ablkcipher *tfm;
Milan Broze48d4bb2006-10-03 01:15:37 -0700138 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 unsigned int key_size;
140 u8 key[0];
141};
142
Milan Broz6a24c712006-10-03 01:15:40 -0700143#define MIN_IOS 16
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144#define MIN_POOL_PAGES 32
145#define MIN_BIO_PAGES 8
146
Christoph Lametere18b8902006-12-06 20:33:20 -0800147static struct kmem_cache *_crypt_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100149static void clone_init(struct dm_crypt_io *, struct bio *);
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000150static void kcryptd_queue_crypt(struct dm_crypt_io *io);
Olaf Kirch027581f2007-05-09 02:32:52 -0700151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 * Different IV generation algorithms:
154 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000155 * plain: the initial vector is the 32-bit little-endian version of the sector
Robert P. J. Day3a4fa0a2007-10-19 23:10:43 +0200156 * number, padded with zeros if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 *
Milan Broz61afef62009-12-10 23:52:25 +0000158 * plain64: the initial vector is the 64-bit little-endian version of the sector
159 * number, padded with zeros if necessary.
160 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000161 * essiv: "encrypted sector|salt initial vector", the sector number is
162 * encrypted with the bulk cipher using a salt as key. The salt
163 * should be derived from the bulk cipher's key via hashing.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 *
Rik Snel48527fa2006-09-03 08:56:39 +1000165 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
166 * (needed for LRW-32-AES and possible other narrow block modes)
167 *
Ludwig Nussel46b47732007-05-09 02:32:55 -0700168 * null: the initial vector is always zero. Provides compatibility with
169 * obsolete loop_fish2 devices. Do not use for new devices.
170 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 * plumb: unimplemented, see:
172 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
173 */
174
175static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
176{
177 memset(iv, 0, cc->iv_size);
178 *(u32 *)iv = cpu_to_le32(sector & 0xffffffff);
179
180 return 0;
181}
182
Milan Broz61afef62009-12-10 23:52:25 +0000183static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
184 sector_t sector)
185{
186 memset(iv, 0, cc->iv_size);
187 *(u64 *)iv = cpu_to_le64(sector);
188
189 return 0;
190}
191
Milan Brozb95bf2d2009-12-10 23:51:56 +0000192/* Initialise ESSIV - compute salt but no local memory allocations */
193static int crypt_iv_essiv_init(struct crypt_config *cc)
194{
195 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
196 struct hash_desc desc;
197 struct scatterlist sg;
198 int err;
199
200 sg_init_one(&sg, cc->key, cc->key_size);
201 desc.tfm = essiv->hash_tfm;
202 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
203
204 err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt);
205 if (err)
206 return err;
207
208 return crypto_cipher_setkey(essiv->tfm, essiv->salt,
209 crypto_hash_digestsize(essiv->hash_tfm));
210}
211
Milan Broz542da312009-12-10 23:51:57 +0000212/* Wipe salt and reset key derived from volume key */
213static int crypt_iv_essiv_wipe(struct crypt_config *cc)
214{
215 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
216 unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
217
218 memset(essiv->salt, 0, salt_size);
219
220 return crypto_cipher_setkey(essiv->tfm, essiv->salt, salt_size);
221}
222
Milan Broz60473592009-12-10 23:51:55 +0000223static void crypt_iv_essiv_dtr(struct crypt_config *cc)
224{
225 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
226
227 crypto_free_cipher(essiv->tfm);
228 essiv->tfm = NULL;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000229
230 crypto_free_hash(essiv->hash_tfm);
231 essiv->hash_tfm = NULL;
232
233 kzfree(essiv->salt);
234 essiv->salt = NULL;
Milan Broz60473592009-12-10 23:51:55 +0000235}
236
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +0100238 const char *opts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239{
Milan Broz5861f1b2009-12-10 23:51:56 +0000240 struct crypto_cipher *essiv_tfm = NULL;
241 struct crypto_hash *hash_tfm = NULL;
Milan Broz5861f1b2009-12-10 23:51:56 +0000242 u8 *salt = NULL;
Herbert Xud1806f62006-08-22 20:29:17 +1000243 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
Milan Broz5861f1b2009-12-10 23:51:56 +0000245 if (!opts) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700246 ti->error = "Digest algorithm missing for ESSIV mode";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 return -EINVAL;
248 }
249
Milan Brozb95bf2d2009-12-10 23:51:56 +0000250 /* Allocate hash algorithm */
Herbert Xu35058682006-08-24 19:10:20 +1000251 hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
252 if (IS_ERR(hash_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700253 ti->error = "Error initializing ESSIV hash";
Milan Broz5861f1b2009-12-10 23:51:56 +0000254 err = PTR_ERR(hash_tfm);
255 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 }
257
Milan Brozb95bf2d2009-12-10 23:51:56 +0000258 salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL);
Milan Broz5861f1b2009-12-10 23:51:56 +0000259 if (!salt) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700260 ti->error = "Error kmallocing salt storage in ESSIV";
Milan Broz5861f1b2009-12-10 23:51:56 +0000261 err = -ENOMEM;
262 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 }
264
Milan Brozb95bf2d2009-12-10 23:51:56 +0000265 /* Allocate essiv_tfm */
Herbert Xud1806f62006-08-22 20:29:17 +1000266 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
267 if (IS_ERR(essiv_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700268 ti->error = "Error allocating crypto tfm for ESSIV";
Milan Broz5861f1b2009-12-10 23:51:56 +0000269 err = PTR_ERR(essiv_tfm);
270 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 }
Herbert Xud1806f62006-08-22 20:29:17 +1000272 if (crypto_cipher_blocksize(essiv_tfm) !=
Milan Broz3a7f6c92008-02-08 02:11:14 +0000273 crypto_ablkcipher_ivsize(cc->tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700274 ti->error = "Block size of ESSIV cipher does "
Milan Brozd469f842007-10-19 22:42:37 +0100275 "not match IV size of block cipher";
Milan Broz5861f1b2009-12-10 23:51:56 +0000276 err = -EINVAL;
277 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Milan Brozb95bf2d2009-12-10 23:51:56 +0000280 cc->iv_gen_private.essiv.salt = salt;
Milan Broz60473592009-12-10 23:51:55 +0000281 cc->iv_gen_private.essiv.tfm = essiv_tfm;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000282 cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
283
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 return 0;
Milan Broz5861f1b2009-12-10 23:51:56 +0000285
286bad:
287 if (essiv_tfm && !IS_ERR(essiv_tfm))
288 crypto_free_cipher(essiv_tfm);
289 if (hash_tfm && !IS_ERR(hash_tfm))
290 crypto_free_hash(hash_tfm);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000291 kfree(salt);
Milan Broz5861f1b2009-12-10 23:51:56 +0000292 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293}
294
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
296{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 memset(iv, 0, cc->iv_size);
298 *(u64 *)iv = cpu_to_le64(sector);
Milan Broz60473592009-12-10 23:51:55 +0000299 crypto_cipher_encrypt_one(cc->iv_gen_private.essiv.tfm, iv, iv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 return 0;
301}
302
Rik Snel48527fa2006-09-03 08:56:39 +1000303static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
304 const char *opts)
305{
Milan Broz3a7f6c92008-02-08 02:11:14 +0000306 unsigned bs = crypto_ablkcipher_blocksize(cc->tfm);
David Howellsf0d1b0b2006-12-08 02:37:49 -0800307 int log = ilog2(bs);
Rik Snel48527fa2006-09-03 08:56:39 +1000308
309 /* we need to calculate how far we must shift the sector count
310 * to get the cipher block count, we use this shift in _gen */
311
312 if (1 << log != bs) {
313 ti->error = "cypher blocksize is not a power of 2";
314 return -EINVAL;
315 }
316
317 if (log > 9) {
318 ti->error = "cypher blocksize is > 512";
319 return -EINVAL;
320 }
321
Milan Broz60473592009-12-10 23:51:55 +0000322 cc->iv_gen_private.benbi.shift = 9 - log;
Rik Snel48527fa2006-09-03 08:56:39 +1000323
324 return 0;
325}
326
327static void crypt_iv_benbi_dtr(struct crypt_config *cc)
328{
Rik Snel48527fa2006-09-03 08:56:39 +1000329}
330
331static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
332{
Herbert Xu79066ad2006-12-05 13:41:52 -0800333 __be64 val;
334
Rik Snel48527fa2006-09-03 08:56:39 +1000335 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
Herbert Xu79066ad2006-12-05 13:41:52 -0800336
Milan Broz60473592009-12-10 23:51:55 +0000337 val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi.shift) + 1);
Herbert Xu79066ad2006-12-05 13:41:52 -0800338 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
Rik Snel48527fa2006-09-03 08:56:39 +1000339
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 return 0;
341}
342
Ludwig Nussel46b47732007-05-09 02:32:55 -0700343static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
344{
345 memset(iv, 0, cc->iv_size);
346
347 return 0;
348}
349
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350static struct crypt_iv_operations crypt_iv_plain_ops = {
351 .generator = crypt_iv_plain_gen
352};
353
Milan Broz61afef62009-12-10 23:52:25 +0000354static struct crypt_iv_operations crypt_iv_plain64_ops = {
355 .generator = crypt_iv_plain64_gen
356};
357
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358static struct crypt_iv_operations crypt_iv_essiv_ops = {
359 .ctr = crypt_iv_essiv_ctr,
360 .dtr = crypt_iv_essiv_dtr,
Milan Brozb95bf2d2009-12-10 23:51:56 +0000361 .init = crypt_iv_essiv_init,
Milan Broz542da312009-12-10 23:51:57 +0000362 .wipe = crypt_iv_essiv_wipe,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 .generator = crypt_iv_essiv_gen
364};
365
Rik Snel48527fa2006-09-03 08:56:39 +1000366static struct crypt_iv_operations crypt_iv_benbi_ops = {
367 .ctr = crypt_iv_benbi_ctr,
368 .dtr = crypt_iv_benbi_dtr,
369 .generator = crypt_iv_benbi_gen
370};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371
Ludwig Nussel46b47732007-05-09 02:32:55 -0700372static struct crypt_iv_operations crypt_iv_null_ops = {
373 .generator = crypt_iv_null_gen
374};
375
Milan Brozd469f842007-10-19 22:42:37 +0100376static void crypt_convert_init(struct crypt_config *cc,
377 struct convert_context *ctx,
378 struct bio *bio_out, struct bio *bio_in,
Milan Brozfcd369d2008-02-08 02:10:41 +0000379 sector_t sector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380{
381 ctx->bio_in = bio_in;
382 ctx->bio_out = bio_out;
383 ctx->offset_in = 0;
384 ctx->offset_out = 0;
385 ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
386 ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
387 ctx->sector = sector + cc->iv_offset;
Milan Broz43d69032008-02-08 02:11:09 +0000388 init_completion(&ctx->restart);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389}
390
Huang Yingb2174ee2009-03-16 17:44:33 +0000391static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
392 struct ablkcipher_request *req)
393{
394 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
395}
396
397static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc,
398 struct dm_crypt_request *dmreq)
399{
400 return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start);
401}
402
Milan Broz01482b72008-02-08 02:11:04 +0000403static int crypt_convert_block(struct crypt_config *cc,
Milan Broz3a7f6c92008-02-08 02:11:14 +0000404 struct convert_context *ctx,
405 struct ablkcipher_request *req)
Milan Broz01482b72008-02-08 02:11:04 +0000406{
407 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
408 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000409 struct dm_crypt_request *dmreq;
410 u8 *iv;
411 int r = 0;
Milan Broz01482b72008-02-08 02:11:04 +0000412
Huang Yingb2174ee2009-03-16 17:44:33 +0000413 dmreq = dmreq_of_req(cc, req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000414 iv = (u8 *)ALIGN((unsigned long)(dmreq + 1),
415 crypto_ablkcipher_alignmask(cc->tfm) + 1);
416
Huang Yingb2174ee2009-03-16 17:44:33 +0000417 dmreq->ctx = ctx;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000418 sg_init_table(&dmreq->sg_in, 1);
419 sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000420 bv_in->bv_offset + ctx->offset_in);
421
Milan Broz3a7f6c92008-02-08 02:11:14 +0000422 sg_init_table(&dmreq->sg_out, 1);
423 sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT,
Milan Broz01482b72008-02-08 02:11:04 +0000424 bv_out->bv_offset + ctx->offset_out);
425
426 ctx->offset_in += 1 << SECTOR_SHIFT;
427 if (ctx->offset_in >= bv_in->bv_len) {
428 ctx->offset_in = 0;
429 ctx->idx_in++;
430 }
431
432 ctx->offset_out += 1 << SECTOR_SHIFT;
433 if (ctx->offset_out >= bv_out->bv_len) {
434 ctx->offset_out = 0;
435 ctx->idx_out++;
436 }
437
Milan Broz3a7f6c92008-02-08 02:11:14 +0000438 if (cc->iv_gen_ops) {
439 r = cc->iv_gen_ops->generator(cc, iv, ctx->sector);
440 if (r < 0)
441 return r;
442 }
443
444 ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
445 1 << SECTOR_SHIFT, iv);
446
447 if (bio_data_dir(ctx->bio_in) == WRITE)
448 r = crypto_ablkcipher_encrypt(req);
449 else
450 r = crypto_ablkcipher_decrypt(req);
451
452 return r;
Milan Broz01482b72008-02-08 02:11:04 +0000453}
454
Milan Broz95497a92008-02-08 02:11:12 +0000455static void kcryptd_async_done(struct crypto_async_request *async_req,
456 int error);
Milan Brozddd42ed2008-02-08 02:11:07 +0000457static void crypt_alloc_req(struct crypt_config *cc,
458 struct convert_context *ctx)
459{
460 if (!cc->req)
461 cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
Milan Broz95497a92008-02-08 02:11:12 +0000462 ablkcipher_request_set_tfm(cc->req, cc->tfm);
463 ablkcipher_request_set_callback(cc->req, CRYPTO_TFM_REQ_MAY_BACKLOG |
Huang Yingb2174ee2009-03-16 17:44:33 +0000464 CRYPTO_TFM_REQ_MAY_SLEEP,
465 kcryptd_async_done,
466 dmreq_of_req(cc, cc->req));
Milan Brozddd42ed2008-02-08 02:11:07 +0000467}
468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469/*
470 * Encrypt / decrypt data from one bio to another one (can be the same one)
471 */
472static int crypt_convert(struct crypt_config *cc,
Milan Brozd469f842007-10-19 22:42:37 +0100473 struct convert_context *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474{
Milan Broz3f1e9072008-03-28 14:16:07 -0700475 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Milan Brozc8081612008-10-10 13:37:08 +0100477 atomic_set(&ctx->pending, 1);
478
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
480 ctx->idx_out < ctx->bio_out->bi_vcnt) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481
Milan Broz3a7f6c92008-02-08 02:11:14 +0000482 crypt_alloc_req(cc, ctx);
483
Milan Broz3f1e9072008-03-28 14:16:07 -0700484 atomic_inc(&ctx->pending);
485
Milan Broz3a7f6c92008-02-08 02:11:14 +0000486 r = crypt_convert_block(cc, ctx, cc->req);
487
488 switch (r) {
Milan Broz3f1e9072008-03-28 14:16:07 -0700489 /* async */
Milan Broz3a7f6c92008-02-08 02:11:14 +0000490 case -EBUSY:
491 wait_for_completion(&ctx->restart);
492 INIT_COMPLETION(ctx->restart);
493 /* fall through*/
494 case -EINPROGRESS:
Milan Broz3a7f6c92008-02-08 02:11:14 +0000495 cc->req = NULL;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000496 ctx->sector++;
497 continue;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000498
Milan Broz3f1e9072008-03-28 14:16:07 -0700499 /* sync */
500 case 0:
501 atomic_dec(&ctx->pending);
502 ctx->sector++;
Milan Brozc7f1b202008-07-02 09:34:28 +0100503 cond_resched();
Milan Broz3f1e9072008-03-28 14:16:07 -0700504 continue;
505
506 /* error */
507 default:
508 atomic_dec(&ctx->pending);
509 return r;
510 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 }
512
Milan Broz3f1e9072008-03-28 14:16:07 -0700513 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514}
515
Milan Brozd469f842007-10-19 22:42:37 +0100516static void dm_crypt_bio_destructor(struct bio *bio)
517{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100518 struct dm_crypt_io *io = bio->bi_private;
Milan Broz6a24c712006-10-03 01:15:40 -0700519 struct crypt_config *cc = io->target->private;
520
521 bio_free(bio, cc->bs);
Milan Brozd469f842007-10-19 22:42:37 +0100522}
Milan Broz6a24c712006-10-03 01:15:40 -0700523
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524/*
525 * Generate a new unfragmented bio with the given size
526 * This should never violate the device limitations
Milan Broz933f01d2008-10-10 13:37:08 +0100527 * May return a smaller bio when running out of pages, indicated by
528 * *out_of_pages set to 1.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 */
Milan Broz933f01d2008-10-10 13:37:08 +0100530static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
531 unsigned *out_of_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532{
Olaf Kirch027581f2007-05-09 02:32:52 -0700533 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -0700534 struct bio *clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Al Virob4e3ca12005-10-21 03:22:34 -0400536 gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
Milan Broz91e10622007-12-13 14:16:10 +0000537 unsigned i, len;
538 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
Olaf Kirch2f9941b2007-05-09 02:32:53 -0700540 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
Milan Broz8b004452006-10-03 01:15:37 -0700541 if (!clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543
Olaf Kirch027581f2007-05-09 02:32:52 -0700544 clone_init(io, clone);
Milan Broz933f01d2008-10-10 13:37:08 +0100545 *out_of_pages = 0;
Milan Broz6a24c712006-10-03 01:15:40 -0700546
Olaf Kirchf97380b2007-05-09 02:32:54 -0700547 for (i = 0; i < nr_iovecs; i++) {
Milan Broz91e10622007-12-13 14:16:10 +0000548 page = mempool_alloc(cc->page_pool, gfp_mask);
Milan Broz933f01d2008-10-10 13:37:08 +0100549 if (!page) {
550 *out_of_pages = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551 break;
Milan Broz933f01d2008-10-10 13:37:08 +0100552 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553
554 /*
555 * if additional pages cannot be allocated without waiting,
556 * return a partially allocated bio, the caller will then try
557 * to allocate additional bios while submitting this partial bio
558 */
Olaf Kirchf97380b2007-05-09 02:32:54 -0700559 if (i == (MIN_BIO_PAGES - 1))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
561
Milan Broz91e10622007-12-13 14:16:10 +0000562 len = (size > PAGE_SIZE) ? PAGE_SIZE : size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
Milan Broz91e10622007-12-13 14:16:10 +0000564 if (!bio_add_page(clone, page, len, 0)) {
565 mempool_free(page, cc->page_pool);
566 break;
567 }
568
569 size -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 }
571
Milan Broz8b004452006-10-03 01:15:37 -0700572 if (!clone->bi_size) {
573 bio_put(clone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 return NULL;
575 }
576
Milan Broz8b004452006-10-03 01:15:37 -0700577 return clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578}
579
Neil Brown644bd2f2007-10-16 13:48:46 +0200580static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581{
Neil Brown644bd2f2007-10-16 13:48:46 +0200582 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 struct bio_vec *bv;
584
Neil Brown644bd2f2007-10-16 13:48:46 +0200585 for (i = 0; i < clone->bi_vcnt; i++) {
Milan Broz8b004452006-10-03 01:15:37 -0700586 bv = bio_iovec_idx(clone, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587 BUG_ON(!bv->bv_page);
588 mempool_free(bv->bv_page, cc->page_pool);
589 bv->bv_page = NULL;
590 }
591}
592
Milan Brozdc440d1e2008-10-10 13:37:03 +0100593static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti,
594 struct bio *bio, sector_t sector)
595{
596 struct crypt_config *cc = ti->private;
597 struct dm_crypt_io *io;
598
599 io = mempool_alloc(cc->io_pool, GFP_NOIO);
600 io->target = ti;
601 io->base_bio = bio;
602 io->sector = sector;
603 io->error = 0;
Milan Broz393b47e2008-10-21 17:45:02 +0100604 io->base_io = NULL;
Milan Brozdc440d1e2008-10-10 13:37:03 +0100605 atomic_set(&io->pending, 0);
606
607 return io;
608}
609
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100610static void crypt_inc_pending(struct dm_crypt_io *io)
611{
612 atomic_inc(&io->pending);
613}
614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615/*
616 * One of the bios was finished. Check for completion of
617 * the whole request and correctly clean up the buffer.
Milan Broz393b47e2008-10-21 17:45:02 +0100618 * If base_io is set, wait for the last fragment to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 */
Milan Broz5742fd72008-02-08 02:10:43 +0000620static void crypt_dec_pending(struct dm_crypt_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621{
Milan Broz5742fd72008-02-08 02:10:43 +0000622 struct crypt_config *cc = io->target->private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000623 struct bio *base_bio = io->base_bio;
624 struct dm_crypt_io *base_io = io->base_io;
625 int error = io->error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626
627 if (!atomic_dec_and_test(&io->pending))
628 return;
629
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 mempool_free(io, cc->io_pool);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000631
632 if (likely(!base_io))
633 bio_endio(base_bio, error);
634 else {
635 if (error && !base_io->error)
636 base_io->error = error;
637 crypt_dec_pending(base_io);
638 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639}
640
641/*
Milan Brozcabf08e2007-10-19 22:38:58 +0100642 * kcryptd/kcryptd_io:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643 *
644 * Needed because it would be very unwise to do decryption in an
Milan Broz23541d22006-10-03 01:15:39 -0700645 * interrupt context.
Milan Brozcabf08e2007-10-19 22:38:58 +0100646 *
647 * kcryptd performs the actual encryption or decryption.
648 *
649 * kcryptd_io performs the IO submission.
650 *
651 * They must be separated as otherwise the final stages could be
652 * starved by new requests which can block in the first stages due
653 * to memory allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 */
NeilBrown6712ecf2007-09-27 12:47:43 +0200655static void crypt_endio(struct bio *clone, int error)
Milan Broz8b004452006-10-03 01:15:37 -0700656{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100657 struct dm_crypt_io *io = clone->bi_private;
Milan Broz8b004452006-10-03 01:15:37 -0700658 struct crypt_config *cc = io->target->private;
Milan Brozee7a4912008-02-08 02:10:46 +0000659 unsigned rw = bio_data_dir(clone);
Milan Broz8b004452006-10-03 01:15:37 -0700660
Milan Brozadfe4772007-12-13 14:15:51 +0000661 if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error))
662 error = -EIO;
663
Milan Broz8b004452006-10-03 01:15:37 -0700664 /*
NeilBrown6712ecf2007-09-27 12:47:43 +0200665 * free the processed pages
Milan Broz8b004452006-10-03 01:15:37 -0700666 */
Milan Brozee7a4912008-02-08 02:10:46 +0000667 if (rw == WRITE)
Neil Brown644bd2f2007-10-16 13:48:46 +0200668 crypt_free_buffer_pages(cc, clone);
Milan Brozee7a4912008-02-08 02:10:46 +0000669
670 bio_put(clone);
671
672 if (rw == READ && !error) {
673 kcryptd_queue_crypt(io);
674 return;
NeilBrown6712ecf2007-09-27 12:47:43 +0200675 }
Milan Broz8b004452006-10-03 01:15:37 -0700676
Milan Brozadfe4772007-12-13 14:15:51 +0000677 if (unlikely(error))
Milan Broz5742fd72008-02-08 02:10:43 +0000678 io->error = error;
679
680 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700681}
682
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100683static void clone_init(struct dm_crypt_io *io, struct bio *clone)
Milan Broz8b004452006-10-03 01:15:37 -0700684{
685 struct crypt_config *cc = io->target->private;
686
687 clone->bi_private = io;
688 clone->bi_end_io = crypt_endio;
689 clone->bi_bdev = cc->dev->bdev;
690 clone->bi_rw = io->base_bio->bi_rw;
Olaf Kirch027581f2007-05-09 02:32:52 -0700691 clone->bi_destructor = dm_crypt_bio_destructor;
Milan Broz8b004452006-10-03 01:15:37 -0700692}
693
Milan Broz4e4eef62008-02-08 02:10:49 +0000694static void kcryptd_io_read(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700695{
696 struct crypt_config *cc = io->target->private;
697 struct bio *base_bio = io->base_bio;
698 struct bio *clone;
Milan Broz93e605c2006-10-03 01:15:38 -0700699
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100700 crypt_inc_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700701
702 /*
703 * The block layer might modify the bvec array, so always
704 * copy the required bvecs because we need the original
705 * one in order to decrypt the whole bio data *afterwards*.
706 */
Milan Broz6a24c712006-10-03 01:15:40 -0700707 clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs);
Milan Broz93e605c2006-10-03 01:15:38 -0700708 if (unlikely(!clone)) {
Milan Broz5742fd72008-02-08 02:10:43 +0000709 io->error = -ENOMEM;
710 crypt_dec_pending(io);
Milan Broz23541d22006-10-03 01:15:39 -0700711 return;
Milan Broz93e605c2006-10-03 01:15:38 -0700712 }
Milan Broz8b004452006-10-03 01:15:37 -0700713
714 clone_init(io, clone);
715 clone->bi_idx = 0;
716 clone->bi_vcnt = bio_segments(base_bio);
717 clone->bi_size = base_bio->bi_size;
Milan Broz0c395b02008-02-08 02:10:54 +0000718 clone->bi_sector = cc->start + io->sector;
Milan Broz8b004452006-10-03 01:15:37 -0700719 memcpy(clone->bi_io_vec, bio_iovec(base_bio),
720 sizeof(struct bio_vec) * clone->bi_vcnt);
Milan Broz8b004452006-10-03 01:15:37 -0700721
Milan Broz93e605c2006-10-03 01:15:38 -0700722 generic_make_request(clone);
Milan Broz8b004452006-10-03 01:15:37 -0700723}
724
Milan Broz4e4eef62008-02-08 02:10:49 +0000725static void kcryptd_io_write(struct dm_crypt_io *io)
726{
Milan Broz95497a92008-02-08 02:11:12 +0000727 struct bio *clone = io->ctx.bio_out;
Milan Broz95497a92008-02-08 02:11:12 +0000728 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +0000729}
730
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000731static void kcryptd_io(struct work_struct *work)
732{
733 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
734
735 if (bio_data_dir(io->base_bio) == READ)
736 kcryptd_io_read(io);
737 else
738 kcryptd_io_write(io);
739}
740
741static void kcryptd_queue_io(struct dm_crypt_io *io)
742{
743 struct crypt_config *cc = io->target->private;
744
745 INIT_WORK(&io->work, kcryptd_io);
746 queue_work(cc->io_queue, &io->work);
747}
748
Milan Broz95497a92008-02-08 02:11:12 +0000749static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io,
750 int error, int async)
Milan Broz4e4eef62008-02-08 02:10:49 +0000751{
Milan Brozdec1ced2008-02-08 02:10:57 +0000752 struct bio *clone = io->ctx.bio_out;
753 struct crypt_config *cc = io->target->private;
754
755 if (unlikely(error < 0)) {
756 crypt_free_buffer_pages(cc, clone);
757 bio_put(clone);
758 io->error = -EIO;
Milan Broz6c031f42008-10-10 13:37:06 +0100759 crypt_dec_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +0000760 return;
761 }
762
763 /* crypt_convert should have filled the clone bio */
764 BUG_ON(io->ctx.idx_out < clone->bi_vcnt);
765
766 clone->bi_sector = cc->start + io->sector;
Milan Broz899c95d2008-02-08 02:11:02 +0000767
Milan Broz95497a92008-02-08 02:11:12 +0000768 if (async)
769 kcryptd_queue_io(io);
Alasdair G Kergon1e37bb82008-10-10 13:37:05 +0100770 else
Milan Broz95497a92008-02-08 02:11:12 +0000771 generic_make_request(clone);
Milan Broz4e4eef62008-02-08 02:10:49 +0000772}
773
Milan Brozfc5a5e92008-10-10 13:37:04 +0100774static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700775{
776 struct crypt_config *cc = io->target->private;
Milan Broz8b004452006-10-03 01:15:37 -0700777 struct bio *clone;
Milan Broz393b47e2008-10-21 17:45:02 +0100778 struct dm_crypt_io *new_io;
Milan Brozc8081612008-10-10 13:37:08 +0100779 int crypt_finished;
Milan Broz933f01d2008-10-10 13:37:08 +0100780 unsigned out_of_pages = 0;
Milan Brozdec1ced2008-02-08 02:10:57 +0000781 unsigned remaining = io->base_bio->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +0100782 sector_t sector = io->sector;
Milan Brozdec1ced2008-02-08 02:10:57 +0000783 int r;
Milan Broz8b004452006-10-03 01:15:37 -0700784
Milan Broz93e605c2006-10-03 01:15:38 -0700785 /*
Milan Brozfc5a5e92008-10-10 13:37:04 +0100786 * Prevent io from disappearing until this function completes.
787 */
788 crypt_inc_pending(io);
Milan Brozb635b002008-10-21 17:45:00 +0100789 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
Milan Brozfc5a5e92008-10-10 13:37:04 +0100790
791 /*
Milan Broz93e605c2006-10-03 01:15:38 -0700792 * The allocated buffers can be smaller than the whole bio,
793 * so repeat the whole process until all the data can be handled.
794 */
795 while (remaining) {
Milan Broz933f01d2008-10-10 13:37:08 +0100796 clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
Milan Broz23541d22006-10-03 01:15:39 -0700797 if (unlikely(!clone)) {
Milan Broz5742fd72008-02-08 02:10:43 +0000798 io->error = -ENOMEM;
Milan Brozfc5a5e92008-10-10 13:37:04 +0100799 break;
Milan Broz23541d22006-10-03 01:15:39 -0700800 }
Milan Broz93e605c2006-10-03 01:15:38 -0700801
Milan Broz53017032008-02-08 02:10:38 +0000802 io->ctx.bio_out = clone;
803 io->ctx.idx_out = 0;
Milan Broz93e605c2006-10-03 01:15:38 -0700804
Milan Broz93e605c2006-10-03 01:15:38 -0700805 remaining -= clone->bi_size;
Milan Brozb635b002008-10-21 17:45:00 +0100806 sector += bio_sectors(clone);
Milan Brozdec1ced2008-02-08 02:10:57 +0000807
Milan Broz4e594092008-10-10 13:37:07 +0100808 crypt_inc_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +0000809 r = crypt_convert(cc, &io->ctx);
Milan Brozc8081612008-10-10 13:37:08 +0100810 crypt_finished = atomic_dec_and_test(&io->ctx.pending);
Milan Brozdec1ced2008-02-08 02:10:57 +0000811
Milan Brozc8081612008-10-10 13:37:08 +0100812 /* Encryption was already finished, submit io now */
813 if (crypt_finished) {
Milan Broz3a7f6c92008-02-08 02:11:14 +0000814 kcryptd_crypt_write_io_submit(io, r, 0);
Milan Brozc8081612008-10-10 13:37:08 +0100815
816 /*
817 * If there was an error, do not try next fragments.
818 * For async, error is processed in async handler.
819 */
Milan Broz6c031f42008-10-10 13:37:06 +0100820 if (unlikely(r < 0))
Milan Brozfc5a5e92008-10-10 13:37:04 +0100821 break;
Milan Brozb635b002008-10-21 17:45:00 +0100822
823 io->sector = sector;
Milan Broz4e594092008-10-10 13:37:07 +0100824 }
Milan Broz93e605c2006-10-03 01:15:38 -0700825
Milan Broz933f01d2008-10-10 13:37:08 +0100826 /*
827 * Out of memory -> run queues
828 * But don't wait if split was due to the io size restriction
829 */
830 if (unlikely(out_of_pages))
Jens Axboe8aa7e842009-07-09 14:52:32 +0200831 congestion_wait(BLK_RW_ASYNC, HZ/100);
Milan Broz933f01d2008-10-10 13:37:08 +0100832
Milan Broz393b47e2008-10-21 17:45:02 +0100833 /*
834 * With async crypto it is unsafe to share the crypto context
835 * between fragments, so switch to a new dm_crypt_io structure.
836 */
837 if (unlikely(!crypt_finished && remaining)) {
838 new_io = crypt_io_alloc(io->target, io->base_bio,
839 sector);
840 crypt_inc_pending(new_io);
841 crypt_convert_init(cc, &new_io->ctx, NULL,
842 io->base_bio, sector);
843 new_io->ctx.idx_in = io->ctx.idx_in;
844 new_io->ctx.offset_in = io->ctx.offset_in;
845
846 /*
847 * Fragments after the first use the base_io
848 * pending count.
849 */
850 if (!io->base_io)
851 new_io->base_io = io;
852 else {
853 new_io->base_io = io->base_io;
854 crypt_inc_pending(io->base_io);
855 crypt_dec_pending(io);
856 }
857
858 io = new_io;
859 }
Milan Broz8b004452006-10-03 01:15:37 -0700860 }
Milan Broz899c95d2008-02-08 02:11:02 +0000861
862 crypt_dec_pending(io);
Milan Broz84131db2008-02-08 02:10:59 +0000863}
864
Milan Broz4e4eef62008-02-08 02:10:49 +0000865static void kcryptd_crypt_read_done(struct dm_crypt_io *io, int error)
Milan Broz5742fd72008-02-08 02:10:43 +0000866{
867 if (unlikely(error < 0))
868 io->error = -EIO;
869
870 crypt_dec_pending(io);
871}
872
Milan Broz4e4eef62008-02-08 02:10:49 +0000873static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -0700874{
875 struct crypt_config *cc = io->target->private;
Milan Broz5742fd72008-02-08 02:10:43 +0000876 int r = 0;
Milan Broz8b004452006-10-03 01:15:37 -0700877
Milan Broz3e1a8bd2008-10-10 13:37:02 +0100878 crypt_inc_pending(io);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000879
Milan Broz53017032008-02-08 02:10:38 +0000880 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
Milan Broz0c395b02008-02-08 02:10:54 +0000881 io->sector);
Milan Broz8b004452006-10-03 01:15:37 -0700882
Milan Broz5742fd72008-02-08 02:10:43 +0000883 r = crypt_convert(cc, &io->ctx);
884
Milan Broz3f1e9072008-03-28 14:16:07 -0700885 if (atomic_dec_and_test(&io->ctx.pending))
Milan Broz3a7f6c92008-02-08 02:11:14 +0000886 kcryptd_crypt_read_done(io, r);
887
888 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -0700889}
890
Milan Broz95497a92008-02-08 02:11:12 +0000891static void kcryptd_async_done(struct crypto_async_request *async_req,
892 int error)
893{
Huang Yingb2174ee2009-03-16 17:44:33 +0000894 struct dm_crypt_request *dmreq = async_req->data;
895 struct convert_context *ctx = dmreq->ctx;
Milan Broz95497a92008-02-08 02:11:12 +0000896 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
897 struct crypt_config *cc = io->target->private;
898
899 if (error == -EINPROGRESS) {
900 complete(&ctx->restart);
901 return;
902 }
903
Huang Yingb2174ee2009-03-16 17:44:33 +0000904 mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool);
Milan Broz95497a92008-02-08 02:11:12 +0000905
906 if (!atomic_dec_and_test(&ctx->pending))
907 return;
908
909 if (bio_data_dir(io->base_bio) == READ)
910 kcryptd_crypt_read_done(io, error);
911 else
912 kcryptd_crypt_write_io_submit(io, error, 1);
913}
914
Milan Broz4e4eef62008-02-08 02:10:49 +0000915static void kcryptd_crypt(struct work_struct *work)
916{
917 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
918
919 if (bio_data_dir(io->base_bio) == READ)
920 kcryptd_crypt_read_convert(io);
921 else
922 kcryptd_crypt_write_convert(io);
Milan Broz8b004452006-10-03 01:15:37 -0700923}
924
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000925static void kcryptd_queue_crypt(struct dm_crypt_io *io)
926{
927 struct crypt_config *cc = io->target->private;
928
929 INIT_WORK(&io->work, kcryptd_crypt);
930 queue_work(cc->crypt_queue, &io->work);
931}
932
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933/*
934 * Decode key from its hex representation
935 */
936static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
937{
938 char buffer[3];
939 char *endp;
940 unsigned int i;
941
942 buffer[2] = '\0';
943
Milan Broz8b004452006-10-03 01:15:37 -0700944 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 buffer[0] = *hex++;
946 buffer[1] = *hex++;
947
948 key[i] = (u8)simple_strtoul(buffer, &endp, 16);
949
950 if (endp != &buffer[2])
951 return -EINVAL;
952 }
953
954 if (*hex != '\0')
955 return -EINVAL;
956
957 return 0;
958}
959
960/*
961 * Encode key into its hex representation
962 */
963static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
964{
965 unsigned int i;
966
Milan Broz8b004452006-10-03 01:15:37 -0700967 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 sprintf(hex, "%02x", *key);
969 hex += 2;
970 key++;
971 }
972}
973
Milan Broze48d4bb2006-10-03 01:15:37 -0700974static int crypt_set_key(struct crypt_config *cc, char *key)
975{
976 unsigned key_size = strlen(key) >> 1;
977
978 if (cc->key_size && cc->key_size != key_size)
979 return -EINVAL;
980
981 cc->key_size = key_size; /* initial settings */
982
983 if ((!key_size && strcmp(key, "-")) ||
Milan Brozd469f842007-10-19 22:42:37 +0100984 (key_size && crypt_decode_key(cc->key, key, key_size) < 0))
Milan Broze48d4bb2006-10-03 01:15:37 -0700985 return -EINVAL;
986
987 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
988
Milan Broz0b430952009-12-10 23:51:55 +0000989 return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size);
Milan Broze48d4bb2006-10-03 01:15:37 -0700990}
991
992static int crypt_wipe_key(struct crypt_config *cc)
993{
994 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
995 memset(&cc->key, 0, cc->key_size * sizeof(u8));
Milan Broz0b430952009-12-10 23:51:55 +0000996 return crypto_ablkcipher_setkey(cc->tfm, cc->key, cc->key_size);
Milan Broze48d4bb2006-10-03 01:15:37 -0700997}
998
Milan Broz28513fc2010-08-12 04:14:06 +0100999static void crypt_dtr(struct dm_target *ti)
1000{
1001 struct crypt_config *cc = ti->private;
1002
1003 ti->private = NULL;
1004
1005 if (!cc)
1006 return;
1007
1008 if (cc->io_queue)
1009 destroy_workqueue(cc->io_queue);
1010 if (cc->crypt_queue)
1011 destroy_workqueue(cc->crypt_queue);
1012
1013 if (cc->bs)
1014 bioset_free(cc->bs);
1015
1016 if (cc->page_pool)
1017 mempool_destroy(cc->page_pool);
1018 if (cc->req_pool)
1019 mempool_destroy(cc->req_pool);
1020 if (cc->io_pool)
1021 mempool_destroy(cc->io_pool);
1022
1023 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
1024 cc->iv_gen_ops->dtr(cc);
1025
1026 if (cc->tfm && !IS_ERR(cc->tfm))
1027 crypto_free_ablkcipher(cc->tfm);
1028
1029 if (cc->dev)
1030 dm_put_device(ti, cc->dev);
1031
Milan Broz5ebaee62010-08-12 04:14:07 +01001032 kzfree(cc->cipher);
1033 kzfree(cc->cipher_mode);
Milan Broz28513fc2010-08-12 04:14:06 +01001034
1035 /* Must zero key material before freeing */
1036 kzfree(cc);
1037}
1038
Milan Broz5ebaee62010-08-12 04:14:07 +01001039static int crypt_ctr_cipher(struct dm_target *ti,
1040 char *cipher_in, char *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041{
Milan Broz5ebaee62010-08-12 04:14:07 +01001042 struct crypt_config *cc = ti->private;
1043 char *tmp, *cipher, *chainmode, *ivmode, *ivopts;
1044 char *cipher_api = NULL;
Milan Broz28513fc2010-08-12 04:14:06 +01001045 int ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046
Milan Broz5ebaee62010-08-12 04:14:07 +01001047 /* Convert to crypto api definition? */
1048 if (strchr(cipher_in, '(')) {
1049 ti->error = "Bad cipher specification";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050 return -EINVAL;
1051 }
1052
Milan Broz5ebaee62010-08-12 04:14:07 +01001053 /*
1054 * Legacy dm-crypt cipher specification
1055 * cipher-mode-iv:ivopts
1056 */
1057 tmp = cipher_in;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 cipher = strsep(&tmp, "-");
Milan Broz5ebaee62010-08-12 04:14:07 +01001059
1060 cc->cipher = kstrdup(cipher, GFP_KERNEL);
1061 if (!cc->cipher)
1062 goto bad_mem;
1063
1064 if (tmp) {
1065 cc->cipher_mode = kstrdup(tmp, GFP_KERNEL);
1066 if (!cc->cipher_mode)
1067 goto bad_mem;
1068 }
1069
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 chainmode = strsep(&tmp, "-");
1071 ivopts = strsep(&tmp, "-");
1072 ivmode = strsep(&ivopts, ":");
1073
1074 if (tmp)
Milan Broz5ebaee62010-08-12 04:14:07 +01001075 DMWARN("Ignoring unexpected additional cipher options");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
Milan Broz5ebaee62010-08-12 04:14:07 +01001077 /* Compatibility mode for old dm-crypt mappings */
1078 if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) {
1079 kfree(cc->cipher_mode);
1080 cc->cipher_mode = kstrdup("cbc-plain", GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 chainmode = "cbc";
1082 ivmode = "plain";
1083 }
1084
Herbert Xud1806f62006-08-22 20:29:17 +10001085 if (strcmp(chainmode, "ecb") && !ivmode) {
Milan Broz5ebaee62010-08-12 04:14:07 +01001086 ti->error = "IV mechanism required";
1087 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 }
1089
Milan Broz5ebaee62010-08-12 04:14:07 +01001090 cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
1091 if (!cipher_api)
1092 goto bad_mem;
1093
1094 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
1095 "%s(%s)", chainmode, cipher);
1096 if (ret < 0) {
1097 kfree(cipher_api);
1098 goto bad_mem;
Herbert Xud1806f62006-08-22 20:29:17 +10001099 }
1100
Milan Broz5ebaee62010-08-12 04:14:07 +01001101 /* Allocate cipher */
1102 cc->tfm = crypto_alloc_ablkcipher(cipher_api, 0, 0);
Milan Broz28513fc2010-08-12 04:14:06 +01001103 if (IS_ERR(cc->tfm)) {
Milan Broz5ebaee62010-08-12 04:14:07 +01001104 ret = PTR_ERR(cc->tfm);
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001105 ti->error = "Error allocating crypto tfm";
Milan Broz28513fc2010-08-12 04:14:06 +01001106 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Milan Broz5ebaee62010-08-12 04:14:07 +01001109 /* Initialize and set key */
1110 ret = crypt_set_key(cc, key);
Milan Broz28513fc2010-08-12 04:14:06 +01001111 if (ret < 0) {
Milan Broz0b430952009-12-10 23:51:55 +00001112 ti->error = "Error decoding and setting key";
Milan Broz28513fc2010-08-12 04:14:06 +01001113 goto bad;
Milan Broz0b430952009-12-10 23:51:55 +00001114 }
1115
Milan Broz5ebaee62010-08-12 04:14:07 +01001116 /* Initialize IV */
1117 cc->iv_size = crypto_ablkcipher_ivsize(cc->tfm);
1118 if (cc->iv_size)
1119 /* at least a 64 bit sector number should fit in our buffer */
1120 cc->iv_size = max(cc->iv_size,
1121 (unsigned int)(sizeof(u64) / sizeof(u8)));
1122 else if (ivmode) {
1123 DMWARN("Selected cipher does not support IVs");
1124 ivmode = NULL;
1125 }
1126
1127 /* Choose ivmode, see comments at iv code. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 if (ivmode == NULL)
1129 cc->iv_gen_ops = NULL;
1130 else if (strcmp(ivmode, "plain") == 0)
1131 cc->iv_gen_ops = &crypt_iv_plain_ops;
Milan Broz61afef62009-12-10 23:52:25 +00001132 else if (strcmp(ivmode, "plain64") == 0)
1133 cc->iv_gen_ops = &crypt_iv_plain64_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 else if (strcmp(ivmode, "essiv") == 0)
1135 cc->iv_gen_ops = &crypt_iv_essiv_ops;
Rik Snel48527fa2006-09-03 08:56:39 +10001136 else if (strcmp(ivmode, "benbi") == 0)
1137 cc->iv_gen_ops = &crypt_iv_benbi_ops;
Ludwig Nussel46b47732007-05-09 02:32:55 -07001138 else if (strcmp(ivmode, "null") == 0)
1139 cc->iv_gen_ops = &crypt_iv_null_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 else {
Milan Broz5ebaee62010-08-12 04:14:07 +01001141 ret = -EINVAL;
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001142 ti->error = "Invalid IV mode";
Milan Broz28513fc2010-08-12 04:14:06 +01001143 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144 }
1145
Milan Broz28513fc2010-08-12 04:14:06 +01001146 /* Allocate IV */
1147 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
1148 ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
1149 if (ret < 0) {
1150 ti->error = "Error creating IV";
1151 goto bad;
1152 }
Milan Brozb95bf2d2009-12-10 23:51:56 +00001153 }
1154
Milan Broz28513fc2010-08-12 04:14:06 +01001155 /* Initialize IV (set keys for ESSIV etc) */
1156 if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
1157 ret = cc->iv_gen_ops->init(cc);
1158 if (ret < 0) {
1159 ti->error = "Error initialising IV";
1160 goto bad;
1161 }
1162 }
1163
Milan Broz5ebaee62010-08-12 04:14:07 +01001164 ret = 0;
1165bad:
1166 kfree(cipher_api);
1167 return ret;
1168
1169bad_mem:
1170 ti->error = "Cannot allocate cipher strings";
1171 return -ENOMEM;
1172}
1173
1174/*
1175 * Construct an encryption mapping:
1176 * <cipher> <key> <iv_offset> <dev_path> <start>
1177 */
1178static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1179{
1180 struct crypt_config *cc;
1181 unsigned int key_size;
1182 unsigned long long tmpll;
1183 int ret;
1184
1185 if (argc != 5) {
1186 ti->error = "Not enough arguments";
1187 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188 }
1189
Milan Broz5ebaee62010-08-12 04:14:07 +01001190 key_size = strlen(argv[1]) >> 1;
1191
1192 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
1193 if (!cc) {
1194 ti->error = "Cannot allocate encryption context";
1195 return -ENOMEM;
1196 }
1197
1198 ti->private = cc;
1199 ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
1200 if (ret < 0)
1201 goto bad;
1202
Milan Broz28513fc2010-08-12 04:14:06 +01001203 ret = -ENOMEM;
Matthew Dobson93d23412006-03-26 01:37:50 -08001204 cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 if (!cc->io_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001206 ti->error = "Cannot allocate crypt io mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001207 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 }
1209
Milan Brozddd42ed2008-02-08 02:11:07 +00001210 cc->dmreq_start = sizeof(struct ablkcipher_request);
Milan Broz28513fc2010-08-12 04:14:06 +01001211 cc->dmreq_start += crypto_ablkcipher_reqsize(cc->tfm);
Milan Brozddd42ed2008-02-08 02:11:07 +00001212 cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment());
Milan Broz28513fc2010-08-12 04:14:06 +01001213 cc->dmreq_start += crypto_ablkcipher_alignmask(cc->tfm) &
Milan Broz3a7f6c92008-02-08 02:11:14 +00001214 ~(crypto_tfm_ctx_alignment() - 1);
Milan Brozddd42ed2008-02-08 02:11:07 +00001215
1216 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
1217 sizeof(struct dm_crypt_request) + cc->iv_size);
1218 if (!cc->req_pool) {
1219 ti->error = "Cannot allocate crypt request mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001220 goto bad;
Milan Brozddd42ed2008-02-08 02:11:07 +00001221 }
1222 cc->req = NULL;
1223
Matthew Dobsona19b27c2006-03-26 01:37:45 -08001224 cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225 if (!cc->page_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001226 ti->error = "Cannot allocate page mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001227 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 }
1229
Jens Axboebb799ca2008-12-10 15:35:05 +01001230 cc->bs = bioset_create(MIN_IOS, 0);
Milan Broz6a24c712006-10-03 01:15:40 -07001231 if (!cc->bs) {
1232 ti->error = "Cannot allocate crypt bioset";
Milan Broz28513fc2010-08-12 04:14:06 +01001233 goto bad;
Milan Broz6a24c712006-10-03 01:15:40 -07001234 }
1235
Milan Broz28513fc2010-08-12 04:14:06 +01001236 ret = -EINVAL;
Andrew Morton4ee218c2006-03-27 01:17:48 -08001237 if (sscanf(argv[2], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001238 ti->error = "Invalid iv_offset sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001239 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001241 cc->iv_offset = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242
Milan Broz28513fc2010-08-12 04:14:06 +01001243 if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) {
1244 ti->error = "Device lookup failed";
1245 goto bad;
1246 }
1247
Andrew Morton4ee218c2006-03-27 01:17:48 -08001248 if (sscanf(argv[4], "%llu", &tmpll) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001249 ti->error = "Invalid device sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001250 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001252 cc->start = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253
Milan Broz28513fc2010-08-12 04:14:06 +01001254 ret = -ENOMEM;
Milan Brozcabf08e2007-10-19 22:38:58 +01001255 cc->io_queue = create_singlethread_workqueue("kcryptd_io");
1256 if (!cc->io_queue) {
1257 ti->error = "Couldn't create kcryptd io queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001258 goto bad;
Milan Brozcabf08e2007-10-19 22:38:58 +01001259 }
1260
1261 cc->crypt_queue = create_singlethread_workqueue("kcryptd");
1262 if (!cc->crypt_queue) {
Milan Broz9934a8b2007-10-19 22:38:57 +01001263 ti->error = "Couldn't create kcryptd queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001264 goto bad;
Milan Broz9934a8b2007-10-19 22:38:57 +01001265 }
1266
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001267 ti->num_flush_requests = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 return 0;
1269
Milan Broz28513fc2010-08-12 04:14:06 +01001270bad:
1271 crypt_dtr(ti);
1272 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001273}
1274
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275static int crypt_map(struct dm_target *ti, struct bio *bio,
1276 union map_info *map_context)
1277{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001278 struct dm_crypt_io *io;
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001279 struct crypt_config *cc;
1280
1281 if (unlikely(bio_empty_barrier(bio))) {
1282 cc = ti->private;
1283 bio->bi_bdev = cc->dev->bdev;
1284 return DM_MAPIO_REMAPPED;
1285 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286
Alasdair G Kergonb441a2622010-08-12 04:14:11 +01001287 io = crypt_io_alloc(ti, bio, dm_target_offset(ti, bio->bi_sector));
Milan Brozcabf08e2007-10-19 22:38:58 +01001288
1289 if (bio_data_dir(io->base_bio) == READ)
1290 kcryptd_queue_io(io);
1291 else
1292 kcryptd_queue_crypt(io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001294 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295}
1296
1297static int crypt_status(struct dm_target *ti, status_type_t type,
1298 char *result, unsigned int maxlen)
1299{
Milan Broz5ebaee62010-08-12 04:14:07 +01001300 struct crypt_config *cc = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 unsigned int sz = 0;
1302
1303 switch (type) {
1304 case STATUSTYPE_INFO:
1305 result[0] = '\0';
1306 break;
1307
1308 case STATUSTYPE_TABLE:
Milan Broz5ebaee62010-08-12 04:14:07 +01001309 if (cc->cipher_mode)
1310 DMEMIT("%s-%s ", cc->cipher, cc->cipher_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 else
Milan Broz5ebaee62010-08-12 04:14:07 +01001312 DMEMIT("%s ", cc->cipher);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313
1314 if (cc->key_size > 0) {
1315 if ((maxlen - sz) < ((cc->key_size << 1) + 1))
1316 return -ENOMEM;
1317
1318 crypt_encode_key(result + sz, cc->key, cc->key_size);
1319 sz += cc->key_size << 1;
1320 } else {
1321 if (sz >= maxlen)
1322 return -ENOMEM;
1323 result[sz++] = '-';
1324 }
1325
Andrew Morton4ee218c2006-03-27 01:17:48 -08001326 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
1327 cc->dev->name, (unsigned long long)cc->start);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328 break;
1329 }
1330 return 0;
1331}
1332
Milan Broze48d4bb2006-10-03 01:15:37 -07001333static void crypt_postsuspend(struct dm_target *ti)
1334{
1335 struct crypt_config *cc = ti->private;
1336
1337 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1338}
1339
1340static int crypt_preresume(struct dm_target *ti)
1341{
1342 struct crypt_config *cc = ti->private;
1343
1344 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
1345 DMERR("aborting resume - crypt key is not set.");
1346 return -EAGAIN;
1347 }
1348
1349 return 0;
1350}
1351
1352static void crypt_resume(struct dm_target *ti)
1353{
1354 struct crypt_config *cc = ti->private;
1355
1356 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
1357}
1358
1359/* Message interface
1360 * key set <key>
1361 * key wipe
1362 */
1363static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
1364{
1365 struct crypt_config *cc = ti->private;
Milan Broz542da312009-12-10 23:51:57 +00001366 int ret = -EINVAL;
Milan Broze48d4bb2006-10-03 01:15:37 -07001367
1368 if (argc < 2)
1369 goto error;
1370
1371 if (!strnicmp(argv[0], MESG_STR("key"))) {
1372 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
1373 DMWARN("not suspended during key manipulation.");
1374 return -EINVAL;
1375 }
Milan Broz542da312009-12-10 23:51:57 +00001376 if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) {
1377 ret = crypt_set_key(cc, argv[2]);
1378 if (ret)
1379 return ret;
1380 if (cc->iv_gen_ops && cc->iv_gen_ops->init)
1381 ret = cc->iv_gen_ops->init(cc);
1382 return ret;
1383 }
1384 if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) {
1385 if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
1386 ret = cc->iv_gen_ops->wipe(cc);
1387 if (ret)
1388 return ret;
1389 }
Milan Broze48d4bb2006-10-03 01:15:37 -07001390 return crypt_wipe_key(cc);
Milan Broz542da312009-12-10 23:51:57 +00001391 }
Milan Broze48d4bb2006-10-03 01:15:37 -07001392 }
1393
1394error:
1395 DMWARN("unrecognised message received.");
1396 return -EINVAL;
1397}
1398
Milan Brozd41e26b2008-07-21 12:00:40 +01001399static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
1400 struct bio_vec *biovec, int max_size)
1401{
1402 struct crypt_config *cc = ti->private;
1403 struct request_queue *q = bdev_get_queue(cc->dev->bdev);
1404
1405 if (!q->merge_bvec_fn)
1406 return max_size;
1407
1408 bvm->bi_bdev = cc->dev->bdev;
Alasdair G Kergonb441a2622010-08-12 04:14:11 +01001409 bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector);
Milan Brozd41e26b2008-07-21 12:00:40 +01001410
1411 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
1412}
1413
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001414static int crypt_iterate_devices(struct dm_target *ti,
1415 iterate_devices_callout_fn fn, void *data)
1416{
1417 struct crypt_config *cc = ti->private;
1418
Mike Snitzer5dea2712009-07-23 20:30:42 +01001419 return fn(ti, cc->dev, cc->start, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001420}
1421
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422static struct target_type crypt_target = {
1423 .name = "crypt",
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001424 .version = {1, 7, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 .module = THIS_MODULE,
1426 .ctr = crypt_ctr,
1427 .dtr = crypt_dtr,
1428 .map = crypt_map,
1429 .status = crypt_status,
Milan Broze48d4bb2006-10-03 01:15:37 -07001430 .postsuspend = crypt_postsuspend,
1431 .preresume = crypt_preresume,
1432 .resume = crypt_resume,
1433 .message = crypt_message,
Milan Brozd41e26b2008-07-21 12:00:40 +01001434 .merge = crypt_merge,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01001435 .iterate_devices = crypt_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436};
1437
1438static int __init dm_crypt_init(void)
1439{
1440 int r;
1441
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001442 _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443 if (!_crypt_io_pool)
1444 return -ENOMEM;
1445
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 r = dm_register_target(&crypt_target);
1447 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001448 DMERR("register failed %d", r);
Milan Broz9934a8b2007-10-19 22:38:57 +01001449 kmem_cache_destroy(_crypt_io_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 }
1451
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 return r;
1453}
1454
1455static void __exit dm_crypt_exit(void)
1456{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001457 dm_unregister_target(&crypt_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 kmem_cache_destroy(_crypt_io_pool);
1459}
1460
1461module_init(dm_crypt_init);
1462module_exit(dm_crypt_exit);
1463
1464MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1465MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
1466MODULE_LICENSE("GPL");