blob: 7a5b75fd39d6f9566a03c627f641a2d32af5a45f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Jana Saoutbf142992014-06-24 14:27:04 -04002 * Copyright (C) 2003 Jana Saout <jana@saout.de>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
Milan Broz54cea3f2015-05-15 17:00:25 +02004 * Copyright (C) 2006-2015 Red Hat, Inc. All rights reserved.
Milan Brozed04d982013-10-28 23:21:04 +01005 * Copyright (C) 2013 Milan Broz <gmazyland@gmail.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
7 * This file is released under the GPL.
8 */
9
Milan Broz43d69032008-02-08 02:11:09 +000010#include <linux/completion.h>
Herbert Xud1806f62006-08-22 20:29:17 +100011#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/bio.h>
16#include <linux/blkdev.h>
17#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/crypto.h>
20#include <linux/workqueue.h>
Mikulas Patockadc267622015-02-13 08:25:59 -050021#include <linux/kthread.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070022#include <linux/backing-dev.h>
Arun Sharma600634972011-07-26 16:09:06 -070023#include <linux/atomic.h>
David Hardeman378f0582005-09-17 17:55:31 +100024#include <linux/scatterlist.h>
Mikulas Patockab3c5fd32015-02-13 08:27:41 -050025#include <linux/rbtree.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <asm/page.h>
Rik Snel48527fa2006-09-03 08:56:39 +100027#include <asm/unaligned.h>
Milan Broz34745782011-01-13 19:59:55 +000028#include <crypto/hash.h>
29#include <crypto/md5.h>
30#include <crypto/algapi.h>
Herbert Xubbdb23b2016-01-24 21:16:36 +080031#include <crypto/skcipher.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Mikulas Patocka586e80e2008-10-21 17:44:59 +010033#include <linux/device-mapper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Alasdair G Kergon72d94862006-06-26 00:27:35 -070035#define DM_MSG_PREFIX "crypt"
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
37/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 * context holding the current state of a multi-part conversion
39 */
40struct convert_context {
Milan Broz43d69032008-02-08 02:11:09 +000041 struct completion restart;
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 struct bio *bio_in;
43 struct bio *bio_out;
Kent Overstreet003b5c52013-10-11 15:45:43 -070044 struct bvec_iter iter_in;
45 struct bvec_iter iter_out;
Mikulas Patockac66029f2012-07-27 15:08:05 +010046 sector_t cc_sector;
Mikulas Patocka40b62292012-07-27 15:08:04 +010047 atomic_t cc_pending;
Herbert Xubbdb23b2016-01-24 21:16:36 +080048 struct skcipher_request *req;
Linus Torvalds1da177e2005-04-16 15:20:36 -070049};
50
Milan Broz53017032008-02-08 02:10:38 +000051/*
52 * per bio private data
53 */
54struct dm_crypt_io {
Alasdair G Kergon49a8a922012-07-27 15:08:05 +010055 struct crypt_config *cc;
Milan Broz53017032008-02-08 02:10:38 +000056 struct bio *base_bio;
57 struct work_struct work;
58
59 struct convert_context ctx;
60
Mikulas Patocka40b62292012-07-27 15:08:04 +010061 atomic_t io_pending;
Milan Broz53017032008-02-08 02:10:38 +000062 int error;
Milan Broz0c395b02008-02-08 02:10:54 +000063 sector_t sector;
Mikulas Patockadc267622015-02-13 08:25:59 -050064
Mikulas Patockab3c5fd32015-02-13 08:27:41 -050065 struct rb_node rb_node;
Mikulas Patocka298a9fa2014-03-28 15:51:55 -040066} CRYPTO_MINALIGN_ATTR;
Milan Broz53017032008-02-08 02:10:38 +000067
Milan Broz01482b72008-02-08 02:11:04 +000068struct dm_crypt_request {
Huang Yingb2174ee2009-03-16 17:44:33 +000069 struct convert_context *ctx;
Milan Broz01482b72008-02-08 02:11:04 +000070 struct scatterlist sg_in;
71 struct scatterlist sg_out;
Milan Broz2dc53272011-01-13 19:59:54 +000072 sector_t iv_sector;
Milan Broz01482b72008-02-08 02:11:04 +000073};
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075struct crypt_config;
76
77struct crypt_iv_operations {
78 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +010079 const char *opts);
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 void (*dtr)(struct crypt_config *cc);
Milan Brozb95bf2d2009-12-10 23:51:56 +000081 int (*init)(struct crypt_config *cc);
Milan Broz542da312009-12-10 23:51:57 +000082 int (*wipe)(struct crypt_config *cc);
Milan Broz2dc53272011-01-13 19:59:54 +000083 int (*generator)(struct crypt_config *cc, u8 *iv,
84 struct dm_crypt_request *dmreq);
85 int (*post)(struct crypt_config *cc, u8 *iv,
86 struct dm_crypt_request *dmreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -070087};
88
Milan Broz60473592009-12-10 23:51:55 +000089struct iv_essiv_private {
Herbert Xubbdb23b2016-01-24 21:16:36 +080090 struct crypto_ahash *hash_tfm;
Milan Brozb95bf2d2009-12-10 23:51:56 +000091 u8 *salt;
Milan Broz60473592009-12-10 23:51:55 +000092};
93
94struct iv_benbi_private {
95 int shift;
96};
97
Milan Broz34745782011-01-13 19:59:55 +000098#define LMK_SEED_SIZE 64 /* hash + 0 */
99struct iv_lmk_private {
100 struct crypto_shash *hash_tfm;
101 u8 *seed;
102};
103
Milan Brozed04d982013-10-28 23:21:04 +0100104#define TCW_WHITENING_SIZE 16
105struct iv_tcw_private {
106 struct crypto_shash *crc32_tfm;
107 u8 *iv_seed;
108 u8 *whitening;
109};
110
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111/*
112 * Crypt: maps a linear range of a block device
113 * and encrypts / decrypts at the same time.
114 */
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -0500115enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
Rabin Vincentf659b102016-09-21 16:22:29 +0200116 DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD };
Andi Kleenc0297722011-01-13 19:59:53 +0000117
118/*
Mikulas Patocka610f2de2014-02-20 18:01:01 -0500119 * The fields in here must be read only after initialization.
Andi Kleenc0297722011-01-13 19:59:53 +0000120 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121struct crypt_config {
122 struct dm_dev *dev;
123 sector_t start;
124
125 /*
Milan Brozddd42ed2008-02-08 02:11:07 +0000126 * pool for per bio private data, crypto requests and
127 * encryption requeusts/buffer pages
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 */
Milan Brozddd42ed2008-02-08 02:11:07 +0000129 mempool_t *req_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 mempool_t *page_pool;
Milan Broz6a24c712006-10-03 01:15:40 -0700131 struct bio_set *bs;
Mikulas Patocka7145c242015-02-13 08:24:41 -0500132 struct mutex bio_alloc_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
Milan Brozcabf08e2007-10-19 22:38:58 +0100134 struct workqueue_struct *io_queue;
135 struct workqueue_struct *crypt_queue;
Milan Broz3f1e9072008-03-28 14:16:07 -0700136
Mikulas Patockadc267622015-02-13 08:25:59 -0500137 struct task_struct *write_thread;
138 wait_queue_head_t write_thread_wait;
Mikulas Patockab3c5fd32015-02-13 08:27:41 -0500139 struct rb_root write_tree;
Mikulas Patockadc267622015-02-13 08:25:59 -0500140
Milan Broz5ebaee62010-08-12 04:14:07 +0100141 char *cipher;
Milan Broz7dbcd132011-01-13 19:59:52 +0000142 char *cipher_string;
Milan Broz5ebaee62010-08-12 04:14:07 +0100143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 struct crypt_iv_operations *iv_gen_ops;
Herbert Xu79066ad2006-12-05 13:41:52 -0800145 union {
Milan Broz60473592009-12-10 23:51:55 +0000146 struct iv_essiv_private essiv;
147 struct iv_benbi_private benbi;
Milan Broz34745782011-01-13 19:59:55 +0000148 struct iv_lmk_private lmk;
Milan Brozed04d982013-10-28 23:21:04 +0100149 struct iv_tcw_private tcw;
Herbert Xu79066ad2006-12-05 13:41:52 -0800150 } iv_gen_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 sector_t iv_offset;
152 unsigned int iv_size;
153
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100154 /* ESSIV: struct crypto_cipher *essiv_tfm */
155 void *iv_private;
Herbert Xubbdb23b2016-01-24 21:16:36 +0800156 struct crypto_skcipher **tfms;
Milan Brozd1f96422011-01-13 19:59:54 +0000157 unsigned tfms_count;
Andi Kleenc0297722011-01-13 19:59:53 +0000158
159 /*
Milan Brozddd42ed2008-02-08 02:11:07 +0000160 * Layout of each crypto request:
161 *
Herbert Xubbdb23b2016-01-24 21:16:36 +0800162 * struct skcipher_request
Milan Brozddd42ed2008-02-08 02:11:07 +0000163 * context
164 * padding
165 * struct dm_crypt_request
166 * padding
167 * IV
168 *
169 * The padding is added so that dm_crypt_request and the IV are
170 * correctly aligned.
171 */
172 unsigned int dmreq_start;
Milan Brozddd42ed2008-02-08 02:11:07 +0000173
Mikulas Patocka298a9fa2014-03-28 15:51:55 -0400174 unsigned int per_bio_data_size;
175
Milan Broze48d4bb2006-10-03 01:15:37 -0700176 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 unsigned int key_size;
Milan Brozda31a072013-10-28 23:21:03 +0100178 unsigned int key_parts; /* independent parts in key buffer */
179 unsigned int key_extra_size; /* additional keys length */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 u8 key[0];
181};
182
Mikulas Patocka0a83df62016-07-15 17:30:20 -0400183#define MIN_IOS 64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100185static void clone_init(struct dm_crypt_io *, struct bio *);
Alasdair G Kergon395b1672008-02-08 02:10:52 +0000186static void kcryptd_queue_crypt(struct dm_crypt_io *io);
Milan Broz2dc53272011-01-13 19:59:54 +0000187static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
Olaf Kirch027581f2007-05-09 02:32:52 -0700188
Andi Kleenc0297722011-01-13 19:59:53 +0000189/*
190 * Use this to access cipher attributes that are the same for each CPU.
191 */
Herbert Xubbdb23b2016-01-24 21:16:36 +0800192static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
Andi Kleenc0297722011-01-13 19:59:53 +0000193{
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100194 return cc->tfms[0];
Andi Kleenc0297722011-01-13 19:59:53 +0000195}
196
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 * Different IV generation algorithms:
199 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000200 * plain: the initial vector is the 32-bit little-endian version of the sector
Robert P. J. Day3a4fa0a2007-10-19 23:10:43 +0200201 * number, padded with zeros if necessary.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 *
Milan Broz61afef62009-12-10 23:52:25 +0000203 * plain64: the initial vector is the 64-bit little-endian version of the sector
204 * number, padded with zeros if necessary.
205 *
Rik Snel3c164bd2006-09-02 18:17:33 +1000206 * essiv: "encrypted sector|salt initial vector", the sector number is
207 * encrypted with the bulk cipher using a salt as key. The salt
208 * should be derived from the bulk cipher's key via hashing.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 *
Rik Snel48527fa2006-09-03 08:56:39 +1000210 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
211 * (needed for LRW-32-AES and possible other narrow block modes)
212 *
Ludwig Nussel46b47732007-05-09 02:32:55 -0700213 * null: the initial vector is always zero. Provides compatibility with
214 * obsolete loop_fish2 devices. Do not use for new devices.
215 *
Milan Broz34745782011-01-13 19:59:55 +0000216 * lmk: Compatible implementation of the block chaining mode used
217 * by the Loop-AES block device encryption system
218 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
219 * It operates on full 512 byte sectors and uses CBC
220 * with an IV derived from the sector number, the data and
221 * optionally extra IV seed.
222 * This means that after decryption the first block
223 * of sector must be tweaked according to decrypted data.
224 * Loop-AES can use three encryption schemes:
225 * version 1: is plain aes-cbc mode
226 * version 2: uses 64 multikey scheme with lmk IV generator
227 * version 3: the same as version 2 with additional IV seed
228 * (it uses 65 keys, last key is used as IV seed)
229 *
Milan Brozed04d982013-10-28 23:21:04 +0100230 * tcw: Compatible implementation of the block chaining mode used
231 * by the TrueCrypt device encryption system (prior to version 4.1).
Milan Broze44f23b2015-04-05 18:03:10 +0200232 * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
Milan Brozed04d982013-10-28 23:21:04 +0100233 * It operates on full 512 byte sectors and uses CBC
234 * with an IV derived from initial key and the sector number.
235 * In addition, whitening value is applied on every sector, whitening
236 * is calculated from initial key, sector number and mixed using CRC32.
237 * Note that this encryption scheme is vulnerable to watermarking attacks
238 * and should be used for old compatible containers access only.
239 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 * plumb: unimplemented, see:
241 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
242 */
243
Milan Broz2dc53272011-01-13 19:59:54 +0000244static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
245 struct dm_crypt_request *dmreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246{
247 memset(iv, 0, cc->iv_size);
Alasdair G Kergon283a8322011-08-02 12:32:01 +0100248 *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
250 return 0;
251}
252
Milan Broz61afef62009-12-10 23:52:25 +0000253static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
Milan Broz2dc53272011-01-13 19:59:54 +0000254 struct dm_crypt_request *dmreq)
Milan Broz61afef62009-12-10 23:52:25 +0000255{
256 memset(iv, 0, cc->iv_size);
Alasdair G Kergon283a8322011-08-02 12:32:01 +0100257 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
Milan Broz61afef62009-12-10 23:52:25 +0000258
259 return 0;
260}
261
Milan Brozb95bf2d2009-12-10 23:51:56 +0000262/* Initialise ESSIV - compute salt but no local memory allocations */
263static int crypt_iv_essiv_init(struct crypt_config *cc)
264{
265 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
Herbert Xubbdb23b2016-01-24 21:16:36 +0800266 AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000267 struct scatterlist sg;
Andi Kleenc0297722011-01-13 19:59:53 +0000268 struct crypto_cipher *essiv_tfm;
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100269 int err;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000270
271 sg_init_one(&sg, cc->key, cc->key_size);
Herbert Xubbdb23b2016-01-24 21:16:36 +0800272 ahash_request_set_tfm(req, essiv->hash_tfm);
273 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
274 ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000275
Herbert Xubbdb23b2016-01-24 21:16:36 +0800276 err = crypto_ahash_digest(req);
277 ahash_request_zero(req);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000278 if (err)
279 return err;
280
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100281 essiv_tfm = cc->iv_private;
Andi Kleenc0297722011-01-13 19:59:53 +0000282
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100283 err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
Herbert Xubbdb23b2016-01-24 21:16:36 +0800284 crypto_ahash_digestsize(essiv->hash_tfm));
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100285 if (err)
286 return err;
Andi Kleenc0297722011-01-13 19:59:53 +0000287
288 return 0;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000289}
290
Milan Broz542da312009-12-10 23:51:57 +0000291/* Wipe salt and reset key derived from volume key */
292static int crypt_iv_essiv_wipe(struct crypt_config *cc)
293{
294 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
Herbert Xubbdb23b2016-01-24 21:16:36 +0800295 unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm);
Andi Kleenc0297722011-01-13 19:59:53 +0000296 struct crypto_cipher *essiv_tfm;
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100297 int r, err = 0;
Milan Broz542da312009-12-10 23:51:57 +0000298
299 memset(essiv->salt, 0, salt_size);
300
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100301 essiv_tfm = cc->iv_private;
302 r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
303 if (r)
304 err = r;
Andi Kleenc0297722011-01-13 19:59:53 +0000305
306 return err;
307}
308
309/* Set up per cpu cipher state */
310static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
311 struct dm_target *ti,
312 u8 *salt, unsigned saltsize)
313{
314 struct crypto_cipher *essiv_tfm;
315 int err;
316
317 /* Setup the essiv_tfm with the given salt */
318 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
319 if (IS_ERR(essiv_tfm)) {
320 ti->error = "Error allocating crypto tfm for ESSIV";
321 return essiv_tfm;
322 }
323
324 if (crypto_cipher_blocksize(essiv_tfm) !=
Herbert Xubbdb23b2016-01-24 21:16:36 +0800325 crypto_skcipher_ivsize(any_tfm(cc))) {
Andi Kleenc0297722011-01-13 19:59:53 +0000326 ti->error = "Block size of ESSIV cipher does "
327 "not match IV size of block cipher";
328 crypto_free_cipher(essiv_tfm);
329 return ERR_PTR(-EINVAL);
330 }
331
332 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
333 if (err) {
334 ti->error = "Failed to set key for ESSIV cipher";
335 crypto_free_cipher(essiv_tfm);
336 return ERR_PTR(err);
337 }
338
339 return essiv_tfm;
Milan Broz542da312009-12-10 23:51:57 +0000340}
341
Milan Broz60473592009-12-10 23:51:55 +0000342static void crypt_iv_essiv_dtr(struct crypt_config *cc)
343{
Andi Kleenc0297722011-01-13 19:59:53 +0000344 struct crypto_cipher *essiv_tfm;
Milan Broz60473592009-12-10 23:51:55 +0000345 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
346
Herbert Xubbdb23b2016-01-24 21:16:36 +0800347 crypto_free_ahash(essiv->hash_tfm);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000348 essiv->hash_tfm = NULL;
349
350 kzfree(essiv->salt);
351 essiv->salt = NULL;
Andi Kleenc0297722011-01-13 19:59:53 +0000352
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100353 essiv_tfm = cc->iv_private;
Andi Kleenc0297722011-01-13 19:59:53 +0000354
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100355 if (essiv_tfm)
356 crypto_free_cipher(essiv_tfm);
Andi Kleenc0297722011-01-13 19:59:53 +0000357
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100358 cc->iv_private = NULL;
Milan Broz60473592009-12-10 23:51:55 +0000359}
360
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
Milan Brozd469f842007-10-19 22:42:37 +0100362 const char *opts)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363{
Milan Broz5861f1b2009-12-10 23:51:56 +0000364 struct crypto_cipher *essiv_tfm = NULL;
Herbert Xubbdb23b2016-01-24 21:16:36 +0800365 struct crypto_ahash *hash_tfm = NULL;
Milan Broz5861f1b2009-12-10 23:51:56 +0000366 u8 *salt = NULL;
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100367 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
Milan Broz5861f1b2009-12-10 23:51:56 +0000369 if (!opts) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700370 ti->error = "Digest algorithm missing for ESSIV mode";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 return -EINVAL;
372 }
373
Milan Brozb95bf2d2009-12-10 23:51:56 +0000374 /* Allocate hash algorithm */
Herbert Xubbdb23b2016-01-24 21:16:36 +0800375 hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC);
Herbert Xu35058682006-08-24 19:10:20 +1000376 if (IS_ERR(hash_tfm)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700377 ti->error = "Error initializing ESSIV hash";
Milan Broz5861f1b2009-12-10 23:51:56 +0000378 err = PTR_ERR(hash_tfm);
379 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 }
381
Herbert Xubbdb23b2016-01-24 21:16:36 +0800382 salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL);
Milan Broz5861f1b2009-12-10 23:51:56 +0000383 if (!salt) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700384 ti->error = "Error kmallocing salt storage in ESSIV";
Milan Broz5861f1b2009-12-10 23:51:56 +0000385 err = -ENOMEM;
386 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 }
388
Milan Brozb95bf2d2009-12-10 23:51:56 +0000389 cc->iv_gen_private.essiv.salt = salt;
Milan Brozb95bf2d2009-12-10 23:51:56 +0000390 cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
391
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100392 essiv_tfm = setup_essiv_cpu(cc, ti, salt,
Herbert Xubbdb23b2016-01-24 21:16:36 +0800393 crypto_ahash_digestsize(hash_tfm));
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100394 if (IS_ERR(essiv_tfm)) {
395 crypt_iv_essiv_dtr(cc);
396 return PTR_ERR(essiv_tfm);
Andi Kleenc0297722011-01-13 19:59:53 +0000397 }
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100398 cc->iv_private = essiv_tfm;
Andi Kleenc0297722011-01-13 19:59:53 +0000399
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 return 0;
Milan Broz5861f1b2009-12-10 23:51:56 +0000401
402bad:
Milan Broz5861f1b2009-12-10 23:51:56 +0000403 if (hash_tfm && !IS_ERR(hash_tfm))
Herbert Xubbdb23b2016-01-24 21:16:36 +0800404 crypto_free_ahash(hash_tfm);
Milan Brozb95bf2d2009-12-10 23:51:56 +0000405 kfree(salt);
Milan Broz5861f1b2009-12-10 23:51:56 +0000406 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407}
408
Milan Broz2dc53272011-01-13 19:59:54 +0000409static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
410 struct dm_crypt_request *dmreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411{
Mikulas Patockafd2d2312012-07-27 15:08:05 +0100412 struct crypto_cipher *essiv_tfm = cc->iv_private;
Andi Kleenc0297722011-01-13 19:59:53 +0000413
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 memset(iv, 0, cc->iv_size);
Alasdair G Kergon283a8322011-08-02 12:32:01 +0100415 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
Andi Kleenc0297722011-01-13 19:59:53 +0000416 crypto_cipher_encrypt_one(essiv_tfm, iv, iv);
417
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 return 0;
419}
420
Rik Snel48527fa2006-09-03 08:56:39 +1000421static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
422 const char *opts)
423{
Herbert Xubbdb23b2016-01-24 21:16:36 +0800424 unsigned bs = crypto_skcipher_blocksize(any_tfm(cc));
David Howellsf0d1b0b2006-12-08 02:37:49 -0800425 int log = ilog2(bs);
Rik Snel48527fa2006-09-03 08:56:39 +1000426
427 /* we need to calculate how far we must shift the sector count
428 * to get the cipher block count, we use this shift in _gen */
429
430 if (1 << log != bs) {
431 ti->error = "cypher blocksize is not a power of 2";
432 return -EINVAL;
433 }
434
435 if (log > 9) {
436 ti->error = "cypher blocksize is > 512";
437 return -EINVAL;
438 }
439
Milan Broz60473592009-12-10 23:51:55 +0000440 cc->iv_gen_private.benbi.shift = 9 - log;
Rik Snel48527fa2006-09-03 08:56:39 +1000441
442 return 0;
443}
444
445static void crypt_iv_benbi_dtr(struct crypt_config *cc)
446{
Rik Snel48527fa2006-09-03 08:56:39 +1000447}
448
Milan Broz2dc53272011-01-13 19:59:54 +0000449static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
450 struct dm_crypt_request *dmreq)
Rik Snel48527fa2006-09-03 08:56:39 +1000451{
Herbert Xu79066ad2006-12-05 13:41:52 -0800452 __be64 val;
453
Rik Snel48527fa2006-09-03 08:56:39 +1000454 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
Herbert Xu79066ad2006-12-05 13:41:52 -0800455
Milan Broz2dc53272011-01-13 19:59:54 +0000456 val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
Herbert Xu79066ad2006-12-05 13:41:52 -0800457 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
Rik Snel48527fa2006-09-03 08:56:39 +1000458
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 return 0;
460}
461
Milan Broz2dc53272011-01-13 19:59:54 +0000462static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
463 struct dm_crypt_request *dmreq)
Ludwig Nussel46b47732007-05-09 02:32:55 -0700464{
465 memset(iv, 0, cc->iv_size);
466
467 return 0;
468}
469
Milan Broz34745782011-01-13 19:59:55 +0000470static void crypt_iv_lmk_dtr(struct crypt_config *cc)
471{
472 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
473
474 if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
475 crypto_free_shash(lmk->hash_tfm);
476 lmk->hash_tfm = NULL;
477
478 kzfree(lmk->seed);
479 lmk->seed = NULL;
480}
481
482static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
483 const char *opts)
484{
485 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
486
487 lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0);
488 if (IS_ERR(lmk->hash_tfm)) {
489 ti->error = "Error initializing LMK hash";
490 return PTR_ERR(lmk->hash_tfm);
491 }
492
493 /* No seed in LMK version 2 */
494 if (cc->key_parts == cc->tfms_count) {
495 lmk->seed = NULL;
496 return 0;
497 }
498
499 lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
500 if (!lmk->seed) {
501 crypt_iv_lmk_dtr(cc);
502 ti->error = "Error kmallocing seed storage in LMK";
503 return -ENOMEM;
504 }
505
506 return 0;
507}
508
509static int crypt_iv_lmk_init(struct crypt_config *cc)
510{
511 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
512 int subkey_size = cc->key_size / cc->key_parts;
513
514 /* LMK seed is on the position of LMK_KEYS + 1 key */
515 if (lmk->seed)
516 memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
517 crypto_shash_digestsize(lmk->hash_tfm));
518
519 return 0;
520}
521
522static int crypt_iv_lmk_wipe(struct crypt_config *cc)
523{
524 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
525
526 if (lmk->seed)
527 memset(lmk->seed, 0, LMK_SEED_SIZE);
528
529 return 0;
530}
531
532static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
533 struct dm_crypt_request *dmreq,
534 u8 *data)
535{
536 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200537 SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
Milan Broz34745782011-01-13 19:59:55 +0000538 struct md5_state md5state;
Milan Brozda31a072013-10-28 23:21:03 +0100539 __le32 buf[4];
Milan Broz34745782011-01-13 19:59:55 +0000540 int i, r;
541
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200542 desc->tfm = lmk->hash_tfm;
543 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Milan Broz34745782011-01-13 19:59:55 +0000544
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200545 r = crypto_shash_init(desc);
Milan Broz34745782011-01-13 19:59:55 +0000546 if (r)
547 return r;
548
549 if (lmk->seed) {
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200550 r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);
Milan Broz34745782011-01-13 19:59:55 +0000551 if (r)
552 return r;
553 }
554
555 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200556 r = crypto_shash_update(desc, data + 16, 16 * 31);
Milan Broz34745782011-01-13 19:59:55 +0000557 if (r)
558 return r;
559
560 /* Sector is cropped to 56 bits here */
561 buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
562 buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
563 buf[2] = cpu_to_le32(4024);
564 buf[3] = 0;
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200565 r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
Milan Broz34745782011-01-13 19:59:55 +0000566 if (r)
567 return r;
568
569 /* No MD5 padding here */
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200570 r = crypto_shash_export(desc, &md5state);
Milan Broz34745782011-01-13 19:59:55 +0000571 if (r)
572 return r;
573
574 for (i = 0; i < MD5_HASH_WORDS; i++)
575 __cpu_to_le32s(&md5state.hash[i]);
576 memcpy(iv, &md5state.hash, cc->iv_size);
577
578 return 0;
579}
580
581static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
582 struct dm_crypt_request *dmreq)
583{
584 u8 *src;
585 int r = 0;
586
587 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
Cong Wangc2e022c2011-11-28 13:26:02 +0800588 src = kmap_atomic(sg_page(&dmreq->sg_in));
Milan Broz34745782011-01-13 19:59:55 +0000589 r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset);
Cong Wangc2e022c2011-11-28 13:26:02 +0800590 kunmap_atomic(src);
Milan Broz34745782011-01-13 19:59:55 +0000591 } else
592 memset(iv, 0, cc->iv_size);
593
594 return r;
595}
596
597static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
598 struct dm_crypt_request *dmreq)
599{
600 u8 *dst;
601 int r;
602
603 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
604 return 0;
605
Cong Wangc2e022c2011-11-28 13:26:02 +0800606 dst = kmap_atomic(sg_page(&dmreq->sg_out));
Milan Broz34745782011-01-13 19:59:55 +0000607 r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset);
608
609 /* Tweak the first block of plaintext sector */
610 if (!r)
611 crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size);
612
Cong Wangc2e022c2011-11-28 13:26:02 +0800613 kunmap_atomic(dst);
Milan Broz34745782011-01-13 19:59:55 +0000614 return r;
615}
616
Milan Brozed04d982013-10-28 23:21:04 +0100617static void crypt_iv_tcw_dtr(struct crypt_config *cc)
618{
619 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
620
621 kzfree(tcw->iv_seed);
622 tcw->iv_seed = NULL;
623 kzfree(tcw->whitening);
624 tcw->whitening = NULL;
625
626 if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
627 crypto_free_shash(tcw->crc32_tfm);
628 tcw->crc32_tfm = NULL;
629}
630
631static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
632 const char *opts)
633{
634 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
635
636 if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
637 ti->error = "Wrong key size for TCW";
638 return -EINVAL;
639 }
640
641 tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0);
642 if (IS_ERR(tcw->crc32_tfm)) {
643 ti->error = "Error initializing CRC32 in TCW";
644 return PTR_ERR(tcw->crc32_tfm);
645 }
646
647 tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
648 tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
649 if (!tcw->iv_seed || !tcw->whitening) {
650 crypt_iv_tcw_dtr(cc);
651 ti->error = "Error allocating seed storage in TCW";
652 return -ENOMEM;
653 }
654
655 return 0;
656}
657
658static int crypt_iv_tcw_init(struct crypt_config *cc)
659{
660 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
661 int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
662
663 memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
664 memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
665 TCW_WHITENING_SIZE);
666
667 return 0;
668}
669
670static int crypt_iv_tcw_wipe(struct crypt_config *cc)
671{
672 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
673
674 memset(tcw->iv_seed, 0, cc->iv_size);
675 memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
676
677 return 0;
678}
679
680static int crypt_iv_tcw_whitening(struct crypt_config *cc,
681 struct dm_crypt_request *dmreq,
682 u8 *data)
683{
684 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
Bart Van Assche350b5392016-06-28 16:32:32 +0200685 __le64 sector = cpu_to_le64(dmreq->iv_sector);
Milan Brozed04d982013-10-28 23:21:04 +0100686 u8 buf[TCW_WHITENING_SIZE];
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200687 SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
Milan Brozed04d982013-10-28 23:21:04 +0100688 int i, r;
689
690 /* xor whitening with sector number */
691 memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE);
692 crypto_xor(buf, (u8 *)&sector, 8);
693 crypto_xor(&buf[8], (u8 *)&sector, 8);
694
695 /* calculate crc32 for every 32bit part and xor it */
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200696 desc->tfm = tcw->crc32_tfm;
697 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
Milan Brozed04d982013-10-28 23:21:04 +0100698 for (i = 0; i < 4; i++) {
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200699 r = crypto_shash_init(desc);
Milan Brozed04d982013-10-28 23:21:04 +0100700 if (r)
701 goto out;
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200702 r = crypto_shash_update(desc, &buf[i * 4], 4);
Milan Brozed04d982013-10-28 23:21:04 +0100703 if (r)
704 goto out;
Jan-Simon Möllerb6106262012-07-02 13:50:54 +0200705 r = crypto_shash_final(desc, &buf[i * 4]);
Milan Brozed04d982013-10-28 23:21:04 +0100706 if (r)
707 goto out;
708 }
709 crypto_xor(&buf[0], &buf[12], 4);
710 crypto_xor(&buf[4], &buf[8], 4);
711
712 /* apply whitening (8 bytes) to whole sector */
713 for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
714 crypto_xor(data + i * 8, buf, 8);
715out:
Milan Broz1a71d6f2014-11-22 09:36:04 +0100716 memzero_explicit(buf, sizeof(buf));
Milan Brozed04d982013-10-28 23:21:04 +0100717 return r;
718}
719
720static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
721 struct dm_crypt_request *dmreq)
722{
723 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
Bart Van Assche350b5392016-06-28 16:32:32 +0200724 __le64 sector = cpu_to_le64(dmreq->iv_sector);
Milan Brozed04d982013-10-28 23:21:04 +0100725 u8 *src;
726 int r = 0;
727
728 /* Remove whitening from ciphertext */
729 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
730 src = kmap_atomic(sg_page(&dmreq->sg_in));
731 r = crypt_iv_tcw_whitening(cc, dmreq, src + dmreq->sg_in.offset);
732 kunmap_atomic(src);
733 }
734
735 /* Calculate IV */
736 memcpy(iv, tcw->iv_seed, cc->iv_size);
737 crypto_xor(iv, (u8 *)&sector, 8);
738 if (cc->iv_size > 8)
739 crypto_xor(&iv[8], (u8 *)&sector, cc->iv_size - 8);
740
741 return r;
742}
743
744static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
745 struct dm_crypt_request *dmreq)
746{
747 u8 *dst;
748 int r;
749
750 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
751 return 0;
752
753 /* Apply whitening on ciphertext */
754 dst = kmap_atomic(sg_page(&dmreq->sg_out));
755 r = crypt_iv_tcw_whitening(cc, dmreq, dst + dmreq->sg_out.offset);
756 kunmap_atomic(dst);
757
758 return r;
759}
760
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761static struct crypt_iv_operations crypt_iv_plain_ops = {
762 .generator = crypt_iv_plain_gen
763};
764
Milan Broz61afef62009-12-10 23:52:25 +0000765static struct crypt_iv_operations crypt_iv_plain64_ops = {
766 .generator = crypt_iv_plain64_gen
767};
768
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769static struct crypt_iv_operations crypt_iv_essiv_ops = {
770 .ctr = crypt_iv_essiv_ctr,
771 .dtr = crypt_iv_essiv_dtr,
Milan Brozb95bf2d2009-12-10 23:51:56 +0000772 .init = crypt_iv_essiv_init,
Milan Broz542da312009-12-10 23:51:57 +0000773 .wipe = crypt_iv_essiv_wipe,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 .generator = crypt_iv_essiv_gen
775};
776
Rik Snel48527fa2006-09-03 08:56:39 +1000777static struct crypt_iv_operations crypt_iv_benbi_ops = {
778 .ctr = crypt_iv_benbi_ctr,
779 .dtr = crypt_iv_benbi_dtr,
780 .generator = crypt_iv_benbi_gen
781};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782
Ludwig Nussel46b47732007-05-09 02:32:55 -0700783static struct crypt_iv_operations crypt_iv_null_ops = {
784 .generator = crypt_iv_null_gen
785};
786
Milan Broz34745782011-01-13 19:59:55 +0000787static struct crypt_iv_operations crypt_iv_lmk_ops = {
788 .ctr = crypt_iv_lmk_ctr,
789 .dtr = crypt_iv_lmk_dtr,
790 .init = crypt_iv_lmk_init,
791 .wipe = crypt_iv_lmk_wipe,
792 .generator = crypt_iv_lmk_gen,
793 .post = crypt_iv_lmk_post
794};
795
Milan Brozed04d982013-10-28 23:21:04 +0100796static struct crypt_iv_operations crypt_iv_tcw_ops = {
797 .ctr = crypt_iv_tcw_ctr,
798 .dtr = crypt_iv_tcw_dtr,
799 .init = crypt_iv_tcw_init,
800 .wipe = crypt_iv_tcw_wipe,
801 .generator = crypt_iv_tcw_gen,
802 .post = crypt_iv_tcw_post
803};
804
Milan Brozd469f842007-10-19 22:42:37 +0100805static void crypt_convert_init(struct crypt_config *cc,
806 struct convert_context *ctx,
807 struct bio *bio_out, struct bio *bio_in,
Milan Brozfcd369d2008-02-08 02:10:41 +0000808 sector_t sector)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809{
810 ctx->bio_in = bio_in;
811 ctx->bio_out = bio_out;
Kent Overstreet003b5c52013-10-11 15:45:43 -0700812 if (bio_in)
813 ctx->iter_in = bio_in->bi_iter;
814 if (bio_out)
815 ctx->iter_out = bio_out->bi_iter;
Mikulas Patockac66029f2012-07-27 15:08:05 +0100816 ctx->cc_sector = sector + cc->iv_offset;
Milan Broz43d69032008-02-08 02:11:09 +0000817 init_completion(&ctx->restart);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818}
819
Huang Yingb2174ee2009-03-16 17:44:33 +0000820static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
Herbert Xubbdb23b2016-01-24 21:16:36 +0800821 struct skcipher_request *req)
Huang Yingb2174ee2009-03-16 17:44:33 +0000822{
823 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
824}
825
Herbert Xubbdb23b2016-01-24 21:16:36 +0800826static struct skcipher_request *req_of_dmreq(struct crypt_config *cc,
Huang Yingb2174ee2009-03-16 17:44:33 +0000827 struct dm_crypt_request *dmreq)
828{
Herbert Xubbdb23b2016-01-24 21:16:36 +0800829 return (struct skcipher_request *)((char *)dmreq - cc->dmreq_start);
Huang Yingb2174ee2009-03-16 17:44:33 +0000830}
831
Milan Broz2dc53272011-01-13 19:59:54 +0000832static u8 *iv_of_dmreq(struct crypt_config *cc,
833 struct dm_crypt_request *dmreq)
834{
835 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
Herbert Xubbdb23b2016-01-24 21:16:36 +0800836 crypto_skcipher_alignmask(any_tfm(cc)) + 1);
Milan Broz2dc53272011-01-13 19:59:54 +0000837}
838
Milan Broz01482b72008-02-08 02:11:04 +0000839static int crypt_convert_block(struct crypt_config *cc,
Milan Broz3a7f6c92008-02-08 02:11:14 +0000840 struct convert_context *ctx,
Herbert Xubbdb23b2016-01-24 21:16:36 +0800841 struct skcipher_request *req)
Milan Broz01482b72008-02-08 02:11:04 +0000842{
Kent Overstreet003b5c52013-10-11 15:45:43 -0700843 struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
844 struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000845 struct dm_crypt_request *dmreq;
846 u8 *iv;
Mikulas Patocka40b62292012-07-27 15:08:04 +0100847 int r;
Milan Broz01482b72008-02-08 02:11:04 +0000848
Huang Yingb2174ee2009-03-16 17:44:33 +0000849 dmreq = dmreq_of_req(cc, req);
Milan Broz2dc53272011-01-13 19:59:54 +0000850 iv = iv_of_dmreq(cc, dmreq);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000851
Mikulas Patockac66029f2012-07-27 15:08:05 +0100852 dmreq->iv_sector = ctx->cc_sector;
Huang Yingb2174ee2009-03-16 17:44:33 +0000853 dmreq->ctx = ctx;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000854 sg_init_table(&dmreq->sg_in, 1);
Kent Overstreet003b5c52013-10-11 15:45:43 -0700855 sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT,
856 bv_in.bv_offset);
Milan Broz01482b72008-02-08 02:11:04 +0000857
Milan Broz3a7f6c92008-02-08 02:11:14 +0000858 sg_init_table(&dmreq->sg_out, 1);
Kent Overstreet003b5c52013-10-11 15:45:43 -0700859 sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT,
860 bv_out.bv_offset);
Milan Broz01482b72008-02-08 02:11:04 +0000861
Kent Overstreet003b5c52013-10-11 15:45:43 -0700862 bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT);
863 bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT);
Milan Broz01482b72008-02-08 02:11:04 +0000864
Milan Broz3a7f6c92008-02-08 02:11:14 +0000865 if (cc->iv_gen_ops) {
Milan Broz2dc53272011-01-13 19:59:54 +0000866 r = cc->iv_gen_ops->generator(cc, iv, dmreq);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000867 if (r < 0)
868 return r;
869 }
870
Herbert Xubbdb23b2016-01-24 21:16:36 +0800871 skcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
872 1 << SECTOR_SHIFT, iv);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000873
874 if (bio_data_dir(ctx->bio_in) == WRITE)
Herbert Xubbdb23b2016-01-24 21:16:36 +0800875 r = crypto_skcipher_encrypt(req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000876 else
Herbert Xubbdb23b2016-01-24 21:16:36 +0800877 r = crypto_skcipher_decrypt(req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000878
Milan Broz2dc53272011-01-13 19:59:54 +0000879 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
880 r = cc->iv_gen_ops->post(cc, iv, dmreq);
881
Milan Broz3a7f6c92008-02-08 02:11:14 +0000882 return r;
Milan Broz01482b72008-02-08 02:11:04 +0000883}
884
Milan Broz95497a92008-02-08 02:11:12 +0000885static void kcryptd_async_done(struct crypto_async_request *async_req,
886 int error);
Andi Kleenc0297722011-01-13 19:59:53 +0000887
Milan Brozddd42ed2008-02-08 02:11:07 +0000888static void crypt_alloc_req(struct crypt_config *cc,
889 struct convert_context *ctx)
890{
Mikulas Patockac66029f2012-07-27 15:08:05 +0100891 unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
Andi Kleenc0297722011-01-13 19:59:53 +0000892
Mikulas Patocka610f2de2014-02-20 18:01:01 -0500893 if (!ctx->req)
894 ctx->req = mempool_alloc(cc->req_pool, GFP_NOIO);
Andi Kleenc0297722011-01-13 19:59:53 +0000895
Herbert Xubbdb23b2016-01-24 21:16:36 +0800896 skcipher_request_set_tfm(ctx->req, cc->tfms[key_index]);
Milan Broz54cea3f2015-05-15 17:00:25 +0200897
898 /*
899 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
900 * requests if driver request queue is full.
901 */
Herbert Xubbdb23b2016-01-24 21:16:36 +0800902 skcipher_request_set_callback(ctx->req,
Andi Kleenc0297722011-01-13 19:59:53 +0000903 CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
Mikulas Patocka610f2de2014-02-20 18:01:01 -0500904 kcryptd_async_done, dmreq_of_req(cc, ctx->req));
Milan Brozddd42ed2008-02-08 02:11:07 +0000905}
906
Mikulas Patocka298a9fa2014-03-28 15:51:55 -0400907static void crypt_free_req(struct crypt_config *cc,
Herbert Xubbdb23b2016-01-24 21:16:36 +0800908 struct skcipher_request *req, struct bio *base_bio)
Mikulas Patocka298a9fa2014-03-28 15:51:55 -0400909{
910 struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
911
Herbert Xubbdb23b2016-01-24 21:16:36 +0800912 if ((struct skcipher_request *)(io + 1) != req)
Mikulas Patocka298a9fa2014-03-28 15:51:55 -0400913 mempool_free(req, cc->req_pool);
914}
915
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916/*
917 * Encrypt / decrypt data from one bio to another one (can be the same one)
918 */
919static int crypt_convert(struct crypt_config *cc,
Milan Brozd469f842007-10-19 22:42:37 +0100920 struct convert_context *ctx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921{
Milan Broz3f1e9072008-03-28 14:16:07 -0700922 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923
Mikulas Patocka40b62292012-07-27 15:08:04 +0100924 atomic_set(&ctx->cc_pending, 1);
Milan Brozc8081612008-10-10 13:37:08 +0100925
Kent Overstreet003b5c52013-10-11 15:45:43 -0700926 while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927
Milan Broz3a7f6c92008-02-08 02:11:14 +0000928 crypt_alloc_req(cc, ctx);
929
Mikulas Patocka40b62292012-07-27 15:08:04 +0100930 atomic_inc(&ctx->cc_pending);
Milan Broz3f1e9072008-03-28 14:16:07 -0700931
Mikulas Patocka610f2de2014-02-20 18:01:01 -0500932 r = crypt_convert_block(cc, ctx, ctx->req);
Milan Broz3a7f6c92008-02-08 02:11:14 +0000933
934 switch (r) {
Milan Broz54cea3f2015-05-15 17:00:25 +0200935 /*
936 * The request was queued by a crypto driver
937 * but the driver request queue is full, let's wait.
938 */
Milan Broz3a7f6c92008-02-08 02:11:14 +0000939 case -EBUSY:
940 wait_for_completion(&ctx->restart);
Wolfram Sang16735d02013-11-14 14:32:02 -0800941 reinit_completion(&ctx->restart);
Milan Broz54cea3f2015-05-15 17:00:25 +0200942 /* fall through */
943 /*
944 * The request is queued and processed asynchronously,
945 * completion function kcryptd_async_done() will be called.
946 */
Rabin Vincentc0403ec2015-05-05 15:15:56 +0200947 case -EINPROGRESS:
Mikulas Patocka610f2de2014-02-20 18:01:01 -0500948 ctx->req = NULL;
Mikulas Patockac66029f2012-07-27 15:08:05 +0100949 ctx->cc_sector++;
Milan Broz3a7f6c92008-02-08 02:11:14 +0000950 continue;
Milan Broz54cea3f2015-05-15 17:00:25 +0200951 /*
952 * The request was already processed (synchronously).
953 */
Milan Broz3f1e9072008-03-28 14:16:07 -0700954 case 0:
Mikulas Patocka40b62292012-07-27 15:08:04 +0100955 atomic_dec(&ctx->cc_pending);
Mikulas Patockac66029f2012-07-27 15:08:05 +0100956 ctx->cc_sector++;
Milan Brozc7f1b202008-07-02 09:34:28 +0100957 cond_resched();
Milan Broz3f1e9072008-03-28 14:16:07 -0700958 continue;
959
Milan Broz54cea3f2015-05-15 17:00:25 +0200960 /* There was an error while processing the request. */
Milan Broz3f1e9072008-03-28 14:16:07 -0700961 default:
Mikulas Patocka40b62292012-07-27 15:08:04 +0100962 atomic_dec(&ctx->cc_pending);
Milan Broz3f1e9072008-03-28 14:16:07 -0700963 return r;
964 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 }
966
Milan Broz3f1e9072008-03-28 14:16:07 -0700967 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968}
969
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -0500970static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
971
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972/*
973 * Generate a new unfragmented bio with the given size
Mike Snitzer586b2862015-09-09 21:34:51 -0400974 * This should never violate the device limitations (but only because
975 * max_segment_size is being constrained to PAGE_SIZE).
Mikulas Patocka7145c242015-02-13 08:24:41 -0500976 *
977 * This function may be called concurrently. If we allocate from the mempool
978 * concurrently, there is a possibility of deadlock. For example, if we have
979 * mempool of 256 pages, two processes, each wanting 256, pages allocate from
980 * the mempool concurrently, it may deadlock in a situation where both processes
981 * have allocated 128 pages and the mempool is exhausted.
982 *
983 * In order to avoid this scenario we allocate the pages under a mutex.
984 *
985 * In order to not degrade performance with excessive locking, we try
986 * non-blocking allocations without a mutex first but on failure we fallback
987 * to blocking allocations with a mutex.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 */
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -0500989static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +0100991 struct crypt_config *cc = io->cc;
Milan Broz8b004452006-10-03 01:15:37 -0700992 struct bio *clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Mikulas Patocka7145c242015-02-13 08:24:41 -0500994 gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
995 unsigned i, len, remaining_size;
Milan Broz91e10622007-12-13 14:16:10 +0000996 struct page *page;
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -0500997 struct bio_vec *bvec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998
Mikulas Patocka7145c242015-02-13 08:24:41 -0500999retry:
Mel Gormand0164ad2015-11-06 16:28:21 -08001000 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
Mikulas Patocka7145c242015-02-13 08:24:41 -05001001 mutex_lock(&cc->bio_alloc_lock);
1002
Olaf Kirch2f9941b2007-05-09 02:32:53 -07001003 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
Milan Broz8b004452006-10-03 01:15:37 -07001004 if (!clone)
Mikulas Patocka7145c242015-02-13 08:24:41 -05001005 goto return_clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
Olaf Kirch027581f2007-05-09 02:32:52 -07001007 clone_init(io, clone);
Milan Broz6a24c712006-10-03 01:15:40 -07001008
Mikulas Patocka7145c242015-02-13 08:24:41 -05001009 remaining_size = size;
1010
Olaf Kirchf97380b2007-05-09 02:32:54 -07001011 for (i = 0; i < nr_iovecs; i++) {
Milan Broz91e10622007-12-13 14:16:10 +00001012 page = mempool_alloc(cc->page_pool, gfp_mask);
Mikulas Patocka7145c242015-02-13 08:24:41 -05001013 if (!page) {
1014 crypt_free_buffer_pages(cc, clone);
1015 bio_put(clone);
Mel Gormand0164ad2015-11-06 16:28:21 -08001016 gfp_mask |= __GFP_DIRECT_RECLAIM;
Mikulas Patocka7145c242015-02-13 08:24:41 -05001017 goto retry;
1018 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019
Mikulas Patocka7145c242015-02-13 08:24:41 -05001020 len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -05001022 bvec = &clone->bi_io_vec[clone->bi_vcnt++];
1023 bvec->bv_page = page;
1024 bvec->bv_len = len;
1025 bvec->bv_offset = 0;
1026
1027 clone->bi_iter.bi_size += len;
Milan Broz91e10622007-12-13 14:16:10 +00001028
Mikulas Patocka7145c242015-02-13 08:24:41 -05001029 remaining_size -= len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030 }
1031
Mikulas Patocka7145c242015-02-13 08:24:41 -05001032return_clone:
Mel Gormand0164ad2015-11-06 16:28:21 -08001033 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
Mikulas Patocka7145c242015-02-13 08:24:41 -05001034 mutex_unlock(&cc->bio_alloc_lock);
1035
Milan Broz8b004452006-10-03 01:15:37 -07001036 return clone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037}
1038
Neil Brown644bd2f2007-10-16 13:48:46 +02001039static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040{
Neil Brown644bd2f2007-10-16 13:48:46 +02001041 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042 struct bio_vec *bv;
1043
Kent Overstreetcb34e052012-09-05 15:22:02 -07001044 bio_for_each_segment_all(bv, clone, i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 BUG_ON(!bv->bv_page);
1046 mempool_free(bv->bv_page, cc->page_pool);
1047 bv->bv_page = NULL;
1048 }
1049}
1050
Mikulas Patocka298a9fa2014-03-28 15:51:55 -04001051static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1052 struct bio *bio, sector_t sector)
Milan Brozdc440d1e2008-10-10 13:37:03 +01001053{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001054 io->cc = cc;
Milan Brozdc440d1e2008-10-10 13:37:03 +01001055 io->base_bio = bio;
1056 io->sector = sector;
1057 io->error = 0;
Mikulas Patocka610f2de2014-02-20 18:01:01 -05001058 io->ctx.req = NULL;
Mikulas Patocka40b62292012-07-27 15:08:04 +01001059 atomic_set(&io->io_pending, 0);
Milan Brozdc440d1e2008-10-10 13:37:03 +01001060}
1061
Milan Broz3e1a8bd2008-10-10 13:37:02 +01001062static void crypt_inc_pending(struct dm_crypt_io *io)
1063{
Mikulas Patocka40b62292012-07-27 15:08:04 +01001064 atomic_inc(&io->io_pending);
Milan Broz3e1a8bd2008-10-10 13:37:02 +01001065}
1066
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067/*
1068 * One of the bios was finished. Check for completion of
1069 * the whole request and correctly clean up the buffer.
1070 */
Milan Broz5742fd72008-02-08 02:10:43 +00001071static void crypt_dec_pending(struct dm_crypt_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001073 struct crypt_config *cc = io->cc;
Milan Brozb35f8ca2009-03-16 17:44:36 +00001074 struct bio *base_bio = io->base_bio;
Milan Brozb35f8ca2009-03-16 17:44:36 +00001075 int error = io->error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
Mikulas Patocka40b62292012-07-27 15:08:04 +01001077 if (!atomic_dec_and_test(&io->io_pending))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078 return;
1079
Mikulas Patocka610f2de2014-02-20 18:01:01 -05001080 if (io->ctx.req)
Mikulas Patocka298a9fa2014-03-28 15:51:55 -04001081 crypt_free_req(cc, io->ctx.req, base_bio);
Milan Brozb35f8ca2009-03-16 17:44:36 +00001082
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001083 base_bio->bi_error = error;
1084 bio_endio(base_bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085}
1086
1087/*
Milan Brozcabf08e2007-10-19 22:38:58 +01001088 * kcryptd/kcryptd_io:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 *
1090 * Needed because it would be very unwise to do decryption in an
Milan Broz23541d22006-10-03 01:15:39 -07001091 * interrupt context.
Milan Brozcabf08e2007-10-19 22:38:58 +01001092 *
1093 * kcryptd performs the actual encryption or decryption.
1094 *
1095 * kcryptd_io performs the IO submission.
1096 *
1097 * They must be separated as otherwise the final stages could be
1098 * starved by new requests which can block in the first stages due
1099 * to memory allocation.
Andi Kleenc0297722011-01-13 19:59:53 +00001100 *
1101 * The work is done per CPU global for all dm-crypt instances.
1102 * They should not depend on each other and do not block.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001104static void crypt_endio(struct bio *clone)
Milan Broz8b004452006-10-03 01:15:37 -07001105{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001106 struct dm_crypt_io *io = clone->bi_private;
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001107 struct crypt_config *cc = io->cc;
Milan Brozee7a4912008-02-08 02:10:46 +00001108 unsigned rw = bio_data_dir(clone);
Sasha Levin9b81c842015-08-10 19:05:18 -04001109 int error;
Milan Broz8b004452006-10-03 01:15:37 -07001110
1111 /*
NeilBrown6712ecf2007-09-27 12:47:43 +02001112 * free the processed pages
Milan Broz8b004452006-10-03 01:15:37 -07001113 */
Milan Brozee7a4912008-02-08 02:10:46 +00001114 if (rw == WRITE)
Neil Brown644bd2f2007-10-16 13:48:46 +02001115 crypt_free_buffer_pages(cc, clone);
Milan Brozee7a4912008-02-08 02:10:46 +00001116
Sasha Levin9b81c842015-08-10 19:05:18 -04001117 error = clone->bi_error;
Milan Brozee7a4912008-02-08 02:10:46 +00001118 bio_put(clone);
1119
Sasha Levin9b81c842015-08-10 19:05:18 -04001120 if (rw == READ && !error) {
Milan Brozee7a4912008-02-08 02:10:46 +00001121 kcryptd_queue_crypt(io);
1122 return;
NeilBrown6712ecf2007-09-27 12:47:43 +02001123 }
Milan Broz8b004452006-10-03 01:15:37 -07001124
Sasha Levin9b81c842015-08-10 19:05:18 -04001125 if (unlikely(error))
1126 io->error = error;
Milan Broz5742fd72008-02-08 02:10:43 +00001127
1128 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -07001129}
1130
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001131static void clone_init(struct dm_crypt_io *io, struct bio *clone)
Milan Broz8b004452006-10-03 01:15:37 -07001132{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001133 struct crypt_config *cc = io->cc;
Milan Broz8b004452006-10-03 01:15:37 -07001134
1135 clone->bi_private = io;
1136 clone->bi_end_io = crypt_endio;
1137 clone->bi_bdev = cc->dev->bdev;
Bart Van Assche4382e332016-09-14 10:45:36 +02001138 bio_set_op_attrs(clone, bio_op(io->base_bio), bio_flags(io->base_bio));
Milan Broz8b004452006-10-03 01:15:37 -07001139}
1140
Milan Broz20c82532011-01-13 19:59:53 +00001141static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
Milan Broz8b004452006-10-03 01:15:37 -07001142{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001143 struct crypt_config *cc = io->cc;
Milan Broz8b004452006-10-03 01:15:37 -07001144 struct bio *clone;
Milan Broz93e605c2006-10-03 01:15:38 -07001145
Milan Broz8b004452006-10-03 01:15:37 -07001146 /*
Mike Snitzer59779072015-04-09 16:53:24 -04001147 * We need the original biovec array in order to decrypt
1148 * the whole bio data *afterwards* -- thanks to immutable
1149 * biovecs we don't need to worry about the block layer
1150 * modifying the biovec array; so leverage bio_clone_fast().
Milan Broz8b004452006-10-03 01:15:37 -07001151 */
Mike Snitzer59779072015-04-09 16:53:24 -04001152 clone = bio_clone_fast(io->base_bio, gfp, cc->bs);
Jens Axboe7eaceac2011-03-10 08:52:07 +01001153 if (!clone)
Milan Broz20c82532011-01-13 19:59:53 +00001154 return 1;
Milan Broz8b004452006-10-03 01:15:37 -07001155
Milan Broz20c82532011-01-13 19:59:53 +00001156 crypt_inc_pending(io);
1157
Milan Broz8b004452006-10-03 01:15:37 -07001158 clone_init(io, clone);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001159 clone->bi_iter.bi_sector = cc->start + io->sector;
Milan Broz8b004452006-10-03 01:15:37 -07001160
Milan Broz93e605c2006-10-03 01:15:38 -07001161 generic_make_request(clone);
Milan Broz20c82532011-01-13 19:59:53 +00001162 return 0;
Milan Broz8b004452006-10-03 01:15:37 -07001163}
1164
Mikulas Patockadc267622015-02-13 08:25:59 -05001165static void kcryptd_io_read_work(struct work_struct *work)
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001166{
1167 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1168
Mikulas Patockadc267622015-02-13 08:25:59 -05001169 crypt_inc_pending(io);
1170 if (kcryptd_io_read(io, GFP_NOIO))
1171 io->error = -ENOMEM;
1172 crypt_dec_pending(io);
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001173}
1174
Mikulas Patockadc267622015-02-13 08:25:59 -05001175static void kcryptd_queue_read(struct dm_crypt_io *io)
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001176{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001177 struct crypt_config *cc = io->cc;
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001178
Mikulas Patockadc267622015-02-13 08:25:59 -05001179 INIT_WORK(&io->work, kcryptd_io_read_work);
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001180 queue_work(cc->io_queue, &io->work);
1181}
1182
Mikulas Patockadc267622015-02-13 08:25:59 -05001183static void kcryptd_io_write(struct dm_crypt_io *io)
1184{
1185 struct bio *clone = io->ctx.bio_out;
1186
1187 generic_make_request(clone);
1188}
1189
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001190#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1191
Mikulas Patockadc267622015-02-13 08:25:59 -05001192static int dmcrypt_write(void *data)
1193{
1194 struct crypt_config *cc = data;
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001195 struct dm_crypt_io *io;
1196
Mikulas Patockadc267622015-02-13 08:25:59 -05001197 while (1) {
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001198 struct rb_root write_tree;
Mikulas Patockadc267622015-02-13 08:25:59 -05001199 struct blk_plug plug;
1200
1201 DECLARE_WAITQUEUE(wait, current);
1202
1203 spin_lock_irq(&cc->write_thread_wait.lock);
1204continue_locked:
1205
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001206 if (!RB_EMPTY_ROOT(&cc->write_tree))
Mikulas Patockadc267622015-02-13 08:25:59 -05001207 goto pop_from_list;
1208
Rabin Vincentf659b102016-09-21 16:22:29 +02001209 set_current_state(TASK_INTERRUPTIBLE);
Mikulas Patockadc267622015-02-13 08:25:59 -05001210 __add_wait_queue(&cc->write_thread_wait, &wait);
1211
1212 spin_unlock_irq(&cc->write_thread_wait.lock);
1213
Rabin Vincentf659b102016-09-21 16:22:29 +02001214 if (unlikely(kthread_should_stop())) {
1215 set_task_state(current, TASK_RUNNING);
1216 remove_wait_queue(&cc->write_thread_wait, &wait);
1217 break;
1218 }
1219
Mikulas Patockadc267622015-02-13 08:25:59 -05001220 schedule();
1221
Rabin Vincentf659b102016-09-21 16:22:29 +02001222 set_task_state(current, TASK_RUNNING);
Mikulas Patockadc267622015-02-13 08:25:59 -05001223 spin_lock_irq(&cc->write_thread_wait.lock);
1224 __remove_wait_queue(&cc->write_thread_wait, &wait);
1225 goto continue_locked;
1226
1227pop_from_list:
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001228 write_tree = cc->write_tree;
1229 cc->write_tree = RB_ROOT;
Mikulas Patockadc267622015-02-13 08:25:59 -05001230 spin_unlock_irq(&cc->write_thread_wait.lock);
1231
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001232 BUG_ON(rb_parent(write_tree.rb_node));
1233
1234 /*
1235 * Note: we cannot walk the tree here with rb_next because
1236 * the structures may be freed when kcryptd_io_write is called.
1237 */
Mikulas Patockadc267622015-02-13 08:25:59 -05001238 blk_start_plug(&plug);
1239 do {
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001240 io = crypt_io_from_node(rb_first(&write_tree));
1241 rb_erase(&io->rb_node, &write_tree);
Mikulas Patockadc267622015-02-13 08:25:59 -05001242 kcryptd_io_write(io);
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001243 } while (!RB_EMPTY_ROOT(&write_tree));
Mikulas Patockadc267622015-02-13 08:25:59 -05001244 blk_finish_plug(&plug);
1245 }
1246 return 0;
1247}
1248
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001249static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
Milan Broz4e4eef62008-02-08 02:10:49 +00001250{
Milan Brozdec1ced2008-02-08 02:10:57 +00001251 struct bio *clone = io->ctx.bio_out;
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001252 struct crypt_config *cc = io->cc;
Mikulas Patockadc267622015-02-13 08:25:59 -05001253 unsigned long flags;
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001254 sector_t sector;
1255 struct rb_node **rbp, *parent;
Milan Brozdec1ced2008-02-08 02:10:57 +00001256
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001257 if (unlikely(io->error < 0)) {
Milan Brozdec1ced2008-02-08 02:10:57 +00001258 crypt_free_buffer_pages(cc, clone);
1259 bio_put(clone);
Milan Broz6c031f42008-10-10 13:37:06 +01001260 crypt_dec_pending(io);
Milan Brozdec1ced2008-02-08 02:10:57 +00001261 return;
1262 }
1263
1264 /* crypt_convert should have filled the clone bio */
Kent Overstreet003b5c52013-10-11 15:45:43 -07001265 BUG_ON(io->ctx.iter_out.bi_size);
Milan Brozdec1ced2008-02-08 02:10:57 +00001266
Kent Overstreet4f024f32013-10-11 15:44:27 -07001267 clone->bi_iter.bi_sector = cc->start + io->sector;
Milan Broz899c95d2008-02-08 02:11:02 +00001268
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -05001269 if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) {
1270 generic_make_request(clone);
1271 return;
1272 }
1273
Mikulas Patockadc267622015-02-13 08:25:59 -05001274 spin_lock_irqsave(&cc->write_thread_wait.lock, flags);
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001275 rbp = &cc->write_tree.rb_node;
1276 parent = NULL;
1277 sector = io->sector;
1278 while (*rbp) {
1279 parent = *rbp;
1280 if (sector < crypt_io_from_node(parent)->sector)
1281 rbp = &(*rbp)->rb_left;
1282 else
1283 rbp = &(*rbp)->rb_right;
1284 }
1285 rb_link_node(&io->rb_node, parent, rbp);
1286 rb_insert_color(&io->rb_node, &cc->write_tree);
1287
Mikulas Patockadc267622015-02-13 08:25:59 -05001288 wake_up_locked(&cc->write_thread_wait);
1289 spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags);
Milan Broz4e4eef62008-02-08 02:10:49 +00001290}
1291
Milan Brozfc5a5e92008-10-10 13:37:04 +01001292static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -07001293{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001294 struct crypt_config *cc = io->cc;
Milan Broz8b004452006-10-03 01:15:37 -07001295 struct bio *clone;
Milan Brozc8081612008-10-10 13:37:08 +01001296 int crypt_finished;
Milan Brozb635b002008-10-21 17:45:00 +01001297 sector_t sector = io->sector;
Milan Brozdec1ced2008-02-08 02:10:57 +00001298 int r;
Milan Broz8b004452006-10-03 01:15:37 -07001299
Milan Broz93e605c2006-10-03 01:15:38 -07001300 /*
Milan Brozfc5a5e92008-10-10 13:37:04 +01001301 * Prevent io from disappearing until this function completes.
1302 */
1303 crypt_inc_pending(io);
Milan Brozb635b002008-10-21 17:45:00 +01001304 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
Milan Brozfc5a5e92008-10-10 13:37:04 +01001305
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -05001306 clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
1307 if (unlikely(!clone)) {
1308 io->error = -EIO;
1309 goto dec;
Milan Broz8b004452006-10-03 01:15:37 -07001310 }
Milan Broz899c95d2008-02-08 02:11:02 +00001311
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -05001312 io->ctx.bio_out = clone;
1313 io->ctx.iter_out = clone->bi_iter;
1314
1315 sector += bio_sectors(clone);
1316
1317 crypt_inc_pending(io);
1318 r = crypt_convert(cc, &io->ctx);
1319 if (r)
1320 io->error = -EIO;
1321 crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
1322
1323 /* Encryption was already finished, submit io now */
1324 if (crypt_finished) {
1325 kcryptd_crypt_write_io_submit(io, 0);
1326 io->sector = sector;
1327 }
1328
1329dec:
Milan Broz899c95d2008-02-08 02:11:02 +00001330 crypt_dec_pending(io);
Milan Broz84131db2008-02-08 02:10:59 +00001331}
1332
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001333static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
Milan Broz5742fd72008-02-08 02:10:43 +00001334{
Milan Broz5742fd72008-02-08 02:10:43 +00001335 crypt_dec_pending(io);
1336}
1337
Milan Broz4e4eef62008-02-08 02:10:49 +00001338static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
Milan Broz8b004452006-10-03 01:15:37 -07001339{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001340 struct crypt_config *cc = io->cc;
Milan Broz5742fd72008-02-08 02:10:43 +00001341 int r = 0;
Milan Broz8b004452006-10-03 01:15:37 -07001342
Milan Broz3e1a8bd2008-10-10 13:37:02 +01001343 crypt_inc_pending(io);
Milan Broz3a7f6c92008-02-08 02:11:14 +00001344
Milan Broz53017032008-02-08 02:10:38 +00001345 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
Milan Broz0c395b02008-02-08 02:10:54 +00001346 io->sector);
Milan Broz8b004452006-10-03 01:15:37 -07001347
Milan Broz5742fd72008-02-08 02:10:43 +00001348 r = crypt_convert(cc, &io->ctx);
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001349 if (r < 0)
1350 io->error = -EIO;
Milan Broz5742fd72008-02-08 02:10:43 +00001351
Mikulas Patocka40b62292012-07-27 15:08:04 +01001352 if (atomic_dec_and_test(&io->ctx.cc_pending))
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001353 kcryptd_crypt_read_done(io);
Milan Broz3a7f6c92008-02-08 02:11:14 +00001354
1355 crypt_dec_pending(io);
Milan Broz8b004452006-10-03 01:15:37 -07001356}
1357
Milan Broz95497a92008-02-08 02:11:12 +00001358static void kcryptd_async_done(struct crypto_async_request *async_req,
1359 int error)
1360{
Huang Yingb2174ee2009-03-16 17:44:33 +00001361 struct dm_crypt_request *dmreq = async_req->data;
1362 struct convert_context *ctx = dmreq->ctx;
Milan Broz95497a92008-02-08 02:11:12 +00001363 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001364 struct crypt_config *cc = io->cc;
Milan Broz95497a92008-02-08 02:11:12 +00001365
Milan Broz54cea3f2015-05-15 17:00:25 +02001366 /*
1367 * A request from crypto driver backlog is going to be processed now,
1368 * finish the completion and continue in crypt_convert().
1369 * (Callback will be called for the second time for this request.)
1370 */
Rabin Vincentc0403ec2015-05-05 15:15:56 +02001371 if (error == -EINPROGRESS) {
1372 complete(&ctx->restart);
Milan Broz95497a92008-02-08 02:11:12 +00001373 return;
Rabin Vincentc0403ec2015-05-05 15:15:56 +02001374 }
Milan Broz95497a92008-02-08 02:11:12 +00001375
Milan Broz2dc53272011-01-13 19:59:54 +00001376 if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
1377 error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq);
1378
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001379 if (error < 0)
1380 io->error = -EIO;
1381
Mikulas Patocka298a9fa2014-03-28 15:51:55 -04001382 crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
Milan Broz95497a92008-02-08 02:11:12 +00001383
Mikulas Patocka40b62292012-07-27 15:08:04 +01001384 if (!atomic_dec_and_test(&ctx->cc_pending))
Rabin Vincentc0403ec2015-05-05 15:15:56 +02001385 return;
Milan Broz95497a92008-02-08 02:11:12 +00001386
1387 if (bio_data_dir(io->base_bio) == READ)
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001388 kcryptd_crypt_read_done(io);
Milan Broz95497a92008-02-08 02:11:12 +00001389 else
Mikulas Patocka72c6e7a2012-03-28 18:41:22 +01001390 kcryptd_crypt_write_io_submit(io, 1);
Milan Broz95497a92008-02-08 02:11:12 +00001391}
1392
Milan Broz4e4eef62008-02-08 02:10:49 +00001393static void kcryptd_crypt(struct work_struct *work)
1394{
1395 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1396
1397 if (bio_data_dir(io->base_bio) == READ)
1398 kcryptd_crypt_read_convert(io);
1399 else
1400 kcryptd_crypt_write_convert(io);
Milan Broz8b004452006-10-03 01:15:37 -07001401}
1402
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001403static void kcryptd_queue_crypt(struct dm_crypt_io *io)
1404{
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001405 struct crypt_config *cc = io->cc;
Alasdair G Kergon395b1672008-02-08 02:10:52 +00001406
1407 INIT_WORK(&io->work, kcryptd_crypt);
1408 queue_work(cc->crypt_queue, &io->work);
1409}
1410
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411/*
1412 * Decode key from its hex representation
1413 */
1414static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
1415{
1416 char buffer[3];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417 unsigned int i;
1418
1419 buffer[2] = '\0';
1420
Milan Broz8b004452006-10-03 01:15:37 -07001421 for (i = 0; i < size; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 buffer[0] = *hex++;
1423 buffer[1] = *hex++;
1424
majianpeng1a66a082012-07-27 15:07:59 +01001425 if (kstrtou8(buffer, 16, &key[i]))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426 return -EINVAL;
1427 }
1428
1429 if (*hex != '\0')
1430 return -EINVAL;
1431
1432 return 0;
1433}
1434
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001435static void crypt_free_tfms(struct crypt_config *cc)
Milan Brozd1f96422011-01-13 19:59:54 +00001436{
Milan Brozd1f96422011-01-13 19:59:54 +00001437 unsigned i;
1438
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001439 if (!cc->tfms)
1440 return;
1441
Milan Brozd1f96422011-01-13 19:59:54 +00001442 for (i = 0; i < cc->tfms_count; i++)
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001443 if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) {
Herbert Xubbdb23b2016-01-24 21:16:36 +08001444 crypto_free_skcipher(cc->tfms[i]);
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001445 cc->tfms[i] = NULL;
Milan Brozd1f96422011-01-13 19:59:54 +00001446 }
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001447
1448 kfree(cc->tfms);
1449 cc->tfms = NULL;
Milan Brozd1f96422011-01-13 19:59:54 +00001450}
1451
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001452static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
Milan Brozd1f96422011-01-13 19:59:54 +00001453{
Milan Brozd1f96422011-01-13 19:59:54 +00001454 unsigned i;
1455 int err;
1456
Eric Biggers5d0be842016-08-30 09:51:44 -07001457 cc->tfms = kzalloc(cc->tfms_count * sizeof(struct crypto_skcipher *),
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001458 GFP_KERNEL);
1459 if (!cc->tfms)
1460 return -ENOMEM;
1461
Milan Brozd1f96422011-01-13 19:59:54 +00001462 for (i = 0; i < cc->tfms_count; i++) {
Herbert Xubbdb23b2016-01-24 21:16:36 +08001463 cc->tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0);
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001464 if (IS_ERR(cc->tfms[i])) {
1465 err = PTR_ERR(cc->tfms[i]);
1466 crypt_free_tfms(cc);
Milan Brozd1f96422011-01-13 19:59:54 +00001467 return err;
1468 }
1469 }
1470
1471 return 0;
1472}
1473
Andi Kleenc0297722011-01-13 19:59:53 +00001474static int crypt_setkey_allcpus(struct crypt_config *cc)
1475{
Milan Brozda31a072013-10-28 23:21:03 +01001476 unsigned subkey_size;
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001477 int err = 0, i, r;
Andi Kleenc0297722011-01-13 19:59:53 +00001478
Milan Brozda31a072013-10-28 23:21:03 +01001479 /* Ignore extra keys (which are used for IV etc) */
1480 subkey_size = (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
1481
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001482 for (i = 0; i < cc->tfms_count; i++) {
Herbert Xubbdb23b2016-01-24 21:16:36 +08001483 r = crypto_skcipher_setkey(cc->tfms[i],
1484 cc->key + (i * subkey_size),
1485 subkey_size);
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001486 if (r)
1487 err = r;
Andi Kleenc0297722011-01-13 19:59:53 +00001488 }
1489
1490 return err;
1491}
1492
Milan Broze48d4bb2006-10-03 01:15:37 -07001493static int crypt_set_key(struct crypt_config *cc, char *key)
1494{
Milan Brozde8be5a2011-03-24 13:54:27 +00001495 int r = -EINVAL;
1496 int key_string_len = strlen(key);
1497
Milan Broz69a8cfc2011-01-13 19:59:49 +00001498 /* The key size may not be changed. */
Milan Brozde8be5a2011-03-24 13:54:27 +00001499 if (cc->key_size != (key_string_len >> 1))
1500 goto out;
Milan Broze48d4bb2006-10-03 01:15:37 -07001501
Milan Broz69a8cfc2011-01-13 19:59:49 +00001502 /* Hyphen (which gives a key_size of zero) means there is no key. */
1503 if (!cc->key_size && strcmp(key, "-"))
Milan Brozde8be5a2011-03-24 13:54:27 +00001504 goto out;
Milan Broze48d4bb2006-10-03 01:15:37 -07001505
Ondrej Kozina2c017f72016-11-02 15:02:08 +01001506 /* clear the flag since following operations may invalidate previously valid key */
1507 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
1508
Milan Broz69a8cfc2011-01-13 19:59:49 +00001509 if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
Milan Brozde8be5a2011-03-24 13:54:27 +00001510 goto out;
Milan Broze48d4bb2006-10-03 01:15:37 -07001511
Milan Brozde8be5a2011-03-24 13:54:27 +00001512 r = crypt_setkey_allcpus(cc);
Ondrej Kozina2c017f72016-11-02 15:02:08 +01001513 if (!r)
1514 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
Milan Brozde8be5a2011-03-24 13:54:27 +00001515
1516out:
1517 /* Hex key string not needed after here, so wipe it. */
1518 memset(key, '0', key_string_len);
1519
1520 return r;
Milan Broze48d4bb2006-10-03 01:15:37 -07001521}
1522
1523static int crypt_wipe_key(struct crypt_config *cc)
1524{
1525 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
1526 memset(&cc->key, 0, cc->key_size * sizeof(u8));
Andi Kleenc0297722011-01-13 19:59:53 +00001527
1528 return crypt_setkey_allcpus(cc);
Milan Broze48d4bb2006-10-03 01:15:37 -07001529}
1530
Milan Broz28513fc2010-08-12 04:14:06 +01001531static void crypt_dtr(struct dm_target *ti)
1532{
1533 struct crypt_config *cc = ti->private;
1534
1535 ti->private = NULL;
1536
1537 if (!cc)
1538 return;
1539
Rabin Vincentf659b102016-09-21 16:22:29 +02001540 if (cc->write_thread)
Mikulas Patockadc267622015-02-13 08:25:59 -05001541 kthread_stop(cc->write_thread);
1542
Milan Broz28513fc2010-08-12 04:14:06 +01001543 if (cc->io_queue)
1544 destroy_workqueue(cc->io_queue);
1545 if (cc->crypt_queue)
1546 destroy_workqueue(cc->crypt_queue);
1547
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001548 crypt_free_tfms(cc);
1549
Milan Broz28513fc2010-08-12 04:14:06 +01001550 if (cc->bs)
1551 bioset_free(cc->bs);
1552
Julia Lawall6f659852015-09-13 14:15:05 +02001553 mempool_destroy(cc->page_pool);
1554 mempool_destroy(cc->req_pool);
Milan Broz28513fc2010-08-12 04:14:06 +01001555
1556 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
1557 cc->iv_gen_ops->dtr(cc);
1558
Milan Broz28513fc2010-08-12 04:14:06 +01001559 if (cc->dev)
1560 dm_put_device(ti, cc->dev);
1561
Milan Broz5ebaee62010-08-12 04:14:07 +01001562 kzfree(cc->cipher);
Milan Broz7dbcd132011-01-13 19:59:52 +00001563 kzfree(cc->cipher_string);
Milan Broz28513fc2010-08-12 04:14:06 +01001564
1565 /* Must zero key material before freeing */
1566 kzfree(cc);
1567}
1568
Milan Broz5ebaee62010-08-12 04:14:07 +01001569static int crypt_ctr_cipher(struct dm_target *ti,
1570 char *cipher_in, char *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571{
Milan Broz5ebaee62010-08-12 04:14:07 +01001572 struct crypt_config *cc = ti->private;
Milan Brozd1f96422011-01-13 19:59:54 +00001573 char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
Milan Broz5ebaee62010-08-12 04:14:07 +01001574 char *cipher_api = NULL;
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001575 int ret = -EINVAL;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001576 char dummy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
Milan Broz5ebaee62010-08-12 04:14:07 +01001578 /* Convert to crypto api definition? */
1579 if (strchr(cipher_in, '(')) {
1580 ti->error = "Bad cipher specification";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 return -EINVAL;
1582 }
1583
Milan Broz7dbcd132011-01-13 19:59:52 +00001584 cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
1585 if (!cc->cipher_string)
1586 goto bad_mem;
1587
Milan Broz5ebaee62010-08-12 04:14:07 +01001588 /*
1589 * Legacy dm-crypt cipher specification
Milan Brozd1f96422011-01-13 19:59:54 +00001590 * cipher[:keycount]-mode-iv:ivopts
Milan Broz5ebaee62010-08-12 04:14:07 +01001591 */
1592 tmp = cipher_in;
Milan Brozd1f96422011-01-13 19:59:54 +00001593 keycount = strsep(&tmp, "-");
1594 cipher = strsep(&keycount, ":");
1595
1596 if (!keycount)
1597 cc->tfms_count = 1;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001598 else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
Milan Brozd1f96422011-01-13 19:59:54 +00001599 !is_power_of_2(cc->tfms_count)) {
1600 ti->error = "Bad cipher key count specification";
1601 return -EINVAL;
1602 }
1603 cc->key_parts = cc->tfms_count;
Milan Brozda31a072013-10-28 23:21:03 +01001604 cc->key_extra_size = 0;
Milan Broz5ebaee62010-08-12 04:14:07 +01001605
1606 cc->cipher = kstrdup(cipher, GFP_KERNEL);
1607 if (!cc->cipher)
1608 goto bad_mem;
1609
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610 chainmode = strsep(&tmp, "-");
1611 ivopts = strsep(&tmp, "-");
1612 ivmode = strsep(&ivopts, ":");
1613
1614 if (tmp)
Milan Broz5ebaee62010-08-12 04:14:07 +01001615 DMWARN("Ignoring unexpected additional cipher options");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616
Milan Broz7dbcd132011-01-13 19:59:52 +00001617 /*
1618 * For compatibility with the original dm-crypt mapping format, if
1619 * only the cipher name is supplied, use cbc-plain.
1620 */
Milan Broz5ebaee62010-08-12 04:14:07 +01001621 if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622 chainmode = "cbc";
1623 ivmode = "plain";
1624 }
1625
Herbert Xud1806f62006-08-22 20:29:17 +10001626 if (strcmp(chainmode, "ecb") && !ivmode) {
Milan Broz5ebaee62010-08-12 04:14:07 +01001627 ti->error = "IV mechanism required";
1628 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 }
1630
Milan Broz5ebaee62010-08-12 04:14:07 +01001631 cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
1632 if (!cipher_api)
1633 goto bad_mem;
1634
1635 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
1636 "%s(%s)", chainmode, cipher);
1637 if (ret < 0) {
1638 kfree(cipher_api);
1639 goto bad_mem;
Herbert Xud1806f62006-08-22 20:29:17 +10001640 }
1641
Milan Broz5ebaee62010-08-12 04:14:07 +01001642 /* Allocate cipher */
Mikulas Patockafd2d2312012-07-27 15:08:05 +01001643 ret = crypt_alloc_tfms(cc, cipher_api);
1644 if (ret < 0) {
1645 ti->error = "Error allocating crypto tfm";
1646 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001648
Milan Broz5ebaee62010-08-12 04:14:07 +01001649 /* Initialize IV */
Herbert Xubbdb23b2016-01-24 21:16:36 +08001650 cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
Milan Broz5ebaee62010-08-12 04:14:07 +01001651 if (cc->iv_size)
1652 /* at least a 64 bit sector number should fit in our buffer */
1653 cc->iv_size = max(cc->iv_size,
1654 (unsigned int)(sizeof(u64) / sizeof(u8)));
1655 else if (ivmode) {
1656 DMWARN("Selected cipher does not support IVs");
1657 ivmode = NULL;
1658 }
1659
1660 /* Choose ivmode, see comments at iv code. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661 if (ivmode == NULL)
1662 cc->iv_gen_ops = NULL;
1663 else if (strcmp(ivmode, "plain") == 0)
1664 cc->iv_gen_ops = &crypt_iv_plain_ops;
Milan Broz61afef62009-12-10 23:52:25 +00001665 else if (strcmp(ivmode, "plain64") == 0)
1666 cc->iv_gen_ops = &crypt_iv_plain64_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 else if (strcmp(ivmode, "essiv") == 0)
1668 cc->iv_gen_ops = &crypt_iv_essiv_ops;
Rik Snel48527fa2006-09-03 08:56:39 +10001669 else if (strcmp(ivmode, "benbi") == 0)
1670 cc->iv_gen_ops = &crypt_iv_benbi_ops;
Ludwig Nussel46b47732007-05-09 02:32:55 -07001671 else if (strcmp(ivmode, "null") == 0)
1672 cc->iv_gen_ops = &crypt_iv_null_ops;
Milan Broz34745782011-01-13 19:59:55 +00001673 else if (strcmp(ivmode, "lmk") == 0) {
1674 cc->iv_gen_ops = &crypt_iv_lmk_ops;
Milan Brozed04d982013-10-28 23:21:04 +01001675 /*
1676 * Version 2 and 3 is recognised according
Milan Broz34745782011-01-13 19:59:55 +00001677 * to length of provided multi-key string.
1678 * If present (version 3), last key is used as IV seed.
Milan Brozed04d982013-10-28 23:21:04 +01001679 * All keys (including IV seed) are always the same size.
Milan Broz34745782011-01-13 19:59:55 +00001680 */
Milan Brozda31a072013-10-28 23:21:03 +01001681 if (cc->key_size % cc->key_parts) {
Milan Broz34745782011-01-13 19:59:55 +00001682 cc->key_parts++;
Milan Brozda31a072013-10-28 23:21:03 +01001683 cc->key_extra_size = cc->key_size / cc->key_parts;
1684 }
Milan Brozed04d982013-10-28 23:21:04 +01001685 } else if (strcmp(ivmode, "tcw") == 0) {
1686 cc->iv_gen_ops = &crypt_iv_tcw_ops;
1687 cc->key_parts += 2; /* IV + whitening */
1688 cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
Milan Broz34745782011-01-13 19:59:55 +00001689 } else {
Milan Broz5ebaee62010-08-12 04:14:07 +01001690 ret = -EINVAL;
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001691 ti->error = "Invalid IV mode";
Milan Broz28513fc2010-08-12 04:14:06 +01001692 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693 }
1694
Milan Brozda31a072013-10-28 23:21:03 +01001695 /* Initialize and set key */
1696 ret = crypt_set_key(cc, key);
1697 if (ret < 0) {
1698 ti->error = "Error decoding and setting key";
1699 goto bad;
1700 }
1701
Milan Broz28513fc2010-08-12 04:14:06 +01001702 /* Allocate IV */
1703 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
1704 ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
1705 if (ret < 0) {
1706 ti->error = "Error creating IV";
1707 goto bad;
1708 }
Milan Brozb95bf2d2009-12-10 23:51:56 +00001709 }
1710
Milan Broz28513fc2010-08-12 04:14:06 +01001711 /* Initialize IV (set keys for ESSIV etc) */
1712 if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
1713 ret = cc->iv_gen_ops->init(cc);
1714 if (ret < 0) {
1715 ti->error = "Error initialising IV";
1716 goto bad;
1717 }
1718 }
1719
Milan Broz5ebaee62010-08-12 04:14:07 +01001720 ret = 0;
1721bad:
1722 kfree(cipher_api);
1723 return ret;
1724
1725bad_mem:
1726 ti->error = "Cannot allocate cipher strings";
1727 return -ENOMEM;
1728}
1729
1730/*
1731 * Construct an encryption mapping:
1732 * <cipher> <key> <iv_offset> <dev_path> <start>
1733 */
1734static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1735{
1736 struct crypt_config *cc;
Milan Broz772ae5f2011-08-02 12:32:08 +01001737 unsigned int key_size, opt_params;
Milan Broz5ebaee62010-08-12 04:14:07 +01001738 unsigned long long tmpll;
1739 int ret;
Mikulas Patockad49ec522014-08-28 11:09:31 -04001740 size_t iv_size_padding;
Milan Broz772ae5f2011-08-02 12:32:08 +01001741 struct dm_arg_set as;
1742 const char *opt_string;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001743 char dummy;
Milan Broz5ebaee62010-08-12 04:14:07 +01001744
Milan Broz772ae5f2011-08-02 12:32:08 +01001745 static struct dm_arg _args[] = {
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -05001746 {0, 3, "Invalid number of feature args"},
Milan Broz772ae5f2011-08-02 12:32:08 +01001747 };
1748
1749 if (argc < 5) {
Milan Broz5ebaee62010-08-12 04:14:07 +01001750 ti->error = "Not enough arguments";
1751 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752 }
1753
Milan Broz5ebaee62010-08-12 04:14:07 +01001754 key_size = strlen(argv[1]) >> 1;
1755
1756 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
1757 if (!cc) {
1758 ti->error = "Cannot allocate encryption context";
1759 return -ENOMEM;
1760 }
Milan Broz69a8cfc2011-01-13 19:59:49 +00001761 cc->key_size = key_size;
Milan Broz5ebaee62010-08-12 04:14:07 +01001762
1763 ti->private = cc;
1764 ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
1765 if (ret < 0)
1766 goto bad;
1767
Herbert Xubbdb23b2016-01-24 21:16:36 +08001768 cc->dmreq_start = sizeof(struct skcipher_request);
1769 cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
Mikulas Patockad49ec522014-08-28 11:09:31 -04001770 cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
1771
Herbert Xubbdb23b2016-01-24 21:16:36 +08001772 if (crypto_skcipher_alignmask(any_tfm(cc)) < CRYPTO_MINALIGN) {
Mikulas Patockad49ec522014-08-28 11:09:31 -04001773 /* Allocate the padding exactly */
1774 iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
Herbert Xubbdb23b2016-01-24 21:16:36 +08001775 & crypto_skcipher_alignmask(any_tfm(cc));
Mikulas Patockad49ec522014-08-28 11:09:31 -04001776 } else {
1777 /*
1778 * If the cipher requires greater alignment than kmalloc
1779 * alignment, we don't know the exact position of the
1780 * initialization vector. We must assume worst case.
1781 */
Herbert Xubbdb23b2016-01-24 21:16:36 +08001782 iv_size_padding = crypto_skcipher_alignmask(any_tfm(cc));
Mikulas Patockad49ec522014-08-28 11:09:31 -04001783 }
Milan Brozddd42ed2008-02-08 02:11:07 +00001784
Mikulas Patocka94f5e022015-02-13 08:25:26 -05001785 ret = -ENOMEM;
Milan Brozddd42ed2008-02-08 02:11:07 +00001786 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
Mikulas Patockad49ec522014-08-28 11:09:31 -04001787 sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size);
Milan Brozddd42ed2008-02-08 02:11:07 +00001788 if (!cc->req_pool) {
1789 ti->error = "Cannot allocate crypt request mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001790 goto bad;
Milan Brozddd42ed2008-02-08 02:11:07 +00001791 }
Milan Brozddd42ed2008-02-08 02:11:07 +00001792
Mike Snitzer30187e12016-01-31 13:28:26 -05001793 cc->per_bio_data_size = ti->per_io_data_size =
Mikulas Patockad49ec522014-08-28 11:09:31 -04001794 ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start +
1795 sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size,
1796 ARCH_KMALLOC_MINALIGN);
Mikulas Patocka298a9fa2014-03-28 15:51:55 -04001797
Mikulas Patockacf2f1ab2015-02-13 08:23:52 -05001798 cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799 if (!cc->page_pool) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001800 ti->error = "Cannot allocate page mempool";
Milan Broz28513fc2010-08-12 04:14:06 +01001801 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 }
1803
Jens Axboebb799ca2008-12-10 15:35:05 +01001804 cc->bs = bioset_create(MIN_IOS, 0);
Milan Broz6a24c712006-10-03 01:15:40 -07001805 if (!cc->bs) {
1806 ti->error = "Cannot allocate crypt bioset";
Milan Broz28513fc2010-08-12 04:14:06 +01001807 goto bad;
Milan Broz6a24c712006-10-03 01:15:40 -07001808 }
1809
Mikulas Patocka7145c242015-02-13 08:24:41 -05001810 mutex_init(&cc->bio_alloc_lock);
1811
Milan Broz28513fc2010-08-12 04:14:06 +01001812 ret = -EINVAL;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001813 if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001814 ti->error = "Invalid iv_offset sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001815 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001816 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001817 cc->iv_offset = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818
Vivek Goyale80d1c82015-07-31 09:20:36 -04001819 ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);
1820 if (ret) {
Milan Broz28513fc2010-08-12 04:14:06 +01001821 ti->error = "Device lookup failed";
1822 goto bad;
1823 }
1824
Vivek Goyale80d1c82015-07-31 09:20:36 -04001825 ret = -EINVAL;
Mikulas Patocka31998ef2012-03-28 18:41:26 +01001826 if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001827 ti->error = "Invalid device sector";
Milan Broz28513fc2010-08-12 04:14:06 +01001828 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 }
Andrew Morton4ee218c2006-03-27 01:17:48 -08001830 cc->start = tmpll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831
Milan Broz772ae5f2011-08-02 12:32:08 +01001832 argv += 5;
1833 argc -= 5;
1834
1835 /* Optional parameters */
1836 if (argc) {
1837 as.argc = argc;
1838 as.argv = argv;
1839
1840 ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
1841 if (ret)
1842 goto bad;
1843
Wei Yongjun44c144f2015-04-16 22:00:50 -04001844 ret = -EINVAL;
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001845 while (opt_params--) {
1846 opt_string = dm_shift_arg(&as);
1847 if (!opt_string) {
1848 ti->error = "Not enough feature arguments";
1849 goto bad;
1850 }
Milan Broz772ae5f2011-08-02 12:32:08 +01001851
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001852 if (!strcasecmp(opt_string, "allow_discards"))
1853 ti->num_discard_bios = 1;
1854
1855 else if (!strcasecmp(opt_string, "same_cpu_crypt"))
1856 set_bit(DM_CRYPT_SAME_CPU, &cc->flags);
1857
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -05001858 else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))
1859 set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
1860
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001861 else {
1862 ti->error = "Invalid feature arguments";
1863 goto bad;
1864 }
Milan Broz772ae5f2011-08-02 12:32:08 +01001865 }
1866 }
1867
Milan Broz28513fc2010-08-12 04:14:06 +01001868 ret = -ENOMEM;
Tim Murraye97c4ed2016-01-19 16:33:27 -08001869 cc->io_queue = alloc_workqueue("kcryptd_io",
1870 WQ_HIGHPRI |
1871 WQ_MEM_RECLAIM,
1872 1);
Milan Brozcabf08e2007-10-19 22:38:58 +01001873 if (!cc->io_queue) {
1874 ti->error = "Couldn't create kcryptd io queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001875 goto bad;
Milan Brozcabf08e2007-10-19 22:38:58 +01001876 }
1877
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001878 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
Tim Murraye97c4ed2016-01-19 16:33:27 -08001879 cc->crypt_queue = alloc_workqueue("kcryptd",
1880 WQ_HIGHPRI |
1881 WQ_MEM_RECLAIM, 1);
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001882 else
Tim Murraye97c4ed2016-01-19 16:33:27 -08001883 cc->crypt_queue = alloc_workqueue("kcryptd",
1884 WQ_HIGHPRI |
1885 WQ_MEM_RECLAIM |
1886 WQ_UNBOUND,
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001887 num_online_cpus());
Milan Brozcabf08e2007-10-19 22:38:58 +01001888 if (!cc->crypt_queue) {
Milan Broz9934a8b2007-10-19 22:38:57 +01001889 ti->error = "Couldn't create kcryptd queue";
Milan Broz28513fc2010-08-12 04:14:06 +01001890 goto bad;
Milan Broz9934a8b2007-10-19 22:38:57 +01001891 }
1892
Mikulas Patockadc267622015-02-13 08:25:59 -05001893 init_waitqueue_head(&cc->write_thread_wait);
Mikulas Patockab3c5fd32015-02-13 08:27:41 -05001894 cc->write_tree = RB_ROOT;
Mikulas Patockadc267622015-02-13 08:25:59 -05001895
1896 cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write");
1897 if (IS_ERR(cc->write_thread)) {
1898 ret = PTR_ERR(cc->write_thread);
1899 cc->write_thread = NULL;
1900 ti->error = "Couldn't spawn write thread";
1901 goto bad;
1902 }
1903 wake_up_process(cc->write_thread);
1904
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00001905 ti->num_flush_bios = 1;
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01001906 ti->discard_zeroes_data_unsupported = true;
Milan Broz983c7db2011-09-25 23:26:21 +01001907
Linus Torvalds1da177e2005-04-16 15:20:36 -07001908 return 0;
1909
Milan Broz28513fc2010-08-12 04:14:06 +01001910bad:
1911 crypt_dtr(ti);
1912 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001913}
1914
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00001915static int crypt_map(struct dm_target *ti, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001916{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001917 struct dm_crypt_io *io;
Alasdair G Kergon49a8a922012-07-27 15:08:05 +01001918 struct crypt_config *cc = ti->private;
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001919
Milan Broz772ae5f2011-08-02 12:32:08 +01001920 /*
Mike Christie28a8f0d2016-06-05 14:32:25 -05001921 * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
1922 * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
Mike Christiee6047142016-06-05 14:32:04 -05001923 * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
Milan Broz772ae5f2011-08-02 12:32:08 +01001924 */
Jens Axboe1eff9d32016-08-05 15:35:16 -06001925 if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
Mike Christie28a8f0d2016-06-05 14:32:25 -05001926 bio_op(bio) == REQ_OP_DISCARD)) {
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001927 bio->bi_bdev = cc->dev->bdev;
Milan Broz772ae5f2011-08-02 12:32:08 +01001928 if (bio_sectors(bio))
Kent Overstreet4f024f32013-10-11 15:44:27 -07001929 bio->bi_iter.bi_sector = cc->start +
1930 dm_target_offset(ti, bio->bi_iter.bi_sector);
Mikulas Patocka647c7db2009-06-22 10:12:23 +01001931 return DM_MAPIO_REMAPPED;
1932 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001933
Mikulas Patocka4e870e92016-08-30 16:38:42 -04001934 /*
1935 * Check if bio is too large, split as needed.
1936 */
1937 if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) &&
1938 bio_data_dir(bio) == WRITE)
1939 dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT));
1940
Mikulas Patocka298a9fa2014-03-28 15:51:55 -04001941 io = dm_per_bio_data(bio, cc->per_bio_data_size);
1942 crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
Herbert Xubbdb23b2016-01-24 21:16:36 +08001943 io->ctx.req = (struct skcipher_request *)(io + 1);
Milan Brozcabf08e2007-10-19 22:38:58 +01001944
Milan Broz20c82532011-01-13 19:59:53 +00001945 if (bio_data_dir(io->base_bio) == READ) {
1946 if (kcryptd_io_read(io, GFP_NOWAIT))
Mikulas Patockadc267622015-02-13 08:25:59 -05001947 kcryptd_queue_read(io);
Milan Broz20c82532011-01-13 19:59:53 +00001948 } else
Milan Brozcabf08e2007-10-19 22:38:58 +01001949 kcryptd_queue_crypt(io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001951 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952}
1953
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001954static void crypt_status(struct dm_target *ti, status_type_t type,
1955 unsigned status_flags, char *result, unsigned maxlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956{
Milan Broz5ebaee62010-08-12 04:14:07 +01001957 struct crypt_config *cc = ti->private;
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001958 unsigned i, sz = 0;
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001959 int num_feature_args = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
1961 switch (type) {
1962 case STATUSTYPE_INFO:
1963 result[0] = '\0';
1964 break;
1965
1966 case STATUSTYPE_TABLE:
Milan Broz7dbcd132011-01-13 19:59:52 +00001967 DMEMIT("%s ", cc->cipher_string);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968
Mikulas Patockafd7c0922013-03-01 22:45:44 +00001969 if (cc->key_size > 0)
1970 for (i = 0; i < cc->key_size; i++)
1971 DMEMIT("%02x", cc->key[i]);
1972 else
1973 DMEMIT("-");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974
Andrew Morton4ee218c2006-03-27 01:17:48 -08001975 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
1976 cc->dev->name, (unsigned long long)cc->start);
Milan Broz772ae5f2011-08-02 12:32:08 +01001977
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001978 num_feature_args += !!ti->num_discard_bios;
1979 num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -05001980 num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001981 if (num_feature_args) {
1982 DMEMIT(" %d", num_feature_args);
1983 if (ti->num_discard_bios)
1984 DMEMIT(" allow_discards");
1985 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
1986 DMEMIT(" same_cpu_crypt");
Mikulas Patocka0f5d8e62015-02-13 08:27:08 -05001987 if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
1988 DMEMIT(" submit_from_crypt_cpus");
Mikulas Patockaf3396c582015-02-13 08:23:09 -05001989 }
Milan Broz772ae5f2011-08-02 12:32:08 +01001990
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 break;
1992 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993}
1994
Milan Broze48d4bb2006-10-03 01:15:37 -07001995static void crypt_postsuspend(struct dm_target *ti)
1996{
1997 struct crypt_config *cc = ti->private;
1998
1999 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
2000}
2001
2002static int crypt_preresume(struct dm_target *ti)
2003{
2004 struct crypt_config *cc = ti->private;
2005
2006 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
2007 DMERR("aborting resume - crypt key is not set.");
2008 return -EAGAIN;
2009 }
2010
2011 return 0;
2012}
2013
2014static void crypt_resume(struct dm_target *ti)
2015{
2016 struct crypt_config *cc = ti->private;
2017
2018 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
2019}
2020
2021/* Message interface
2022 * key set <key>
2023 * key wipe
2024 */
2025static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
2026{
2027 struct crypt_config *cc = ti->private;
Milan Broz542da312009-12-10 23:51:57 +00002028 int ret = -EINVAL;
Milan Broze48d4bb2006-10-03 01:15:37 -07002029
2030 if (argc < 2)
2031 goto error;
2032
Mike Snitzer498f0102011-08-02 12:32:04 +01002033 if (!strcasecmp(argv[0], "key")) {
Milan Broze48d4bb2006-10-03 01:15:37 -07002034 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
2035 DMWARN("not suspended during key manipulation.");
2036 return -EINVAL;
2037 }
Mike Snitzer498f0102011-08-02 12:32:04 +01002038 if (argc == 3 && !strcasecmp(argv[1], "set")) {
Milan Broz542da312009-12-10 23:51:57 +00002039 ret = crypt_set_key(cc, argv[2]);
2040 if (ret)
2041 return ret;
2042 if (cc->iv_gen_ops && cc->iv_gen_ops->init)
2043 ret = cc->iv_gen_ops->init(cc);
2044 return ret;
2045 }
Mike Snitzer498f0102011-08-02 12:32:04 +01002046 if (argc == 2 && !strcasecmp(argv[1], "wipe")) {
Milan Broz542da312009-12-10 23:51:57 +00002047 if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
2048 ret = cc->iv_gen_ops->wipe(cc);
2049 if (ret)
2050 return ret;
2051 }
Milan Broze48d4bb2006-10-03 01:15:37 -07002052 return crypt_wipe_key(cc);
Milan Broz542da312009-12-10 23:51:57 +00002053 }
Milan Broze48d4bb2006-10-03 01:15:37 -07002054 }
2055
2056error:
2057 DMWARN("unrecognised message received.");
2058 return -EINVAL;
2059}
2060
Mike Snitzeraf4874e2009-06-22 10:12:33 +01002061static int crypt_iterate_devices(struct dm_target *ti,
2062 iterate_devices_callout_fn fn, void *data)
2063{
2064 struct crypt_config *cc = ti->private;
2065
Mike Snitzer5dea2712009-07-23 20:30:42 +01002066 return fn(ti, cc->dev, cc->start, ti->len, data);
Mike Snitzeraf4874e2009-06-22 10:12:33 +01002067}
2068
Mike Snitzer586b2862015-09-09 21:34:51 -04002069static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
2070{
2071 /*
2072 * Unfortunate constraint that is required to avoid the potential
2073 * for exceeding underlying device's max_segments limits -- due to
2074 * crypt_alloc_buffer() possibly allocating pages for the encryption
2075 * bio that are not as physically contiguous as the original bio.
2076 */
2077 limits->max_segment_size = PAGE_SIZE;
2078}
2079
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080static struct target_type crypt_target = {
2081 .name = "crypt",
Mike Snitzer586b2862015-09-09 21:34:51 -04002082 .version = {1, 14, 1},
Linus Torvalds1da177e2005-04-16 15:20:36 -07002083 .module = THIS_MODULE,
2084 .ctr = crypt_ctr,
2085 .dtr = crypt_dtr,
2086 .map = crypt_map,
2087 .status = crypt_status,
Milan Broze48d4bb2006-10-03 01:15:37 -07002088 .postsuspend = crypt_postsuspend,
2089 .preresume = crypt_preresume,
2090 .resume = crypt_resume,
2091 .message = crypt_message,
Mike Snitzeraf4874e2009-06-22 10:12:33 +01002092 .iterate_devices = crypt_iterate_devices,
Mike Snitzer586b2862015-09-09 21:34:51 -04002093 .io_hints = crypt_io_hints,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094};
2095
2096static int __init dm_crypt_init(void)
2097{
2098 int r;
2099
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 r = dm_register_target(&crypt_target);
Mikulas Patocka94f5e022015-02-13 08:25:26 -05002101 if (r < 0)
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002102 DMERR("register failed %d", r);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002103
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104 return r;
2105}
2106
2107static void __exit dm_crypt_exit(void)
2108{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00002109 dm_unregister_target(&crypt_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002110}
2111
2112module_init(dm_crypt_init);
2113module_exit(dm_crypt_exit);
2114
Jana Saoutbf142992014-06-24 14:27:04 -04002115MODULE_AUTHOR("Jana Saout <jana@saout.de>");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
2117MODULE_LICENSE("GPL");