blob: 4a62ef14e93275a881f967ceabd66c6bd306345c [file] [log] [blame]
Jaegeuk Kim57e50552015-04-20 19:52:47 -07001/*
2 * linux/fs/f2fs/crypto.c
3 *
4 * Copied from linux/fs/ext4/crypto.c
5 *
6 * Copyright (C) 2015, Google, Inc.
7 * Copyright (C) 2015, Motorola Mobility
8 *
9 * This contains encryption functions for f2fs
10 *
11 * Written by Michael Halcrow, 2014.
12 *
13 * Filename encryption additions
14 * Uday Savagaonkar, 2014
15 * Encryption policy handling additions
16 * Ildar Muslukhov, 2014
17 * Remove ext4_encrypted_zeroout(),
18 * add f2fs_restore_and_release_control_page()
19 * Jaegeuk Kim, 2015.
20 *
21 * This has not yet undergone a rigorous security audit.
22 *
23 * The usage of AES-XTS should conform to recommendations in NIST
24 * Special Publication 800-38E and IEEE P1619/D16.
25 */
26#include <crypto/hash.h>
27#include <crypto/sha.h>
28#include <keys/user-type.h>
29#include <keys/encrypted-type.h>
30#include <linux/crypto.h>
31#include <linux/ecryptfs.h>
32#include <linux/gfp.h>
33#include <linux/kernel.h>
34#include <linux/key.h>
35#include <linux/list.h>
36#include <linux/mempool.h>
37#include <linux/module.h>
38#include <linux/mutex.h>
39#include <linux/random.h>
40#include <linux/scatterlist.h>
41#include <linux/spinlock_types.h>
42#include <linux/f2fs_fs.h>
43#include <linux/ratelimit.h>
44#include <linux/bio.h>
45
46#include "f2fs.h"
47#include "xattr.h"
48
49/* Encryption added and removed here! (L: */
50
51static unsigned int num_prealloc_crypto_pages = 32;
52static unsigned int num_prealloc_crypto_ctxs = 128;
53
54module_param(num_prealloc_crypto_pages, uint, 0444);
55MODULE_PARM_DESC(num_prealloc_crypto_pages,
56 "Number of crypto pages to preallocate");
57module_param(num_prealloc_crypto_ctxs, uint, 0444);
58MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
59 "Number of crypto contexts to preallocate");
60
61static mempool_t *f2fs_bounce_page_pool;
62
63static LIST_HEAD(f2fs_free_crypto_ctxs);
64static DEFINE_SPINLOCK(f2fs_crypto_ctx_lock);
65
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -070066static struct workqueue_struct *f2fs_read_workqueue;
Jaegeuk Kim57e50552015-04-20 19:52:47 -070067static DEFINE_MUTEX(crypto_init);
68
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -070069static struct kmem_cache *f2fs_crypto_ctx_cachep;
70struct kmem_cache *f2fs_crypt_info_cachep;
71
Jaegeuk Kim57e50552015-04-20 19:52:47 -070072/**
73 * f2fs_release_crypto_ctx() - Releases an encryption context
74 * @ctx: The encryption context to release.
75 *
76 * If the encryption context was allocated from the pre-allocated pool, returns
77 * it to that pool. Else, frees it.
78 *
79 * If there's a bounce page in the context, this frees that.
80 */
81void f2fs_release_crypto_ctx(struct f2fs_crypto_ctx *ctx)
82{
83 unsigned long flags;
84
Jaegeuk Kimca40b032015-05-12 13:40:20 -070085 if (ctx->flags & F2FS_WRITE_PATH_FL && ctx->w.bounce_page) {
Jaegeuk Kim4683ff82015-06-01 12:39:30 -070086 mempool_free(ctx->w.bounce_page, f2fs_bounce_page_pool);
Jaegeuk Kimca40b032015-05-12 13:40:20 -070087 ctx->w.bounce_page = NULL;
Jaegeuk Kim57e50552015-04-20 19:52:47 -070088 }
Jaegeuk Kimca40b032015-05-12 13:40:20 -070089 ctx->w.control_page = NULL;
Jaegeuk Kim57e50552015-04-20 19:52:47 -070090 if (ctx->flags & F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -070091 kmem_cache_free(f2fs_crypto_ctx_cachep, ctx);
Jaegeuk Kim57e50552015-04-20 19:52:47 -070092 } else {
93 spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags);
94 list_add(&ctx->free_list, &f2fs_free_crypto_ctxs);
95 spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags);
96 }
97}
98
99/**
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700100 * f2fs_get_crypto_ctx() - Gets an encryption context
101 * @inode: The inode for which we are doing the crypto
102 *
103 * Allocates and initializes an encryption context.
104 *
105 * Return: An allocated and initialized encryption context on success; error
106 * value or NULL otherwise.
107 */
108struct f2fs_crypto_ctx *f2fs_get_crypto_ctx(struct inode *inode)
109{
110 struct f2fs_crypto_ctx *ctx = NULL;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700111 unsigned long flags;
112 struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info;
113
Jaegeuk Kimedf3fb82015-05-05 20:20:29 -0700114 if (ci == NULL)
Jaegeuk Kim7e8e7542015-06-11 14:23:05 -0700115 return ERR_PTR(-ENOKEY);
Jaegeuk Kimedf3fb82015-05-05 20:20:29 -0700116
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700117 /*
118 * We first try getting the ctx from a free list because in
119 * the common case the ctx will have an allocated and
120 * initialized crypto tfm, so it's probably a worthwhile
121 * optimization. For the bounce page, we first try getting it
122 * from the kernel allocator because that's just about as fast
123 * as getting it from a list and because a cache of free pages
124 * should generally be a "last resort" option for a filesystem
125 * to be able to do its job.
126 */
127 spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags);
128 ctx = list_first_entry_or_null(&f2fs_free_crypto_ctxs,
129 struct f2fs_crypto_ctx, free_list);
130 if (ctx)
131 list_del(&ctx->free_list);
132 spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags);
133 if (!ctx) {
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700134 ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_NOFS);
Jaegeuk Kim26bf3dc2015-05-19 22:26:54 -0700135 if (!ctx)
136 return ERR_PTR(-ENOMEM);
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700137 ctx->flags |= F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
138 } else {
139 ctx->flags &= ~F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
140 }
Jaegeuk Kimca40b032015-05-12 13:40:20 -0700141 ctx->flags &= ~F2FS_WRITE_PATH_FL;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700142 return ctx;
143}
144
145/*
146 * Call f2fs_decrypt on every single page, reusing the encryption
147 * context.
148 */
149static void completion_pages(struct work_struct *work)
150{
151 struct f2fs_crypto_ctx *ctx =
Jaegeuk Kimca40b032015-05-12 13:40:20 -0700152 container_of(work, struct f2fs_crypto_ctx, r.work);
153 struct bio *bio = ctx->r.bio;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700154 struct bio_vec *bv;
155 int i;
156
157 bio_for_each_segment_all(bv, bio, i) {
158 struct page *page = bv->bv_page;
159 int ret = f2fs_decrypt(ctx, page);
160
161 if (ret) {
162 WARN_ON_ONCE(1);
163 SetPageError(page);
164 } else
165 SetPageUptodate(page);
166 unlock_page(page);
167 }
168 f2fs_release_crypto_ctx(ctx);
169 bio_put(bio);
170}
171
172void f2fs_end_io_crypto_work(struct f2fs_crypto_ctx *ctx, struct bio *bio)
173{
Jaegeuk Kimca40b032015-05-12 13:40:20 -0700174 INIT_WORK(&ctx->r.work, completion_pages);
175 ctx->r.bio = bio;
176 queue_work(f2fs_read_workqueue, &ctx->r.work);
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700177}
178
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700179static void f2fs_crypto_destroy(void)
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700180{
181 struct f2fs_crypto_ctx *pos, *n;
182
Jaegeuk Kim26bf3dc2015-05-19 22:26:54 -0700183 list_for_each_entry_safe(pos, n, &f2fs_free_crypto_ctxs, free_list)
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700184 kmem_cache_free(f2fs_crypto_ctx_cachep, pos);
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700185 INIT_LIST_HEAD(&f2fs_free_crypto_ctxs);
186 if (f2fs_bounce_page_pool)
187 mempool_destroy(f2fs_bounce_page_pool);
188 f2fs_bounce_page_pool = NULL;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700189}
190
191/**
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700192 * f2fs_crypto_initialize() - Set up for f2fs encryption.
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700193 *
194 * We only call this when we start accessing encrypted files, since it
195 * results in memory getting allocated that wouldn't otherwise be used.
196 *
197 * Return: Zero on success, non-zero otherwise.
198 */
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700199int f2fs_crypto_initialize(void)
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700200{
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700201 int i, res = -ENOMEM;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700202
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700203 if (f2fs_bounce_page_pool)
204 return 0;
205
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700206 mutex_lock(&crypto_init);
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700207 if (f2fs_bounce_page_pool)
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700208 goto already_initialized;
209
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700210 for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
211 struct f2fs_crypto_ctx *ctx;
212
213 ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_KERNEL);
214 if (!ctx)
215 goto fail;
216 list_add(&ctx->free_list, &f2fs_free_crypto_ctxs);
217 }
218
219 /* must be allocated at the last step to avoid race condition above */
220 f2fs_bounce_page_pool =
221 mempool_create_page_pool(num_prealloc_crypto_pages, 0);
222 if (!f2fs_bounce_page_pool)
223 goto fail;
224
225already_initialized:
226 mutex_unlock(&crypto_init);
227 return 0;
228fail:
229 f2fs_crypto_destroy();
230 mutex_unlock(&crypto_init);
231 return res;
232}
233
234/**
235 * f2fs_exit_crypto() - Shutdown the f2fs encryption system
236 */
237void f2fs_exit_crypto(void)
238{
239 f2fs_crypto_destroy();
240
241 if (f2fs_read_workqueue)
242 destroy_workqueue(f2fs_read_workqueue);
243 if (f2fs_crypto_ctx_cachep)
244 kmem_cache_destroy(f2fs_crypto_ctx_cachep);
245 if (f2fs_crypt_info_cachep)
246 kmem_cache_destroy(f2fs_crypt_info_cachep);
247}
248
249int __init f2fs_init_crypto(void)
250{
251 int res = -ENOMEM;
252
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700253 f2fs_read_workqueue = alloc_workqueue("f2fs_crypto", WQ_HIGHPRI, 0);
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700254 if (!f2fs_read_workqueue)
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700255 goto fail;
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700256
257 f2fs_crypto_ctx_cachep = KMEM_CACHE(f2fs_crypto_ctx,
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700258 SLAB_RECLAIM_ACCOUNT);
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700259 if (!f2fs_crypto_ctx_cachep)
260 goto fail;
261
262 f2fs_crypt_info_cachep = KMEM_CACHE(f2fs_crypt_info,
Jaegeuk Kimcfc4d972015-05-15 15:37:24 -0700263 SLAB_RECLAIM_ACCOUNT);
Jaegeuk Kim8bacf6de2015-05-12 13:26:54 -0700264 if (!f2fs_crypt_info_cachep)
265 goto fail;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700266
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700267 return 0;
268fail:
269 f2fs_exit_crypto();
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700270 return res;
271}
272
273void f2fs_restore_and_release_control_page(struct page **page)
274{
275 struct f2fs_crypto_ctx *ctx;
276 struct page *bounce_page;
277
278 /* The bounce data pages are unmapped. */
279 if ((*page)->mapping)
280 return;
281
282 /* The bounce data page is unmapped. */
283 bounce_page = *page;
284 ctx = (struct f2fs_crypto_ctx *)page_private(bounce_page);
285
286 /* restore control page */
Jaegeuk Kimca40b032015-05-12 13:40:20 -0700287 *page = ctx->w.control_page;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700288
289 f2fs_restore_control_page(bounce_page);
290}
291
292void f2fs_restore_control_page(struct page *data_page)
293{
294 struct f2fs_crypto_ctx *ctx =
295 (struct f2fs_crypto_ctx *)page_private(data_page);
296
297 set_page_private(data_page, (unsigned long)NULL);
298 ClearPagePrivate(data_page);
299 unlock_page(data_page);
300 f2fs_release_crypto_ctx(ctx);
301}
302
303/**
304 * f2fs_crypt_complete() - The completion callback for page encryption
305 * @req: The asynchronous encryption request context
306 * @res: The result of the encryption operation
307 */
308static void f2fs_crypt_complete(struct crypto_async_request *req, int res)
309{
310 struct f2fs_completion_result *ecr = req->data;
311
312 if (res == -EINPROGRESS)
313 return;
314 ecr->res = res;
315 complete(&ecr->completion);
316}
317
318typedef enum {
319 F2FS_DECRYPT = 0,
320 F2FS_ENCRYPT,
321} f2fs_direction_t;
322
323static int f2fs_page_crypto(struct f2fs_crypto_ctx *ctx,
324 struct inode *inode,
325 f2fs_direction_t rw,
326 pgoff_t index,
327 struct page *src_page,
328 struct page *dest_page)
329{
330 u8 xts_tweak[F2FS_XTS_TWEAK_SIZE];
331 struct ablkcipher_request *req = NULL;
332 DECLARE_F2FS_COMPLETION_RESULT(ecr);
333 struct scatterlist dst, src;
Jaegeuk Kim26bf3dc2015-05-19 22:26:54 -0700334 struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info;
335 struct crypto_ablkcipher *tfm = ci->ci_ctfm;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700336 int res = 0;
337
Jaegeuk Kim26bf3dc2015-05-19 22:26:54 -0700338 req = ablkcipher_request_alloc(tfm, GFP_NOFS);
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700339 if (!req) {
340 printk_ratelimited(KERN_ERR
341 "%s: crypto_request_alloc() failed\n",
342 __func__);
343 return -ENOMEM;
344 }
345 ablkcipher_request_set_callback(
346 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
347 f2fs_crypt_complete, &ecr);
348
349 BUILD_BUG_ON(F2FS_XTS_TWEAK_SIZE < sizeof(index));
350 memcpy(xts_tweak, &index, sizeof(index));
351 memset(&xts_tweak[sizeof(index)], 0,
352 F2FS_XTS_TWEAK_SIZE - sizeof(index));
353
354 sg_init_table(&dst, 1);
355 sg_set_page(&dst, dest_page, PAGE_CACHE_SIZE, 0);
356 sg_init_table(&src, 1);
357 sg_set_page(&src, src_page, PAGE_CACHE_SIZE, 0);
358 ablkcipher_request_set_crypt(req, &src, &dst, PAGE_CACHE_SIZE,
359 xts_tweak);
360 if (rw == F2FS_DECRYPT)
361 res = crypto_ablkcipher_decrypt(req);
362 else
363 res = crypto_ablkcipher_encrypt(req);
364 if (res == -EINPROGRESS || res == -EBUSY) {
365 BUG_ON(req->base.data != &ecr);
366 wait_for_completion(&ecr.completion);
367 res = ecr.res;
368 }
369 ablkcipher_request_free(req);
370 if (res) {
371 printk_ratelimited(KERN_ERR
372 "%s: crypto_ablkcipher_encrypt() returned %d\n",
373 __func__, res);
374 return res;
375 }
376 return 0;
377}
378
Jaegeuk Kim43f54cd2015-06-11 14:33:29 -0700379static struct page *alloc_bounce_page(struct f2fs_crypto_ctx *ctx)
380{
381 ctx->w.bounce_page = mempool_alloc(f2fs_bounce_page_pool, GFP_NOWAIT);
382 if (ctx->w.bounce_page == NULL)
383 return ERR_PTR(-ENOMEM);
384 ctx->flags |= F2FS_WRITE_PATH_FL;
385 return ctx->w.bounce_page;
386}
387
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700388/**
389 * f2fs_encrypt() - Encrypts a page
390 * @inode: The inode for which the encryption should take place
391 * @plaintext_page: The page to encrypt. Must be locked.
392 *
393 * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
394 * encryption context.
395 *
396 * Called on the page write path. The caller must call
397 * f2fs_restore_control_page() on the returned ciphertext page to
398 * release the bounce buffer and the encryption context.
399 *
400 * Return: An allocated page with the encrypted content on success. Else, an
401 * error value or NULL.
402 */
403struct page *f2fs_encrypt(struct inode *inode,
404 struct page *plaintext_page)
405{
406 struct f2fs_crypto_ctx *ctx;
407 struct page *ciphertext_page = NULL;
408 int err;
409
410 BUG_ON(!PageLocked(plaintext_page));
411
412 ctx = f2fs_get_crypto_ctx(inode);
413 if (IS_ERR(ctx))
414 return (struct page *)ctx;
415
416 /* The encryption operation will require a bounce page. */
Jaegeuk Kim43f54cd2015-06-11 14:33:29 -0700417 ciphertext_page = alloc_bounce_page(ctx);
418 if (IS_ERR(ciphertext_page))
Jaegeuk Kim4683ff82015-06-01 12:39:30 -0700419 goto err_out;
Jaegeuk Kim4683ff82015-06-01 12:39:30 -0700420
Jaegeuk Kimca40b032015-05-12 13:40:20 -0700421 ctx->w.control_page = plaintext_page;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700422 err = f2fs_page_crypto(ctx, inode, F2FS_ENCRYPT, plaintext_page->index,
423 plaintext_page, ciphertext_page);
Jaegeuk Kim43f54cd2015-06-11 14:33:29 -0700424 if (err) {
425 ciphertext_page = ERR_PTR(err);
Jaegeuk Kim4683ff82015-06-01 12:39:30 -0700426 goto err_out;
Jaegeuk Kim43f54cd2015-06-11 14:33:29 -0700427 }
Jaegeuk Kim4683ff82015-06-01 12:39:30 -0700428
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700429 SetPagePrivate(ciphertext_page);
430 set_page_private(ciphertext_page, (unsigned long)ctx);
431 lock_page(ciphertext_page);
432 return ciphertext_page;
Jaegeuk Kim4683ff82015-06-01 12:39:30 -0700433
434err_out:
435 f2fs_release_crypto_ctx(ctx);
Jaegeuk Kim43f54cd2015-06-11 14:33:29 -0700436 return ciphertext_page;
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700437}
438
439/**
440 * f2fs_decrypt() - Decrypts a page in-place
441 * @ctx: The encryption context.
442 * @page: The page to decrypt. Must be locked.
443 *
444 * Decrypts page in-place using the ctx encryption context.
445 *
446 * Called from the read completion callback.
447 *
448 * Return: Zero on success, non-zero otherwise.
449 */
450int f2fs_decrypt(struct f2fs_crypto_ctx *ctx, struct page *page)
451{
452 BUG_ON(!PageLocked(page));
453
454 return f2fs_page_crypto(ctx, page->mapping->host,
455 F2FS_DECRYPT, page->index, page, page);
456}
457
458/*
459 * Convenience function which takes care of allocating and
460 * deallocating the encryption context
461 */
462int f2fs_decrypt_one(struct inode *inode, struct page *page)
463{
464 struct f2fs_crypto_ctx *ctx = f2fs_get_crypto_ctx(inode);
465 int ret;
466
Jaegeuk Kim7e8e7542015-06-11 14:23:05 -0700467 if (IS_ERR(ctx))
468 return PTR_ERR(ctx);
Jaegeuk Kim57e50552015-04-20 19:52:47 -0700469 ret = f2fs_decrypt(ctx, page);
470 f2fs_release_crypto_ctx(ctx);
471 return ret;
472}
473
474bool f2fs_valid_contents_enc_mode(uint32_t mode)
475{
476 return (mode == F2FS_ENCRYPTION_MODE_AES_256_XTS);
477}
478
479/**
480 * f2fs_validate_encryption_key_size() - Validate the encryption key size
481 * @mode: The key mode.
482 * @size: The key size to validate.
483 *
484 * Return: The validated key size for @mode. Zero if invalid.
485 */
486uint32_t f2fs_validate_encryption_key_size(uint32_t mode, uint32_t size)
487{
488 if (size == f2fs_encryption_key_size(mode))
489 return size;
490 return 0;
491}