blob: 18d8e87119a6039dcb1db6fdf236e116e71e1072 [file] [log] [blame]
Seth Jennings2b281112013-07-10 16:05:03 -07001/*
2 * zswap.c - zswap driver file
3 *
4 * zswap is a backend for frontswap that takes pages that are in the process
5 * of being swapped out and attempts to compress and store them in a
6 * RAM-based memory pool. This can result in a significant I/O reduction on
7 * the swap device and, in the case where decompressing from RAM is faster
8 * than reading from the swap device, can also improve workload performance.
9 *
10 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21*/
22
23#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24
25#include <linux/module.h>
26#include <linux/cpu.h>
27#include <linux/highmem.h>
28#include <linux/slab.h>
29#include <linux/spinlock.h>
30#include <linux/types.h>
31#include <linux/atomic.h>
32#include <linux/frontswap.h>
33#include <linux/rbtree.h>
34#include <linux/swap.h>
35#include <linux/crypto.h>
36#include <linux/mempool.h>
Dan Streetman12d79d62014-08-06 16:08:40 -070037#include <linux/zpool.h>
Seth Jennings2b281112013-07-10 16:05:03 -070038
39#include <linux/mm_types.h>
40#include <linux/page-flags.h>
41#include <linux/swapops.h>
42#include <linux/writeback.h>
43#include <linux/pagemap.h>
44
45/*********************************
46* statistics
47**********************************/
Dan Streetman12d79d62014-08-06 16:08:40 -070048/* Total bytes used by the compressed storage */
49static u64 zswap_pool_total_size;
Seth Jennings2b281112013-07-10 16:05:03 -070050/* The number of compressed pages currently stored in zswap */
51static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
52
53/*
54 * The statistics below are not protected from concurrent access for
55 * performance reasons so they may not be a 100% accurate. However,
56 * they do provide useful information on roughly how many times a
57 * certain event is occurring.
58*/
59
60/* Pool limit was hit (see zswap_max_pool_percent) */
61static u64 zswap_pool_limit_hit;
62/* Pages written back when pool limit was reached */
63static u64 zswap_written_back_pages;
64/* Store failed due to a reclaim failure after pool limit was reached */
65static u64 zswap_reject_reclaim_fail;
66/* Compressed page was too big for the allocator to (optimally) store */
67static u64 zswap_reject_compress_poor;
68/* Store failed because underlying allocator could not get memory */
69static u64 zswap_reject_alloc_fail;
70/* Store failed because the entry metadata could not be allocated (rare) */
71static u64 zswap_reject_kmemcache_fail;
72/* Duplicate store was encountered (rare) */
73static u64 zswap_duplicate_entry;
74
75/*********************************
76* tunables
77**********************************/
Dan Streetmanc00ed162015-06-25 15:00:35 -070078
Dan Streetmanbae21db2017-02-27 14:26:50 -080079#define ZSWAP_PARAM_UNSET ""
80
Dan Streetmanc00ed162015-06-25 15:00:35 -070081/* Enable/disable zswap (disabled by default) */
82static bool zswap_enabled;
Dan Streetmand7b028f2017-02-03 13:13:09 -080083static int zswap_enabled_param_set(const char *,
84 const struct kernel_param *);
85static struct kernel_param_ops zswap_enabled_param_ops = {
86 .set = zswap_enabled_param_set,
87 .get = param_get_bool,
88};
89module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
Seth Jennings2b281112013-07-10 16:05:03 -070090
Dan Streetman90b0fc22015-09-09 15:35:21 -070091/* Crypto compressor to use */
Seth Jennings2b281112013-07-10 16:05:03 -070092#define ZSWAP_COMPRESSOR_DEFAULT "lzo"
Dan Streetmanc99b42c2015-11-06 16:29:15 -080093static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
Dan Streetman90b0fc22015-09-09 15:35:21 -070094static int zswap_compressor_param_set(const char *,
95 const struct kernel_param *);
96static struct kernel_param_ops zswap_compressor_param_ops = {
97 .set = zswap_compressor_param_set,
Dan Streetmanc99b42c2015-11-06 16:29:15 -080098 .get = param_get_charp,
99 .free = param_free_charp,
Dan Streetman90b0fc22015-09-09 15:35:21 -0700100};
101module_param_cb(compressor, &zswap_compressor_param_ops,
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800102 &zswap_compressor, 0644);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700103
104/* Compressed storage zpool to use */
105#define ZSWAP_ZPOOL_DEFAULT "zbud"
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800106static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700107static int zswap_zpool_param_set(const char *, const struct kernel_param *);
108static struct kernel_param_ops zswap_zpool_param_ops = {
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800109 .set = zswap_zpool_param_set,
110 .get = param_get_charp,
111 .free = param_free_charp,
Dan Streetman90b0fc22015-09-09 15:35:21 -0700112};
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800113module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
Seth Jennings2b281112013-07-10 16:05:03 -0700114
115/* The maximum percentage of memory that the compressed pool can occupy */
116static unsigned int zswap_max_pool_percent = 20;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700117module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
Minchan Kim60105e12014-04-07 15:38:27 -0700118
Seth Jennings2b281112013-07-10 16:05:03 -0700119/*********************************
Seth Jennings2b281112013-07-10 16:05:03 -0700120* data structures
121**********************************/
Dan Streetmanf1c54842015-09-09 15:35:19 -0700122
123struct zswap_pool {
124 struct zpool *zpool;
125 struct crypto_comp * __percpu *tfm;
126 struct kref kref;
127 struct list_head list;
Dan Streetman200867a2016-05-20 16:59:54 -0700128 struct work_struct work;
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100129 struct hlist_node node;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700130 char tfm_name[CRYPTO_MAX_ALG_NAME];
131};
132
Seth Jennings2b281112013-07-10 16:05:03 -0700133/*
134 * struct zswap_entry
135 *
136 * This structure contains the metadata for tracking a single compressed
137 * page within zswap.
138 *
139 * rbnode - links the entry into red-black tree for the appropriate swap type
Dan Streetmanf1c54842015-09-09 15:35:19 -0700140 * offset - the swap offset for the entry. Index into the red-black tree.
Seth Jennings2b281112013-07-10 16:05:03 -0700141 * refcount - the number of outstanding reference to the entry. This is needed
142 * to protect against premature freeing of the entry by code
SeongJae Park6b452512014-04-07 15:38:25 -0700143 * concurrent calls to load, invalidate, and writeback. The lock
Seth Jennings2b281112013-07-10 16:05:03 -0700144 * for the zswap_tree structure that contains the entry must
145 * be held while changing the refcount. Since the lock must
146 * be held, there is no reason to also make refcount atomic.
Seth Jennings2b281112013-07-10 16:05:03 -0700147 * length - the length in bytes of the compressed page data. Needed during
SeongJae Park6b452512014-04-07 15:38:25 -0700148 * decompression
Dan Streetmanf1c54842015-09-09 15:35:19 -0700149 * pool - the zswap_pool the entry's data is in
150 * handle - zpool allocation handle that stores the compressed page data
Seth Jennings2b281112013-07-10 16:05:03 -0700151 */
152struct zswap_entry {
153 struct rb_node rbnode;
154 pgoff_t offset;
155 int refcount;
156 unsigned int length;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700157 struct zswap_pool *pool;
Seth Jennings2b281112013-07-10 16:05:03 -0700158 unsigned long handle;
159};
160
161struct zswap_header {
162 swp_entry_t swpentry;
163};
164
165/*
166 * The tree lock in the zswap_tree struct protects a few things:
167 * - the rbtree
168 * - the refcount field of each entry in the tree
169 */
170struct zswap_tree {
171 struct rb_root rbroot;
172 spinlock_t lock;
Seth Jennings2b281112013-07-10 16:05:03 -0700173};
174
175static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
176
Dan Streetmanf1c54842015-09-09 15:35:19 -0700177/* RCU-protected iteration */
178static LIST_HEAD(zswap_pools);
179/* protects zswap_pools list modification */
180static DEFINE_SPINLOCK(zswap_pools_lock);
Dan Streetman32a4e1692016-05-05 16:22:23 -0700181/* pool counter to provide unique names to zpool */
182static atomic_t zswap_pools_count = ATOMIC_INIT(0);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700183
Dan Streetman90b0fc22015-09-09 15:35:21 -0700184/* used by param callback function */
185static bool zswap_init_started;
186
Dan Streetmand7b028f2017-02-03 13:13:09 -0800187/* fatal error during init */
188static bool zswap_init_failed;
189
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800190/* init completed, but couldn't create the initial pool */
191static bool zswap_has_pool;
192
Dan Streetmanf1c54842015-09-09 15:35:19 -0700193/*********************************
194* helpers and fwd declarations
195**********************************/
196
197#define zswap_pool_debug(msg, p) \
198 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
199 zpool_get_type((p)->zpool))
200
201static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
202static int zswap_pool_get(struct zswap_pool *pool);
203static void zswap_pool_put(struct zswap_pool *pool);
204
205static const struct zpool_ops zswap_zpool_ops = {
206 .evict = zswap_writeback_entry
207};
208
209static bool zswap_is_full(void)
210{
211 return totalram_pages * zswap_max_pool_percent / 100 <
212 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
213}
214
215static void zswap_update_total_size(void)
216{
217 struct zswap_pool *pool;
218 u64 total = 0;
219
220 rcu_read_lock();
221
222 list_for_each_entry_rcu(pool, &zswap_pools, list)
223 total += zpool_get_total_size(pool->zpool);
224
225 rcu_read_unlock();
226
227 zswap_pool_total_size = total;
228}
229
Seth Jennings2b281112013-07-10 16:05:03 -0700230/*********************************
231* zswap entry functions
232**********************************/
233static struct kmem_cache *zswap_entry_cache;
234
Mahendran Ganeshdd01d7d2014-12-12 16:57:15 -0800235static int __init zswap_entry_cache_create(void)
Seth Jennings2b281112013-07-10 16:05:03 -0700236{
237 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
SeongJae Park5d2d42d2014-04-07 15:38:28 -0700238 return zswap_entry_cache == NULL;
Seth Jennings2b281112013-07-10 16:05:03 -0700239}
240
Fabian Frederickc1192392014-08-08 14:19:35 -0700241static void __init zswap_entry_cache_destroy(void)
Seth Jennings2b281112013-07-10 16:05:03 -0700242{
243 kmem_cache_destroy(zswap_entry_cache);
244}
245
246static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
247{
248 struct zswap_entry *entry;
249 entry = kmem_cache_alloc(zswap_entry_cache, gfp);
250 if (!entry)
251 return NULL;
252 entry->refcount = 1;
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800253 RB_CLEAR_NODE(&entry->rbnode);
Seth Jennings2b281112013-07-10 16:05:03 -0700254 return entry;
255}
256
257static void zswap_entry_cache_free(struct zswap_entry *entry)
258{
259 kmem_cache_free(zswap_entry_cache, entry);
260}
261
Seth Jennings2b281112013-07-10 16:05:03 -0700262/*********************************
263* rbtree functions
264**********************************/
265static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
266{
267 struct rb_node *node = root->rb_node;
268 struct zswap_entry *entry;
269
270 while (node) {
271 entry = rb_entry(node, struct zswap_entry, rbnode);
272 if (entry->offset > offset)
273 node = node->rb_left;
274 else if (entry->offset < offset)
275 node = node->rb_right;
276 else
277 return entry;
278 }
279 return NULL;
280}
281
282/*
283 * In the case that a entry with the same offset is found, a pointer to
284 * the existing entry is stored in dupentry and the function returns -EEXIST
285 */
286static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
287 struct zswap_entry **dupentry)
288{
289 struct rb_node **link = &root->rb_node, *parent = NULL;
290 struct zswap_entry *myentry;
291
292 while (*link) {
293 parent = *link;
294 myentry = rb_entry(parent, struct zswap_entry, rbnode);
295 if (myentry->offset > entry->offset)
296 link = &(*link)->rb_left;
297 else if (myentry->offset < entry->offset)
298 link = &(*link)->rb_right;
299 else {
300 *dupentry = myentry;
301 return -EEXIST;
302 }
303 }
304 rb_link_node(&entry->rbnode, parent, link);
305 rb_insert_color(&entry->rbnode, root);
306 return 0;
307}
308
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800309static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
310{
311 if (!RB_EMPTY_NODE(&entry->rbnode)) {
312 rb_erase(&entry->rbnode, root);
313 RB_CLEAR_NODE(&entry->rbnode);
314 }
315}
316
317/*
Dan Streetman12d79d62014-08-06 16:08:40 -0700318 * Carries out the common pattern of freeing and entry's zpool allocation,
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800319 * freeing the entry itself, and decrementing the number of stored pages.
320 */
Minchan Kim60105e12014-04-07 15:38:27 -0700321static void zswap_free_entry(struct zswap_entry *entry)
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800322{
Dan Streetmanf1c54842015-09-09 15:35:19 -0700323 zpool_free(entry->pool->zpool, entry->handle);
324 zswap_pool_put(entry->pool);
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800325 zswap_entry_cache_free(entry);
326 atomic_dec(&zswap_stored_pages);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700327 zswap_update_total_size();
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800328}
329
330/* caller must hold the tree lock */
331static void zswap_entry_get(struct zswap_entry *entry)
332{
333 entry->refcount++;
334}
335
336/* caller must hold the tree lock
337* remove from the tree and free it, if nobody reference the entry
338*/
339static void zswap_entry_put(struct zswap_tree *tree,
340 struct zswap_entry *entry)
341{
342 int refcount = --entry->refcount;
343
344 BUG_ON(refcount < 0);
345 if (refcount == 0) {
346 zswap_rb_erase(&tree->rbroot, entry);
Minchan Kim60105e12014-04-07 15:38:27 -0700347 zswap_free_entry(entry);
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800348 }
349}
350
351/* caller must hold the tree lock */
352static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
353 pgoff_t offset)
354{
Alexey Klimovb0c98652015-11-06 16:29:09 -0800355 struct zswap_entry *entry;
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800356
357 entry = zswap_rb_search(root, offset);
358 if (entry)
359 zswap_entry_get(entry);
360
361 return entry;
362}
363
Seth Jennings2b281112013-07-10 16:05:03 -0700364/*********************************
365* per-cpu code
366**********************************/
367static DEFINE_PER_CPU(u8 *, zswap_dstmem);
368
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100369static int zswap_dstmem_prepare(unsigned int cpu)
Seth Jennings2b281112013-07-10 16:05:03 -0700370{
Seth Jennings2b281112013-07-10 16:05:03 -0700371 u8 *dst;
372
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100373 dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
374 if (!dst) {
375 pr_err("can't allocate compressor buffer\n");
376 return -ENOMEM;
Seth Jennings2b281112013-07-10 16:05:03 -0700377 }
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100378 per_cpu(zswap_dstmem, cpu) = dst;
Seth Jennings2b281112013-07-10 16:05:03 -0700379 return 0;
Seth Jennings2b281112013-07-10 16:05:03 -0700380}
381
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100382static int zswap_dstmem_dead(unsigned int cpu)
Seth Jennings2b281112013-07-10 16:05:03 -0700383{
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100384 u8 *dst;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700385
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +0100386 dst = per_cpu(zswap_dstmem, cpu);
387 kfree(dst);
388 per_cpu(zswap_dstmem, cpu) = NULL;
389
390 return 0;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700391}
392
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100393static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700394{
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100395 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700396 struct crypto_comp *tfm;
397
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100398 if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
399 return 0;
400
401 tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
402 if (IS_ERR_OR_NULL(tfm)) {
403 pr_err("could not alloc crypto comp %s : %ld\n",
404 pool->tfm_name, PTR_ERR(tfm));
405 return -ENOMEM;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700406 }
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100407 *per_cpu_ptr(pool->tfm, cpu) = tfm;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700408 return 0;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700409}
410
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100411static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700412{
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100413 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
414 struct crypto_comp *tfm;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700415
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100416 tfm = *per_cpu_ptr(pool->tfm, cpu);
417 if (!IS_ERR_OR_NULL(tfm))
418 crypto_free_comp(tfm);
419 *per_cpu_ptr(pool->tfm, cpu) = NULL;
420 return 0;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700421}
422
423/*********************************
424* pool functions
425**********************************/
426
427static struct zswap_pool *__zswap_pool_current(void)
428{
429 struct zswap_pool *pool;
430
431 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800432 WARN_ONCE(!pool && zswap_has_pool,
433 "%s: no page storage pool!\n", __func__);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700434
435 return pool;
436}
437
438static struct zswap_pool *zswap_pool_current(void)
439{
440 assert_spin_locked(&zswap_pools_lock);
441
442 return __zswap_pool_current();
443}
444
445static struct zswap_pool *zswap_pool_current_get(void)
446{
447 struct zswap_pool *pool;
448
449 rcu_read_lock();
450
451 pool = __zswap_pool_current();
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800452 if (!zswap_pool_get(pool))
Dan Streetmanf1c54842015-09-09 15:35:19 -0700453 pool = NULL;
454
455 rcu_read_unlock();
456
457 return pool;
458}
459
460static struct zswap_pool *zswap_pool_last_get(void)
461{
462 struct zswap_pool *pool, *last = NULL;
463
464 rcu_read_lock();
465
466 list_for_each_entry_rcu(pool, &zswap_pools, list)
467 last = pool;
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800468 WARN_ONCE(!last && zswap_has_pool,
469 "%s: no page storage pool!\n", __func__);
470 if (!zswap_pool_get(last))
Dan Streetmanf1c54842015-09-09 15:35:19 -0700471 last = NULL;
472
473 rcu_read_unlock();
474
475 return last;
476}
477
Dan Streetman8bc8b222015-12-18 14:22:04 -0800478/* type and compressor must be null-terminated */
Dan Streetmanf1c54842015-09-09 15:35:19 -0700479static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
480{
481 struct zswap_pool *pool;
482
483 assert_spin_locked(&zswap_pools_lock);
484
485 list_for_each_entry_rcu(pool, &zswap_pools, list) {
Dan Streetman8bc8b222015-12-18 14:22:04 -0800486 if (strcmp(pool->tfm_name, compressor))
Dan Streetmanf1c54842015-09-09 15:35:19 -0700487 continue;
Dan Streetman8bc8b222015-12-18 14:22:04 -0800488 if (strcmp(zpool_get_type(pool->zpool), type))
Dan Streetmanf1c54842015-09-09 15:35:19 -0700489 continue;
490 /* if we can't get it, it's about to be destroyed */
491 if (!zswap_pool_get(pool))
492 continue;
493 return pool;
494 }
495
496 return NULL;
497}
498
499static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
500{
501 struct zswap_pool *pool;
Dan Streetman32a4e1692016-05-05 16:22:23 -0700502 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
Mel Gormand0164ad2015-11-06 16:28:21 -0800503 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100504 int ret;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700505
Dan Streetmanbae21db2017-02-27 14:26:50 -0800506 if (!zswap_has_pool) {
507 /* if either are unset, pool initialization failed, and we
508 * need both params to be set correctly before trying to
509 * create a pool.
510 */
511 if (!strcmp(type, ZSWAP_PARAM_UNSET))
512 return NULL;
513 if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
514 return NULL;
515 }
516
Dan Streetmanf1c54842015-09-09 15:35:19 -0700517 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
Markus Elfringf4ae0ce2017-07-06 15:40:34 -0700518 if (!pool)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700519 return NULL;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700520
Dan Streetman32a4e1692016-05-05 16:22:23 -0700521 /* unique name for each pool specifically required by zsmalloc */
522 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
523
524 pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700525 if (!pool->zpool) {
526 pr_err("%s zpool not available\n", type);
527 goto error;
528 }
529 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
530
531 strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
532 pool->tfm = alloc_percpu(struct crypto_comp *);
533 if (!pool->tfm) {
534 pr_err("percpu alloc failed\n");
535 goto error;
536 }
537
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100538 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
539 &pool->node);
540 if (ret)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700541 goto error;
542 pr_debug("using %s compressor\n", pool->tfm_name);
543
544 /* being the current pool takes 1 ref; this func expects the
545 * caller to always add the new pool as the current pool
546 */
547 kref_init(&pool->kref);
548 INIT_LIST_HEAD(&pool->list);
549
550 zswap_pool_debug("created", pool);
551
552 return pool;
553
554error:
555 free_percpu(pool->tfm);
556 if (pool->zpool)
557 zpool_destroy_pool(pool->zpool);
558 kfree(pool);
559 return NULL;
560}
561
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800562static __init struct zswap_pool *__zswap_pool_create_fallback(void)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700563{
Dan Streetmanbae21db2017-02-27 14:26:50 -0800564 bool has_comp, has_zpool;
565
566 has_comp = crypto_has_comp(zswap_compressor, 0, 0);
567 if (!has_comp && strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
Dan Streetmanf1c54842015-09-09 15:35:19 -0700568 pr_err("compressor %s not available, using default %s\n",
569 zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800570 param_free_charp(&zswap_compressor);
571 zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
Dan Streetmanbae21db2017-02-27 14:26:50 -0800572 has_comp = crypto_has_comp(zswap_compressor, 0, 0);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700573 }
Dan Streetmanbae21db2017-02-27 14:26:50 -0800574 if (!has_comp) {
575 pr_err("default compressor %s not available\n",
576 zswap_compressor);
577 param_free_charp(&zswap_compressor);
578 zswap_compressor = ZSWAP_PARAM_UNSET;
579 }
580
581 has_zpool = zpool_has_pool(zswap_zpool_type);
582 if (!has_zpool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
Dan Streetmanf1c54842015-09-09 15:35:19 -0700583 pr_err("zpool %s not available, using default %s\n",
584 zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800585 param_free_charp(&zswap_zpool_type);
586 zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
Dan Streetmanbae21db2017-02-27 14:26:50 -0800587 has_zpool = zpool_has_pool(zswap_zpool_type);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700588 }
Dan Streetmanbae21db2017-02-27 14:26:50 -0800589 if (!has_zpool) {
590 pr_err("default zpool %s not available\n",
591 zswap_zpool_type);
592 param_free_charp(&zswap_zpool_type);
593 zswap_zpool_type = ZSWAP_PARAM_UNSET;
594 }
595
596 if (!has_comp || !has_zpool)
597 return NULL;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700598
599 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
600}
601
602static void zswap_pool_destroy(struct zswap_pool *pool)
603{
604 zswap_pool_debug("destroying", pool);
605
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +0100606 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700607 free_percpu(pool->tfm);
608 zpool_destroy_pool(pool->zpool);
609 kfree(pool);
610}
611
612static int __must_check zswap_pool_get(struct zswap_pool *pool)
613{
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800614 if (!pool)
615 return 0;
616
Dan Streetmanf1c54842015-09-09 15:35:19 -0700617 return kref_get_unless_zero(&pool->kref);
618}
619
Dan Streetman200867a2016-05-20 16:59:54 -0700620static void __zswap_pool_release(struct work_struct *work)
Dan Streetmanf1c54842015-09-09 15:35:19 -0700621{
Dan Streetman200867a2016-05-20 16:59:54 -0700622 struct zswap_pool *pool = container_of(work, typeof(*pool), work);
623
624 synchronize_rcu();
Dan Streetmanf1c54842015-09-09 15:35:19 -0700625
626 /* nobody should have been able to get a kref... */
627 WARN_ON(kref_get_unless_zero(&pool->kref));
628
629 /* pool is now off zswap_pools list and has no references. */
630 zswap_pool_destroy(pool);
631}
632
633static void __zswap_pool_empty(struct kref *kref)
634{
635 struct zswap_pool *pool;
636
637 pool = container_of(kref, typeof(*pool), kref);
638
639 spin_lock(&zswap_pools_lock);
640
641 WARN_ON(pool == zswap_pool_current());
642
643 list_del_rcu(&pool->list);
Dan Streetman200867a2016-05-20 16:59:54 -0700644
645 INIT_WORK(&pool->work, __zswap_pool_release);
646 schedule_work(&pool->work);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700647
648 spin_unlock(&zswap_pools_lock);
649}
650
651static void zswap_pool_put(struct zswap_pool *pool)
652{
653 kref_put(&pool->kref, __zswap_pool_empty);
Seth Jennings2b281112013-07-10 16:05:03 -0700654}
655
Seth Jennings2b281112013-07-10 16:05:03 -0700656/*********************************
Dan Streetman90b0fc22015-09-09 15:35:21 -0700657* param callbacks
658**********************************/
659
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800660/* val must be a null-terminated string */
Dan Streetman90b0fc22015-09-09 15:35:21 -0700661static int __zswap_param_set(const char *val, const struct kernel_param *kp,
662 char *type, char *compressor)
663{
664 struct zswap_pool *pool, *put_pool = NULL;
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800665 char *s = strstrip((char *)val);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700666 int ret;
667
Dan Streetmand7b028f2017-02-03 13:13:09 -0800668 if (zswap_init_failed) {
669 pr_err("can't set param, initialization failed\n");
670 return -ENODEV;
671 }
672
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800673 /* no change required */
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800674 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800675 return 0;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700676
677 /* if this is load-time (pre-init) param setting,
678 * don't create a pool; that's done during init.
679 */
680 if (!zswap_init_started)
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800681 return param_set_charp(s, kp);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700682
683 if (!type) {
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800684 if (!zpool_has_pool(s)) {
685 pr_err("zpool %s not available\n", s);
686 return -ENOENT;
687 }
Dan Streetman90b0fc22015-09-09 15:35:21 -0700688 type = s;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700689 } else if (!compressor) {
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800690 if (!crypto_has_comp(s, 0, 0)) {
691 pr_err("compressor %s not available\n", s);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700692 return -ENOENT;
693 }
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800694 compressor = s;
695 } else {
696 WARN_ON(1);
697 return -EINVAL;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700698 }
699
700 spin_lock(&zswap_pools_lock);
701
702 pool = zswap_pool_find_get(type, compressor);
703 if (pool) {
704 zswap_pool_debug("using existing", pool);
Dan Streetmanfd5bb662017-02-27 14:26:53 -0800705 WARN_ON(pool == zswap_pool_current());
Dan Streetman90b0fc22015-09-09 15:35:21 -0700706 list_del_rcu(&pool->list);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700707 }
708
Dan Streetmanfd5bb662017-02-27 14:26:53 -0800709 spin_unlock(&zswap_pools_lock);
710
711 if (!pool)
712 pool = zswap_pool_create(type, compressor);
713
Dan Streetman90b0fc22015-09-09 15:35:21 -0700714 if (pool)
Dan Streetmanc99b42c2015-11-06 16:29:15 -0800715 ret = param_set_charp(s, kp);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700716 else
717 ret = -EINVAL;
718
Dan Streetmanfd5bb662017-02-27 14:26:53 -0800719 spin_lock(&zswap_pools_lock);
720
Dan Streetman90b0fc22015-09-09 15:35:21 -0700721 if (!ret) {
722 put_pool = zswap_pool_current();
723 list_add_rcu(&pool->list, &zswap_pools);
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800724 zswap_has_pool = true;
Dan Streetman90b0fc22015-09-09 15:35:21 -0700725 } else if (pool) {
726 /* add the possibly pre-existing pool to the end of the pools
727 * list; if it's new (and empty) then it'll be removed and
728 * destroyed by the put after we drop the lock
729 */
730 list_add_tail_rcu(&pool->list, &zswap_pools);
731 put_pool = pool;
Dan Streetmanfd5bb662017-02-27 14:26:53 -0800732 }
733
734 spin_unlock(&zswap_pools_lock);
735
736 if (!zswap_has_pool && !pool) {
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800737 /* if initial pool creation failed, and this pool creation also
738 * failed, maybe both compressor and zpool params were bad.
739 * Allow changing this param, so pool creation will succeed
740 * when the other param is changed. We already verified this
741 * param is ok in the zpool_has_pool() or crypto_has_comp()
742 * checks above.
743 */
744 ret = param_set_charp(s, kp);
Dan Streetman90b0fc22015-09-09 15:35:21 -0700745 }
746
Dan Streetman90b0fc22015-09-09 15:35:21 -0700747 /* drop the ref from either the old current pool,
748 * or the new pool we failed to add
749 */
750 if (put_pool)
751 zswap_pool_put(put_pool);
752
753 return ret;
754}
755
756static int zswap_compressor_param_set(const char *val,
757 const struct kernel_param *kp)
758{
759 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
760}
761
762static int zswap_zpool_param_set(const char *val,
763 const struct kernel_param *kp)
764{
765 return __zswap_param_set(val, kp, NULL, zswap_compressor);
766}
767
Dan Streetmand7b028f2017-02-03 13:13:09 -0800768static int zswap_enabled_param_set(const char *val,
769 const struct kernel_param *kp)
770{
771 if (zswap_init_failed) {
772 pr_err("can't enable, initialization failed\n");
773 return -ENODEV;
774 }
Dan Streetmanae3d89a2017-02-27 14:26:47 -0800775 if (!zswap_has_pool && zswap_init_started) {
776 pr_err("can't enable, no pool configured\n");
777 return -ENODEV;
778 }
Dan Streetmand7b028f2017-02-03 13:13:09 -0800779
780 return param_set_bool(val, kp);
781}
782
Dan Streetman90b0fc22015-09-09 15:35:21 -0700783/*********************************
Seth Jennings2b281112013-07-10 16:05:03 -0700784* writeback code
785**********************************/
786/* return enum for zswap_get_swap_cache_page */
787enum zswap_get_swap_ret {
788 ZSWAP_SWAPCACHE_NEW,
789 ZSWAP_SWAPCACHE_EXIST,
Weijie Yang67d13fe2013-11-12 15:08:26 -0800790 ZSWAP_SWAPCACHE_FAIL,
Seth Jennings2b281112013-07-10 16:05:03 -0700791};
792
793/*
794 * zswap_get_swap_cache_page
795 *
796 * This is an adaption of read_swap_cache_async()
797 *
798 * This function tries to find a page with the given swap entry
799 * in the swapper_space address space (the swap cache). If the page
800 * is found, it is returned in retpage. Otherwise, a page is allocated,
801 * added to the swap cache, and returned in retpage.
802 *
803 * If success, the swap cache page is returned in retpage
Weijie Yang67d13fe2013-11-12 15:08:26 -0800804 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
805 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
806 * the new page is added to swapcache and locked
807 * Returns ZSWAP_SWAPCACHE_FAIL on error
Seth Jennings2b281112013-07-10 16:05:03 -0700808 */
809static int zswap_get_swap_cache_page(swp_entry_t entry,
810 struct page **retpage)
811{
Dmitry Safonov5b999aa2015-09-08 15:05:00 -0700812 bool page_was_allocated;
Seth Jennings2b281112013-07-10 16:05:03 -0700813
Dmitry Safonov5b999aa2015-09-08 15:05:00 -0700814 *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
815 NULL, 0, &page_was_allocated);
816 if (page_was_allocated)
817 return ZSWAP_SWAPCACHE_NEW;
818 if (!*retpage)
Weijie Yang67d13fe2013-11-12 15:08:26 -0800819 return ZSWAP_SWAPCACHE_FAIL;
Seth Jennings2b281112013-07-10 16:05:03 -0700820 return ZSWAP_SWAPCACHE_EXIST;
821}
822
823/*
824 * Attempts to free an entry by adding a page to the swap cache,
825 * decompressing the entry data into the page, and issuing a
826 * bio write to write the page back to the swap device.
827 *
828 * This can be thought of as a "resumed writeback" of the page
829 * to the swap device. We are basically resuming the same swap
830 * writeback path that was intercepted with the frontswap_store()
831 * in the first place. After the page has been decompressed into
832 * the swap cache, the compressed version stored by zswap can be
833 * freed.
834 */
Dan Streetman12d79d62014-08-06 16:08:40 -0700835static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
Seth Jennings2b281112013-07-10 16:05:03 -0700836{
837 struct zswap_header *zhdr;
838 swp_entry_t swpentry;
839 struct zswap_tree *tree;
840 pgoff_t offset;
841 struct zswap_entry *entry;
842 struct page *page;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700843 struct crypto_comp *tfm;
Seth Jennings2b281112013-07-10 16:05:03 -0700844 u8 *src, *dst;
845 unsigned int dlen;
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800846 int ret;
Seth Jennings2b281112013-07-10 16:05:03 -0700847 struct writeback_control wbc = {
848 .sync_mode = WB_SYNC_NONE,
849 };
850
851 /* extract swpentry from data */
Dan Streetman12d79d62014-08-06 16:08:40 -0700852 zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
Seth Jennings2b281112013-07-10 16:05:03 -0700853 swpentry = zhdr->swpentry; /* here */
Dan Streetman12d79d62014-08-06 16:08:40 -0700854 zpool_unmap_handle(pool, handle);
Seth Jennings2b281112013-07-10 16:05:03 -0700855 tree = zswap_trees[swp_type(swpentry)];
856 offset = swp_offset(swpentry);
Seth Jennings2b281112013-07-10 16:05:03 -0700857
858 /* find and ref zswap entry */
859 spin_lock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800860 entry = zswap_entry_find_get(&tree->rbroot, offset);
Seth Jennings2b281112013-07-10 16:05:03 -0700861 if (!entry) {
862 /* entry was invalidated */
863 spin_unlock(&tree->lock);
864 return 0;
865 }
Seth Jennings2b281112013-07-10 16:05:03 -0700866 spin_unlock(&tree->lock);
867 BUG_ON(offset != entry->offset);
868
869 /* try to allocate swap cache page */
870 switch (zswap_get_swap_cache_page(swpentry, &page)) {
Weijie Yang67d13fe2013-11-12 15:08:26 -0800871 case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
Seth Jennings2b281112013-07-10 16:05:03 -0700872 ret = -ENOMEM;
873 goto fail;
874
Weijie Yang67d13fe2013-11-12 15:08:26 -0800875 case ZSWAP_SWAPCACHE_EXIST:
Seth Jennings2b281112013-07-10 16:05:03 -0700876 /* page is already in the swap cache, ignore for now */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300877 put_page(page);
Seth Jennings2b281112013-07-10 16:05:03 -0700878 ret = -EEXIST;
879 goto fail;
880
881 case ZSWAP_SWAPCACHE_NEW: /* page is locked */
882 /* decompress */
883 dlen = PAGE_SIZE;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700884 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
Dan Streetman12d79d62014-08-06 16:08:40 -0700885 ZPOOL_MM_RO) + sizeof(struct zswap_header);
Seth Jennings2b281112013-07-10 16:05:03 -0700886 dst = kmap_atomic(page);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700887 tfm = *get_cpu_ptr(entry->pool->tfm);
888 ret = crypto_comp_decompress(tfm, src, entry->length,
889 dst, &dlen);
890 put_cpu_ptr(entry->pool->tfm);
Seth Jennings2b281112013-07-10 16:05:03 -0700891 kunmap_atomic(dst);
Dan Streetmanf1c54842015-09-09 15:35:19 -0700892 zpool_unmap_handle(entry->pool->zpool, entry->handle);
Seth Jennings2b281112013-07-10 16:05:03 -0700893 BUG_ON(ret);
894 BUG_ON(dlen != PAGE_SIZE);
895
896 /* page is up to date */
897 SetPageUptodate(page);
898 }
899
Weijie Yangb349acc2013-11-12 15:07:52 -0800900 /* move it to the tail of the inactive list after end_writeback */
901 SetPageReclaim(page);
902
Seth Jennings2b281112013-07-10 16:05:03 -0700903 /* start writeback */
904 __swap_writepage(page, &wbc, end_swap_bio_write);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300905 put_page(page);
Seth Jennings2b281112013-07-10 16:05:03 -0700906 zswap_written_back_pages++;
907
908 spin_lock(&tree->lock);
Seth Jennings2b281112013-07-10 16:05:03 -0700909 /* drop local reference */
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800910 zswap_entry_put(tree, entry);
Seth Jennings2b281112013-07-10 16:05:03 -0700911
912 /*
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800913 * There are two possible situations for entry here:
914 * (1) refcount is 1(normal case), entry is valid and on the tree
915 * (2) refcount is 0, entry is freed and not on the tree
916 * because invalidate happened during writeback
917 * search the tree and free the entry if find entry
918 */
919 if (entry == zswap_rb_search(&tree->rbroot, offset))
920 zswap_entry_put(tree, entry);
Seth Jennings2b281112013-07-10 16:05:03 -0700921 spin_unlock(&tree->lock);
Seth Jennings2b281112013-07-10 16:05:03 -0700922
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800923 goto end;
924
925 /*
926 * if we get here due to ZSWAP_SWAPCACHE_EXIST
927 * a load may happening concurrently
928 * it is safe and okay to not free the entry
929 * if we free the entry in the following put
930 * it it either okay to return !0
931 */
Seth Jennings2b281112013-07-10 16:05:03 -0700932fail:
933 spin_lock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800934 zswap_entry_put(tree, entry);
Seth Jennings2b281112013-07-10 16:05:03 -0700935 spin_unlock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -0800936
937end:
Seth Jennings2b281112013-07-10 16:05:03 -0700938 return ret;
939}
940
Dan Streetmanf1c54842015-09-09 15:35:19 -0700941static int zswap_shrink(void)
942{
943 struct zswap_pool *pool;
944 int ret;
945
946 pool = zswap_pool_last_get();
947 if (!pool)
948 return -ENOENT;
949
950 ret = zpool_shrink(pool->zpool, 1, NULL);
951
952 zswap_pool_put(pool);
953
954 return ret;
955}
956
Seth Jennings2b281112013-07-10 16:05:03 -0700957/*********************************
958* frontswap hooks
959**********************************/
960/* attempts to compress and store an single page */
961static int zswap_frontswap_store(unsigned type, pgoff_t offset,
962 struct page *page)
963{
964 struct zswap_tree *tree = zswap_trees[type];
965 struct zswap_entry *entry, *dupentry;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700966 struct crypto_comp *tfm;
Seth Jennings2b281112013-07-10 16:05:03 -0700967 int ret;
968 unsigned int dlen = PAGE_SIZE, len;
969 unsigned long handle;
970 char *buf;
971 u8 *src, *dst;
972 struct zswap_header *zhdr;
973
Dan Streetmanc00ed162015-06-25 15:00:35 -0700974 if (!zswap_enabled || !tree) {
Seth Jennings2b281112013-07-10 16:05:03 -0700975 ret = -ENODEV;
976 goto reject;
977 }
978
979 /* reclaim space if needed */
980 if (zswap_is_full()) {
981 zswap_pool_limit_hit++;
Dan Streetmanf1c54842015-09-09 15:35:19 -0700982 if (zswap_shrink()) {
Seth Jennings2b281112013-07-10 16:05:03 -0700983 zswap_reject_reclaim_fail++;
984 ret = -ENOMEM;
985 goto reject;
986 }
987 }
988
989 /* allocate entry */
990 entry = zswap_entry_cache_alloc(GFP_KERNEL);
991 if (!entry) {
992 zswap_reject_kmemcache_fail++;
993 ret = -ENOMEM;
994 goto reject;
995 }
996
Dan Streetmanf1c54842015-09-09 15:35:19 -0700997 /* if entry is successfully added, it keeps the reference */
998 entry->pool = zswap_pool_current_get();
999 if (!entry->pool) {
Seth Jennings2b281112013-07-10 16:05:03 -07001000 ret = -EINVAL;
1001 goto freepage;
1002 }
1003
Dan Streetmanf1c54842015-09-09 15:35:19 -07001004 /* compress */
1005 dst = get_cpu_var(zswap_dstmem);
1006 tfm = *get_cpu_ptr(entry->pool->tfm);
1007 src = kmap_atomic(page);
1008 ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
1009 kunmap_atomic(src);
1010 put_cpu_ptr(entry->pool->tfm);
1011 if (ret) {
1012 ret = -EINVAL;
1013 goto put_dstmem;
1014 }
1015
Seth Jennings2b281112013-07-10 16:05:03 -07001016 /* store */
1017 len = dlen + sizeof(struct zswap_header);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001018 ret = zpool_malloc(entry->pool->zpool, len,
Mel Gormand0164ad2015-11-06 16:28:21 -08001019 __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
1020 &handle);
Seth Jennings2b281112013-07-10 16:05:03 -07001021 if (ret == -ENOSPC) {
1022 zswap_reject_compress_poor++;
Dan Streetmanf1c54842015-09-09 15:35:19 -07001023 goto put_dstmem;
Seth Jennings2b281112013-07-10 16:05:03 -07001024 }
1025 if (ret) {
1026 zswap_reject_alloc_fail++;
Dan Streetmanf1c54842015-09-09 15:35:19 -07001027 goto put_dstmem;
Seth Jennings2b281112013-07-10 16:05:03 -07001028 }
Dan Streetmanf1c54842015-09-09 15:35:19 -07001029 zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
Seth Jennings2b281112013-07-10 16:05:03 -07001030 zhdr->swpentry = swp_entry(type, offset);
1031 buf = (u8 *)(zhdr + 1);
1032 memcpy(buf, dst, dlen);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001033 zpool_unmap_handle(entry->pool->zpool, handle);
Seth Jennings2b281112013-07-10 16:05:03 -07001034 put_cpu_var(zswap_dstmem);
1035
1036 /* populate entry */
1037 entry->offset = offset;
1038 entry->handle = handle;
1039 entry->length = dlen;
1040
1041 /* map */
1042 spin_lock(&tree->lock);
1043 do {
1044 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
1045 if (ret == -EEXIST) {
1046 zswap_duplicate_entry++;
1047 /* remove from rbtree */
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001048 zswap_rb_erase(&tree->rbroot, dupentry);
1049 zswap_entry_put(tree, dupentry);
Seth Jennings2b281112013-07-10 16:05:03 -07001050 }
1051 } while (ret == -EEXIST);
1052 spin_unlock(&tree->lock);
1053
1054 /* update stats */
1055 atomic_inc(&zswap_stored_pages);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001056 zswap_update_total_size();
Seth Jennings2b281112013-07-10 16:05:03 -07001057
1058 return 0;
1059
Dan Streetmanf1c54842015-09-09 15:35:19 -07001060put_dstmem:
Seth Jennings2b281112013-07-10 16:05:03 -07001061 put_cpu_var(zswap_dstmem);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001062 zswap_pool_put(entry->pool);
1063freepage:
Seth Jennings2b281112013-07-10 16:05:03 -07001064 zswap_entry_cache_free(entry);
1065reject:
1066 return ret;
1067}
1068
1069/*
1070 * returns 0 if the page was successfully decompressed
1071 * return -1 on entry not found or error
1072*/
1073static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1074 struct page *page)
1075{
1076 struct zswap_tree *tree = zswap_trees[type];
1077 struct zswap_entry *entry;
Dan Streetmanf1c54842015-09-09 15:35:19 -07001078 struct crypto_comp *tfm;
Seth Jennings2b281112013-07-10 16:05:03 -07001079 u8 *src, *dst;
1080 unsigned int dlen;
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001081 int ret;
Seth Jennings2b281112013-07-10 16:05:03 -07001082
1083 /* find */
1084 spin_lock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001085 entry = zswap_entry_find_get(&tree->rbroot, offset);
Seth Jennings2b281112013-07-10 16:05:03 -07001086 if (!entry) {
1087 /* entry was written back */
1088 spin_unlock(&tree->lock);
1089 return -1;
1090 }
Seth Jennings2b281112013-07-10 16:05:03 -07001091 spin_unlock(&tree->lock);
1092
1093 /* decompress */
1094 dlen = PAGE_SIZE;
Dan Streetmanf1c54842015-09-09 15:35:19 -07001095 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
Dan Streetman12d79d62014-08-06 16:08:40 -07001096 ZPOOL_MM_RO) + sizeof(struct zswap_header);
Seth Jennings2b281112013-07-10 16:05:03 -07001097 dst = kmap_atomic(page);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001098 tfm = *get_cpu_ptr(entry->pool->tfm);
1099 ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
1100 put_cpu_ptr(entry->pool->tfm);
Seth Jennings2b281112013-07-10 16:05:03 -07001101 kunmap_atomic(dst);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001102 zpool_unmap_handle(entry->pool->zpool, entry->handle);
Seth Jennings2b281112013-07-10 16:05:03 -07001103 BUG_ON(ret);
1104
1105 spin_lock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001106 zswap_entry_put(tree, entry);
Seth Jennings2b281112013-07-10 16:05:03 -07001107 spin_unlock(&tree->lock);
1108
Seth Jennings2b281112013-07-10 16:05:03 -07001109 return 0;
1110}
1111
1112/* frees an entry in zswap */
1113static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1114{
1115 struct zswap_tree *tree = zswap_trees[type];
1116 struct zswap_entry *entry;
Seth Jennings2b281112013-07-10 16:05:03 -07001117
1118 /* find */
1119 spin_lock(&tree->lock);
1120 entry = zswap_rb_search(&tree->rbroot, offset);
1121 if (!entry) {
1122 /* entry was written back */
1123 spin_unlock(&tree->lock);
1124 return;
1125 }
1126
1127 /* remove from rbtree */
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001128 zswap_rb_erase(&tree->rbroot, entry);
Seth Jennings2b281112013-07-10 16:05:03 -07001129
1130 /* drop the initial reference from entry creation */
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001131 zswap_entry_put(tree, entry);
Seth Jennings2b281112013-07-10 16:05:03 -07001132
1133 spin_unlock(&tree->lock);
Seth Jennings2b281112013-07-10 16:05:03 -07001134}
1135
1136/* frees all zswap entries for the given swap type */
1137static void zswap_frontswap_invalidate_area(unsigned type)
1138{
1139 struct zswap_tree *tree = zswap_trees[type];
Cody P Schafer0bd42132013-09-11 14:25:33 -07001140 struct zswap_entry *entry, *n;
Seth Jennings2b281112013-07-10 16:05:03 -07001141
1142 if (!tree)
1143 return;
1144
1145 /* walk the tree and free everything */
1146 spin_lock(&tree->lock);
Weijie Yang0ab0abc2013-11-12 15:08:27 -08001147 rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
Minchan Kim60105e12014-04-07 15:38:27 -07001148 zswap_free_entry(entry);
Seth Jennings2b281112013-07-10 16:05:03 -07001149 tree->rbroot = RB_ROOT;
1150 spin_unlock(&tree->lock);
Weijie Yangaa9bca02013-10-16 13:46:54 -07001151 kfree(tree);
1152 zswap_trees[type] = NULL;
Seth Jennings2b281112013-07-10 16:05:03 -07001153}
1154
Seth Jennings2b281112013-07-10 16:05:03 -07001155static void zswap_frontswap_init(unsigned type)
1156{
1157 struct zswap_tree *tree;
1158
1159 tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
Minchan Kim60105e12014-04-07 15:38:27 -07001160 if (!tree) {
1161 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1162 return;
1163 }
1164
Seth Jennings2b281112013-07-10 16:05:03 -07001165 tree->rbroot = RB_ROOT;
1166 spin_lock_init(&tree->lock);
1167 zswap_trees[type] = tree;
Seth Jennings2b281112013-07-10 16:05:03 -07001168}
1169
1170static struct frontswap_ops zswap_frontswap_ops = {
1171 .store = zswap_frontswap_store,
1172 .load = zswap_frontswap_load,
1173 .invalidate_page = zswap_frontswap_invalidate_page,
1174 .invalidate_area = zswap_frontswap_invalidate_area,
1175 .init = zswap_frontswap_init
1176};
1177
1178/*********************************
1179* debugfs functions
1180**********************************/
1181#ifdef CONFIG_DEBUG_FS
1182#include <linux/debugfs.h>
1183
1184static struct dentry *zswap_debugfs_root;
1185
1186static int __init zswap_debugfs_init(void)
1187{
1188 if (!debugfs_initialized())
1189 return -ENODEV;
1190
1191 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1192 if (!zswap_debugfs_root)
1193 return -ENOMEM;
1194
1195 debugfs_create_u64("pool_limit_hit", S_IRUGO,
1196 zswap_debugfs_root, &zswap_pool_limit_hit);
1197 debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
1198 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1199 debugfs_create_u64("reject_alloc_fail", S_IRUGO,
1200 zswap_debugfs_root, &zswap_reject_alloc_fail);
1201 debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
1202 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1203 debugfs_create_u64("reject_compress_poor", S_IRUGO,
1204 zswap_debugfs_root, &zswap_reject_compress_poor);
1205 debugfs_create_u64("written_back_pages", S_IRUGO,
1206 zswap_debugfs_root, &zswap_written_back_pages);
1207 debugfs_create_u64("duplicate_entry", S_IRUGO,
1208 zswap_debugfs_root, &zswap_duplicate_entry);
Dan Streetman12d79d62014-08-06 16:08:40 -07001209 debugfs_create_u64("pool_total_size", S_IRUGO,
1210 zswap_debugfs_root, &zswap_pool_total_size);
Seth Jennings2b281112013-07-10 16:05:03 -07001211 debugfs_create_atomic_t("stored_pages", S_IRUGO,
1212 zswap_debugfs_root, &zswap_stored_pages);
1213
1214 return 0;
1215}
1216
1217static void __exit zswap_debugfs_exit(void)
1218{
1219 debugfs_remove_recursive(zswap_debugfs_root);
1220}
1221#else
1222static int __init zswap_debugfs_init(void)
1223{
1224 return 0;
1225}
1226
1227static void __exit zswap_debugfs_exit(void) { }
1228#endif
1229
1230/*********************************
1231* module init and exit
1232**********************************/
1233static int __init init_zswap(void)
1234{
Dan Streetmanf1c54842015-09-09 15:35:19 -07001235 struct zswap_pool *pool;
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +01001236 int ret;
Minchan Kim60105e12014-04-07 15:38:27 -07001237
Dan Streetman90b0fc22015-09-09 15:35:21 -07001238 zswap_init_started = true;
1239
Seth Jennings2b281112013-07-10 16:05:03 -07001240 if (zswap_entry_cache_create()) {
1241 pr_err("entry cache creation failed\n");
Dan Streetmanf1c54842015-09-09 15:35:19 -07001242 goto cache_fail;
Seth Jennings2b281112013-07-10 16:05:03 -07001243 }
Dan Streetmanf1c54842015-09-09 15:35:19 -07001244
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +01001245 ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
1246 zswap_dstmem_prepare, zswap_dstmem_dead);
1247 if (ret) {
Dan Streetmanf1c54842015-09-09 15:35:19 -07001248 pr_err("dstmem alloc failed\n");
1249 goto dstmem_fail;
Seth Jennings2b281112013-07-10 16:05:03 -07001250 }
Dan Streetmanf1c54842015-09-09 15:35:19 -07001251
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +01001252 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1253 "mm/zswap_pool:prepare",
1254 zswap_cpu_comp_prepare,
1255 zswap_cpu_comp_dead);
1256 if (ret)
1257 goto hp_fail;
1258
Dan Streetmanf1c54842015-09-09 15:35:19 -07001259 pool = __zswap_pool_create_fallback();
Dan Streetmanae3d89a2017-02-27 14:26:47 -08001260 if (pool) {
1261 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1262 zpool_get_type(pool->zpool));
1263 list_add(&pool->list, &zswap_pools);
1264 zswap_has_pool = true;
1265 } else {
Dan Streetmanf1c54842015-09-09 15:35:19 -07001266 pr_err("pool creation failed\n");
Dan Streetmanae3d89a2017-02-27 14:26:47 -08001267 zswap_enabled = false;
Seth Jennings2b281112013-07-10 16:05:03 -07001268 }
Minchan Kim60105e12014-04-07 15:38:27 -07001269
Seth Jennings2b281112013-07-10 16:05:03 -07001270 frontswap_register_ops(&zswap_frontswap_ops);
1271 if (zswap_debugfs_init())
1272 pr_warn("debugfs initialization failed\n");
1273 return 0;
Dan Streetmanf1c54842015-09-09 15:35:19 -07001274
Sebastian Andrzej Siewiorcab7a7e2016-11-27 00:13:40 +01001275hp_fail:
Sebastian Andrzej Siewiorad7ed772016-11-27 00:13:39 +01001276 cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
Dan Streetmanf1c54842015-09-09 15:35:19 -07001277dstmem_fail:
Fabian Frederickc1192392014-08-08 14:19:35 -07001278 zswap_entry_cache_destroy();
Dan Streetmanf1c54842015-09-09 15:35:19 -07001279cache_fail:
Dan Streetmand7b028f2017-02-03 13:13:09 -08001280 /* if built-in, we aren't unloaded on failure; don't allow use */
1281 zswap_init_failed = true;
1282 zswap_enabled = false;
Seth Jennings2b281112013-07-10 16:05:03 -07001283 return -ENOMEM;
1284}
1285/* must be late so crypto has time to come up */
1286late_initcall(init_zswap);
1287
1288MODULE_LICENSE("GPL");
Seth Jennings68386da2014-11-12 21:08:46 -06001289MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
Seth Jennings2b281112013-07-10 16:05:03 -07001290MODULE_DESCRIPTION("Compressed cache for swap pages");