blob: 7f138196b3c955d2251f256f8080ece6119328c8 [file] [log] [blame]
Nitin Gupta306b0c92009-09-22 10:26:53 +05301/*
Nitin Guptaf1e3cff2010-06-01 13:31:25 +05302 * Compressed RAM block device
Nitin Gupta306b0c92009-09-22 10:26:53 +05303 *
Nitin Gupta1130ebb2010-01-28 21:21:35 +05304 * Copyright (C) 2008, 2009, 2010 Nitin Gupta
Nitin Gupta306b0c92009-09-22 10:26:53 +05305 *
6 * This code is released using a dual license strategy: BSD/GPL
7 * You can choose the licence that better fits your requirements.
8 *
9 * Released under the terms of 3-clause BSD License
10 * Released under the terms of GNU General Public License Version 2.0
11 *
12 * Project home: http://compcache.googlecode.com
13 */
14
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053015#define KMSG_COMPONENT "zram"
Nitin Gupta306b0c92009-09-22 10:26:53 +053016#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
Robert Jenningsb1f5b812011-01-28 08:59:26 -060018#ifdef CONFIG_ZRAM_DEBUG
19#define DEBUG
20#endif
21
Nitin Gupta306b0c92009-09-22 10:26:53 +053022#include <linux/module.h>
23#include <linux/kernel.h>
Randy Dunlap8946a082010-06-23 20:27:09 -070024#include <linux/bio.h>
Nitin Gupta306b0c92009-09-22 10:26:53 +053025#include <linux/bitops.h>
26#include <linux/blkdev.h>
27#include <linux/buffer_head.h>
28#include <linux/device.h>
29#include <linux/genhd.h>
30#include <linux/highmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090031#include <linux/slab.h>
Nitin Gupta306b0c92009-09-22 10:26:53 +053032#include <linux/lzo.h>
Nitin Gupta306b0c92009-09-22 10:26:53 +053033#include <linux/string.h>
Nitin Gupta306b0c92009-09-22 10:26:53 +053034#include <linux/vmalloc.h>
Nitin Gupta306b0c92009-09-22 10:26:53 +053035
Nitin Gupta16a4bfb2010-06-01 13:31:24 +053036#include "zram_drv.h"
Nitin Gupta306b0c92009-09-22 10:26:53 +053037
38/* Globals */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053039static int zram_major;
Noah Watkins43801f62011-07-20 17:05:57 -060040struct zram *zram_devices;
Nitin Gupta306b0c92009-09-22 10:26:53 +053041
Nitin Gupta306b0c92009-09-22 10:26:53 +053042/* Module params (documentation at end) */
Nitin Gupta5fa5a902012-02-12 23:04:45 -050043static unsigned int num_devices;
Nitin Gupta33863c22010-08-09 22:56:47 +053044
45static void zram_stat_inc(u32 *v)
46{
47 *v = *v + 1;
48}
49
50static void zram_stat_dec(u32 *v)
51{
52 *v = *v - 1;
53}
54
55static void zram_stat64_add(struct zram *zram, u64 *v, u64 inc)
56{
57 spin_lock(&zram->stat64_lock);
58 *v = *v + inc;
59 spin_unlock(&zram->stat64_lock);
60}
61
62static void zram_stat64_sub(struct zram *zram, u64 *v, u64 dec)
63{
64 spin_lock(&zram->stat64_lock);
65 *v = *v - dec;
66 spin_unlock(&zram->stat64_lock);
67}
68
69static void zram_stat64_inc(struct zram *zram, u64 *v)
70{
71 zram_stat64_add(zram, v, 1);
72}
Nitin Gupta306b0c92009-09-22 10:26:53 +053073
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053074static int zram_test_flag(struct zram *zram, u32 index,
75 enum zram_pageflags flag)
Nitin Gupta306b0c92009-09-22 10:26:53 +053076{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053077 return zram->table[index].flags & BIT(flag);
Nitin Gupta306b0c92009-09-22 10:26:53 +053078}
79
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053080static void zram_set_flag(struct zram *zram, u32 index,
81 enum zram_pageflags flag)
Nitin Gupta306b0c92009-09-22 10:26:53 +053082{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053083 zram->table[index].flags |= BIT(flag);
Nitin Gupta306b0c92009-09-22 10:26:53 +053084}
85
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053086static void zram_clear_flag(struct zram *zram, u32 index,
87 enum zram_pageflags flag)
Nitin Gupta306b0c92009-09-22 10:26:53 +053088{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +053089 zram->table[index].flags &= ~BIT(flag);
Nitin Gupta306b0c92009-09-22 10:26:53 +053090}
91
92static int page_zero_filled(void *ptr)
93{
94 unsigned int pos;
95 unsigned long *page;
96
97 page = (unsigned long *)ptr;
98
99 for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
100 if (page[pos])
101 return 0;
102 }
103
104 return 1;
105}
106
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530107static void zram_set_disksize(struct zram *zram, size_t totalram_bytes)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530108{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530109 if (!zram->disksize) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530110 pr_info(
111 "disk size not provided. You can use disksize_kb module "
112 "param to specify size.\nUsing default: (%u%% of RAM).\n",
113 default_disksize_perc_ram
114 );
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530115 zram->disksize = default_disksize_perc_ram *
Nitin Gupta306b0c92009-09-22 10:26:53 +0530116 (totalram_bytes / 100);
117 }
118
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530119 if (zram->disksize > 2 * (totalram_bytes)) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530120 pr_info(
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530121 "There is little point creating a zram of greater than "
Nitin Gupta306b0c92009-09-22 10:26:53 +0530122 "twice the size of memory since we expect a 2:1 compression "
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530123 "ratio. Note that zram uses about 0.1%% of the size of "
124 "the disk when not in use so a huge zram is "
Nitin Gupta306b0c92009-09-22 10:26:53 +0530125 "wasteful.\n"
126 "\tMemory Size: %zu kB\n"
Nitin Gupta33863c22010-08-09 22:56:47 +0530127 "\tSize you selected: %llu kB\n"
Nitin Gupta306b0c92009-09-22 10:26:53 +0530128 "Continuing anyway ...\n",
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530129 totalram_bytes >> 10, zram->disksize
Nitin Gupta306b0c92009-09-22 10:26:53 +0530130 );
131 }
132
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530133 zram->disksize &= PAGE_MASK;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530134}
135
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530136static void zram_free_page(struct zram *zram, size_t index)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530137{
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600138 void *handle = zram->table[index].handle;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530139
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600140 if (unlikely(!handle)) {
Nitin Gupta2e882282010-01-28 21:13:41 +0530141 /*
142 * No memory is allocated for zero filled pages.
143 * Simply clear zero page flag.
144 */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530145 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
146 zram_clear_flag(zram, index, ZRAM_ZERO);
147 zram_stat_dec(&zram->stats.pages_zero);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530148 }
149 return;
150 }
151
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530152 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600153 __free_page(handle);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530154 zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED);
155 zram_stat_dec(&zram->stats.pages_expand);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530156 goto out;
157 }
158
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600159 zs_free(zram->mem_pool, handle);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530160
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600161 if (zram->table[index].size <= PAGE_SIZE / 2)
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530162 zram_stat_dec(&zram->stats.good_compress);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530163
164out:
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600165 zram_stat64_sub(zram, &zram->stats.compr_size,
166 zram->table[index].size);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530167 zram_stat_dec(&zram->stats.pages_stored);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530168
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600169 zram->table[index].handle = NULL;
170 zram->table[index].size = 0;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530171}
172
Jerome Marchand924bd882011-06-10 15:28:48 +0200173static void handle_zero_page(struct bio_vec *bvec)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530174{
Jerome Marchand924bd882011-06-10 15:28:48 +0200175 struct page *page = bvec->bv_page;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530176 void *user_mem;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530177
178 user_mem = kmap_atomic(page, KM_USER0);
Jerome Marchand924bd882011-06-10 15:28:48 +0200179 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530180 kunmap_atomic(user_mem, KM_USER0);
181
Nitin Gupta30fb8a72009-12-12 11:44:46 +0530182 flush_dcache_page(page);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530183}
184
Jerome Marchand924bd882011-06-10 15:28:48 +0200185static void handle_uncompressed_page(struct zram *zram, struct bio_vec *bvec,
186 u32 index, int offset)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530187{
Jerome Marchand924bd882011-06-10 15:28:48 +0200188 struct page *page = bvec->bv_page;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530189 unsigned char *user_mem, *cmem;
190
Nitin Gupta306b0c92009-09-22 10:26:53 +0530191 user_mem = kmap_atomic(page, KM_USER0);
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600192 cmem = kmap_atomic(zram->table[index].handle, KM_USER1);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530193
Jerome Marchand924bd882011-06-10 15:28:48 +0200194 memcpy(user_mem + bvec->bv_offset, cmem + offset, bvec->bv_len);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530195 kunmap_atomic(cmem, KM_USER1);
Jerome Marchanddffbb442011-07-13 17:20:05 +0200196 kunmap_atomic(user_mem, KM_USER0);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530197
Nitin Gupta30fb8a72009-12-12 11:44:46 +0530198 flush_dcache_page(page);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530199}
200
Jerome Marchand924bd882011-06-10 15:28:48 +0200201static inline int is_partial_io(struct bio_vec *bvec)
202{
203 return bvec->bv_len != PAGE_SIZE;
204}
205
Jerome Marchand8c921b22011-06-10 15:28:47 +0200206static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
Jerome Marchand924bd882011-06-10 15:28:48 +0200207 u32 index, int offset, struct bio *bio)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530208{
Jerome Marchand8c921b22011-06-10 15:28:47 +0200209 int ret;
210 size_t clen;
211 struct page *page;
212 struct zobj_header *zheader;
Jerome Marchand924bd882011-06-10 15:28:48 +0200213 unsigned char *user_mem, *cmem, *uncmem = NULL;
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530214
Jerome Marchand8c921b22011-06-10 15:28:47 +0200215 page = bvec->bv_page;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530216
Jerome Marchand8c921b22011-06-10 15:28:47 +0200217 if (zram_test_flag(zram, index, ZRAM_ZERO)) {
Jerome Marchand924bd882011-06-10 15:28:48 +0200218 handle_zero_page(bvec);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200219 return 0;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530220 }
221
Jerome Marchand8c921b22011-06-10 15:28:47 +0200222 /* Requested page is not present in compressed area */
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600223 if (unlikely(!zram->table[index].handle)) {
Jerome Marchand8c921b22011-06-10 15:28:47 +0200224 pr_debug("Read before write: sector=%lu, size=%u",
225 (ulong)(bio->bi_sector), bio->bi_size);
Jerome Marchand924bd882011-06-10 15:28:48 +0200226 handle_zero_page(bvec);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200227 return 0;
228 }
Nitin Gupta306b0c92009-09-22 10:26:53 +0530229
Jerome Marchand8c921b22011-06-10 15:28:47 +0200230 /* Page is stored uncompressed since it's incompressible */
231 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
Jerome Marchand924bd882011-06-10 15:28:48 +0200232 handle_uncompressed_page(zram, bvec, index, offset);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200233 return 0;
234 }
235
Jerome Marchand924bd882011-06-10 15:28:48 +0200236 if (is_partial_io(bvec)) {
237 /* Use a temporary buffer to decompress the page */
238 uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL);
239 if (!uncmem) {
240 pr_info("Error allocating temp memory!\n");
241 return -ENOMEM;
242 }
243 }
244
Jerome Marchand8c921b22011-06-10 15:28:47 +0200245 user_mem = kmap_atomic(page, KM_USER0);
Jerome Marchand924bd882011-06-10 15:28:48 +0200246 if (!is_partial_io(bvec))
247 uncmem = user_mem;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200248 clen = PAGE_SIZE;
249
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600250 cmem = zs_map_object(zram->mem_pool, zram->table[index].handle);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200251
252 ret = lzo1x_decompress_safe(cmem + sizeof(*zheader),
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600253 zram->table[index].size,
Jerome Marchand924bd882011-06-10 15:28:48 +0200254 uncmem, &clen);
255
256 if (is_partial_io(bvec)) {
257 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
258 bvec->bv_len);
259 kfree(uncmem);
260 }
Jerome Marchand8c921b22011-06-10 15:28:47 +0200261
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600262 zs_unmap_object(zram->mem_pool, zram->table[index].handle);
Jerome Marchanddffbb442011-07-13 17:20:05 +0200263 kunmap_atomic(user_mem, KM_USER0);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200264
265 /* Should NEVER happen. Return bio error if it does. */
266 if (unlikely(ret != LZO_E_OK)) {
267 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
268 zram_stat64_inc(zram, &zram->stats.failed_reads);
269 return ret;
270 }
271
272 flush_dcache_page(page);
273
274 return 0;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530275}
276
Jerome Marchand924bd882011-06-10 15:28:48 +0200277static int zram_read_before_write(struct zram *zram, char *mem, u32 index)
Jerome Marchand8c921b22011-06-10 15:28:47 +0200278{
279 int ret;
Jerome Marchand924bd882011-06-10 15:28:48 +0200280 size_t clen = PAGE_SIZE;
281 struct zobj_header *zheader;
282 unsigned char *cmem;
283
284 if (zram_test_flag(zram, index, ZRAM_ZERO) ||
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600285 !zram->table[index].handle) {
Jerome Marchand924bd882011-06-10 15:28:48 +0200286 memset(mem, 0, PAGE_SIZE);
287 return 0;
288 }
289
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600290 cmem = zs_map_object(zram->mem_pool, zram->table[index].handle);
Jerome Marchand924bd882011-06-10 15:28:48 +0200291
292 /* Page is stored uncompressed since it's incompressible */
293 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
294 memcpy(mem, cmem, PAGE_SIZE);
295 kunmap_atomic(cmem, KM_USER0);
296 return 0;
297 }
298
299 ret = lzo1x_decompress_safe(cmem + sizeof(*zheader),
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600300 zram->table[index].size,
Jerome Marchand924bd882011-06-10 15:28:48 +0200301 mem, &clen);
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600302 zs_unmap_object(zram->mem_pool, zram->table[index].handle);
Jerome Marchand924bd882011-06-10 15:28:48 +0200303
304 /* Should NEVER happen. Return bio error if it does. */
305 if (unlikely(ret != LZO_E_OK)) {
306 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
307 zram_stat64_inc(zram, &zram->stats.failed_reads);
308 return ret;
309 }
310
311 return 0;
312}
313
314static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
315 int offset)
316{
317 int ret;
318 u32 store_offset;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200319 size_t clen;
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600320 void *handle;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200321 struct zobj_header *zheader;
322 struct page *page, *page_store;
Jerome Marchand924bd882011-06-10 15:28:48 +0200323 unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200324
325 page = bvec->bv_page;
326 src = zram->compress_buffer;
327
Jerome Marchand924bd882011-06-10 15:28:48 +0200328 if (is_partial_io(bvec)) {
329 /*
330 * This is a partial IO. We need to read the full page
331 * before to write the changes.
332 */
333 uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL);
334 if (!uncmem) {
335 pr_info("Error allocating temp memory!\n");
336 ret = -ENOMEM;
337 goto out;
338 }
339 ret = zram_read_before_write(zram, uncmem, index);
340 if (ret) {
341 kfree(uncmem);
342 goto out;
343 }
344 }
345
Jerome Marchand8c921b22011-06-10 15:28:47 +0200346 /*
347 * System overwrites unused sectors. Free memory associated
348 * with this sector now.
349 */
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600350 if (zram->table[index].handle ||
Jerome Marchand8c921b22011-06-10 15:28:47 +0200351 zram_test_flag(zram, index, ZRAM_ZERO))
352 zram_free_page(zram, index);
353
Jerome Marchand8c921b22011-06-10 15:28:47 +0200354 user_mem = kmap_atomic(page, KM_USER0);
Jerome Marchand924bd882011-06-10 15:28:48 +0200355
356 if (is_partial_io(bvec))
357 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
358 bvec->bv_len);
359 else
360 uncmem = user_mem;
361
362 if (page_zero_filled(uncmem)) {
Jerome Marchand8c921b22011-06-10 15:28:47 +0200363 kunmap_atomic(user_mem, KM_USER0);
Jerome Marchand924bd882011-06-10 15:28:48 +0200364 if (is_partial_io(bvec))
365 kfree(uncmem);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200366 zram_stat_inc(&zram->stats.pages_zero);
367 zram_set_flag(zram, index, ZRAM_ZERO);
Jerome Marchand924bd882011-06-10 15:28:48 +0200368 ret = 0;
369 goto out;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200370 }
371
Jerome Marchand924bd882011-06-10 15:28:48 +0200372 ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen,
Jerome Marchand8c921b22011-06-10 15:28:47 +0200373 zram->compress_workmem);
374
375 kunmap_atomic(user_mem, KM_USER0);
Jerome Marchand924bd882011-06-10 15:28:48 +0200376 if (is_partial_io(bvec))
377 kfree(uncmem);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200378
379 if (unlikely(ret != LZO_E_OK)) {
Jerome Marchand8c921b22011-06-10 15:28:47 +0200380 pr_err("Compression failed! err=%d\n", ret);
Jerome Marchand924bd882011-06-10 15:28:48 +0200381 goto out;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200382 }
383
384 /*
385 * Page is incompressible. Store it as-is (uncompressed)
386 * since we do not want to return too many disk write
387 * errors which has side effect of hanging the system.
388 */
389 if (unlikely(clen > max_zpage_size)) {
390 clen = PAGE_SIZE;
391 page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
392 if (unlikely(!page_store)) {
Jerome Marchand8c921b22011-06-10 15:28:47 +0200393 pr_info("Error allocating memory for "
394 "incompressible page: %u\n", index);
Jerome Marchand924bd882011-06-10 15:28:48 +0200395 ret = -ENOMEM;
396 goto out;
397 }
Jerome Marchand8c921b22011-06-10 15:28:47 +0200398
Jerome Marchand924bd882011-06-10 15:28:48 +0200399 store_offset = 0;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200400 zram_set_flag(zram, index, ZRAM_UNCOMPRESSED);
401 zram_stat_inc(&zram->stats.pages_expand);
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600402 handle = page_store;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200403 src = kmap_atomic(page, KM_USER0);
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600404 cmem = kmap_atomic(page_store, KM_USER1);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200405 goto memstore;
406 }
407
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600408 handle = zs_malloc(zram->mem_pool, clen + sizeof(*zheader));
409 if (!handle) {
Jerome Marchand8c921b22011-06-10 15:28:47 +0200410 pr_info("Error allocating memory for compressed "
411 "page: %u, size=%zu\n", index, clen);
Jerome Marchand924bd882011-06-10 15:28:48 +0200412 ret = -ENOMEM;
413 goto out;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200414 }
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600415 cmem = zs_map_object(zram->mem_pool, handle);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200416
417memstore:
Jerome Marchand8c921b22011-06-10 15:28:47 +0200418#if 0
419 /* Back-reference needed for memory defragmentation */
420 if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) {
421 zheader = (struct zobj_header *)cmem;
422 zheader->table_idx = index;
423 cmem += sizeof(*zheader);
424 }
425#endif
426
427 memcpy(cmem, src, clen);
428
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600429 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
430 kunmap_atomic(cmem, KM_USER1);
Jerome Marchand8c921b22011-06-10 15:28:47 +0200431 kunmap_atomic(src, KM_USER0);
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600432 } else {
433 zs_unmap_object(zram->mem_pool, handle);
434 }
435
436 zram->table[index].handle = handle;
437 zram->table[index].size = clen;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200438
439 /* Update stats */
440 zram_stat64_add(zram, &zram->stats.compr_size, clen);
441 zram_stat_inc(&zram->stats.pages_stored);
442 if (clen <= PAGE_SIZE / 2)
443 zram_stat_inc(&zram->stats.good_compress);
444
Jerome Marchand8c921b22011-06-10 15:28:47 +0200445 return 0;
Jerome Marchand924bd882011-06-10 15:28:48 +0200446
447out:
448 if (ret)
449 zram_stat64_inc(zram, &zram->stats.failed_writes);
450 return ret;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200451}
452
453static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
Jerome Marchand924bd882011-06-10 15:28:48 +0200454 int offset, struct bio *bio, int rw)
Jerome Marchand8c921b22011-06-10 15:28:47 +0200455{
Jerome Marchandc5bde232011-06-10 15:28:49 +0200456 int ret;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200457
Jerome Marchandc5bde232011-06-10 15:28:49 +0200458 if (rw == READ) {
459 down_read(&zram->lock);
460 ret = zram_bvec_read(zram, bvec, index, offset, bio);
461 up_read(&zram->lock);
462 } else {
463 down_write(&zram->lock);
464 ret = zram_bvec_write(zram, bvec, index, offset);
465 up_write(&zram->lock);
466 }
467
468 return ret;
Jerome Marchand924bd882011-06-10 15:28:48 +0200469}
470
471static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
472{
473 if (*offset + bvec->bv_len >= PAGE_SIZE)
474 (*index)++;
475 *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
Jerome Marchand8c921b22011-06-10 15:28:47 +0200476}
477
478static void __zram_make_request(struct zram *zram, struct bio *bio, int rw)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530479{
Jerome Marchand924bd882011-06-10 15:28:48 +0200480 int i, offset;
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530481 u32 index;
482 struct bio_vec *bvec;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530483
Jerome Marchand8c921b22011-06-10 15:28:47 +0200484 switch (rw) {
485 case READ:
486 zram_stat64_inc(zram, &zram->stats.num_reads);
487 break;
488 case WRITE:
489 zram_stat64_inc(zram, &zram->stats.num_writes);
490 break;
491 }
492
Nitin Gupta306b0c92009-09-22 10:26:53 +0530493 index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
Jerome Marchand924bd882011-06-10 15:28:48 +0200494 offset = (bio->bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530495
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530496 bio_for_each_segment(bvec, bio, i) {
Jerome Marchand924bd882011-06-10 15:28:48 +0200497 int max_transfer_size = PAGE_SIZE - offset;
498
499 if (bvec->bv_len > max_transfer_size) {
500 /*
501 * zram_bvec_rw() can only make operation on a single
502 * zram page. Split the bio vector.
503 */
504 struct bio_vec bv;
505
506 bv.bv_page = bvec->bv_page;
507 bv.bv_len = max_transfer_size;
508 bv.bv_offset = bvec->bv_offset;
509
510 if (zram_bvec_rw(zram, &bv, index, offset, bio, rw) < 0)
511 goto out;
512
513 bv.bv_len = bvec->bv_len - max_transfer_size;
514 bv.bv_offset += max_transfer_size;
515 if (zram_bvec_rw(zram, &bv, index+1, 0, bio, rw) < 0)
516 goto out;
517 } else
518 if (zram_bvec_rw(zram, bvec, index, offset, bio, rw)
519 < 0)
520 goto out;
521
522 update_position(&index, &offset, bvec);
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530523 }
Nitin Gupta306b0c92009-09-22 10:26:53 +0530524
525 set_bit(BIO_UPTODATE, &bio->bi_flags);
526 bio_endio(bio, 0);
Nitin Gupta7d7854b2011-01-22 07:36:15 -0500527 return;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530528
529out:
Nitin Gupta306b0c92009-09-22 10:26:53 +0530530 bio_io_error(bio);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530531}
532
Nitin Gupta306b0c92009-09-22 10:26:53 +0530533/*
Jerome Marchand924bd882011-06-10 15:28:48 +0200534 * Check if request is within bounds and aligned on zram logical blocks.
Nitin Gupta306b0c92009-09-22 10:26:53 +0530535 */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530536static inline int valid_io_request(struct zram *zram, struct bio *bio)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530537{
538 if (unlikely(
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530539 (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) ||
Jerome Marchand924bd882011-06-10 15:28:48 +0200540 (bio->bi_sector & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)) ||
541 (bio->bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530542
543 return 0;
544 }
545
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530546 /* I/O request is valid */
Nitin Gupta306b0c92009-09-22 10:26:53 +0530547 return 1;
548}
549
550/*
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530551 * Handler function for all zram I/O requests.
Nitin Gupta306b0c92009-09-22 10:26:53 +0530552 */
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +0200553static void zram_make_request(struct request_queue *queue, struct bio *bio)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530554{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530555 struct zram *zram = queue->queuedata;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530556
Jerome Marchand0900bea2011-09-06 15:02:11 +0200557 if (unlikely(!zram->init_done) && zram_init_device(zram))
558 goto error;
559
560 down_read(&zram->init_lock);
561 if (unlikely(!zram->init_done))
562 goto error_unlock;
563
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530564 if (!valid_io_request(zram, bio)) {
565 zram_stat64_inc(zram, &zram->stats.invalid_io);
Jerome Marchand0900bea2011-09-06 15:02:11 +0200566 goto error_unlock;
Jerome Marchand6642a672011-02-17 17:11:49 +0100567 }
568
Jerome Marchand8c921b22011-06-10 15:28:47 +0200569 __zram_make_request(zram, bio, bio_data_dir(bio));
Jerome Marchand0900bea2011-09-06 15:02:11 +0200570 up_read(&zram->init_lock);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530571
Linus Torvaldsb4fdcb02011-11-04 17:06:58 -0700572 return;
Jerome Marchand0900bea2011-09-06 15:02:11 +0200573
574error_unlock:
575 up_read(&zram->init_lock);
576error:
577 bio_io_error(bio);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530578}
579
Jerome Marchand0900bea2011-09-06 15:02:11 +0200580void __zram_reset_device(struct zram *zram)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530581{
Nitin Gupta97a06382010-05-13 14:24:21 +0530582 size_t index;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530583
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530584 zram->init_done = 0;
Nitin Gupta7eef7532010-01-28 21:13:38 +0530585
Nitin Gupta306b0c92009-09-22 10:26:53 +0530586 /* Free various per-device buffers */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530587 kfree(zram->compress_workmem);
588 free_pages((unsigned long)zram->compress_buffer, 1);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530589
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530590 zram->compress_workmem = NULL;
591 zram->compress_buffer = NULL;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530592
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530593 /* Free all pages that are still in this zram device */
594 for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600595 void *handle = zram->table[index].handle;
596 if (!handle)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530597 continue;
598
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530599 if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600600 __free_page(handle);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530601 else
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600602 zs_free(zram->mem_pool, handle);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530603 }
604
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530605 vfree(zram->table);
606 zram->table = NULL;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530607
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600608 zs_destroy_pool(zram->mem_pool);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530609 zram->mem_pool = NULL;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530610
Nitin Gupta306b0c92009-09-22 10:26:53 +0530611 /* Reset stats */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530612 memset(&zram->stats, 0, sizeof(zram->stats));
Nitin Gupta306b0c92009-09-22 10:26:53 +0530613
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530614 zram->disksize = 0;
Jerome Marchand0900bea2011-09-06 15:02:11 +0200615}
616
617void zram_reset_device(struct zram *zram)
618{
619 down_write(&zram->init_lock);
620 __zram_reset_device(zram);
621 up_write(&zram->init_lock);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530622}
623
Nitin Gupta33863c22010-08-09 22:56:47 +0530624int zram_init_device(struct zram *zram)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530625{
626 int ret;
627 size_t num_pages;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530628
Jerome Marchand0900bea2011-09-06 15:02:11 +0200629 down_write(&zram->init_lock);
Nitin Gupta484875a2010-08-09 22:56:48 +0530630
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530631 if (zram->init_done) {
Jerome Marchand0900bea2011-09-06 15:02:11 +0200632 up_write(&zram->init_lock);
Nitin Gupta484875a2010-08-09 22:56:48 +0530633 return 0;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530634 }
635
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530636 zram_set_disksize(zram, totalram_pages << PAGE_SHIFT);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530637
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530638 zram->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
639 if (!zram->compress_workmem) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530640 pr_err("Error allocating compressor working memory!\n");
641 ret = -ENOMEM;
Jerome Marchand5a18c532011-09-06 15:02:12 +0200642 goto fail_no_table;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530643 }
644
Jerome Marchandfb927282011-11-30 14:16:16 +0100645 zram->compress_buffer =
646 (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530647 if (!zram->compress_buffer) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530648 pr_err("Error allocating compressor buffer space\n");
649 ret = -ENOMEM;
Jerome Marchand5a18c532011-09-06 15:02:12 +0200650 goto fail_no_table;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530651 }
652
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530653 num_pages = zram->disksize >> PAGE_SHIFT;
Joe Perches5b84cc72010-11-04 20:07:59 -0700654 zram->table = vzalloc(num_pages * sizeof(*zram->table));
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530655 if (!zram->table) {
656 pr_err("Error allocating zram address table\n");
Nitin Gupta306b0c92009-09-22 10:26:53 +0530657 ret = -ENOMEM;
Jerome Marchand5a18c532011-09-06 15:02:12 +0200658 goto fail_no_table;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530659 }
Nitin Gupta306b0c92009-09-22 10:26:53 +0530660
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530661 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530662
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530663 /* zram devices sort of resembles non-rotational disks */
664 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530665
Nitin Guptafd1a30d2012-01-09 16:51:59 -0600666 zram->mem_pool = zs_create_pool("zram", GFP_NOIO | __GFP_HIGHMEM);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530667 if (!zram->mem_pool) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530668 pr_err("Error creating memory pool\n");
669 ret = -ENOMEM;
670 goto fail;
671 }
672
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530673 zram->init_done = 1;
Jerome Marchand0900bea2011-09-06 15:02:11 +0200674 up_write(&zram->init_lock);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530675
676 pr_debug("Initialization done!\n");
677 return 0;
678
Jerome Marchand5a18c532011-09-06 15:02:12 +0200679fail_no_table:
680 /* To prevent accessing table entries during cleanup */
681 zram->disksize = 0;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530682fail:
Jerome Marchand0900bea2011-09-06 15:02:11 +0200683 __zram_reset_device(zram);
684 up_write(&zram->init_lock);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530685 pr_err("Initialization failed: err=%d\n", ret);
686 return ret;
687}
688
Nitin Gupta2ccbec02011-09-09 19:01:00 -0400689static void zram_slot_free_notify(struct block_device *bdev,
690 unsigned long index)
Nitin Gupta107c1612010-05-17 11:02:44 +0530691{
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530692 struct zram *zram;
Nitin Gupta107c1612010-05-17 11:02:44 +0530693
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530694 zram = bdev->bd_disk->private_data;
695 zram_free_page(zram, index);
696 zram_stat64_inc(zram, &zram->stats.notify_free);
Nitin Gupta107c1612010-05-17 11:02:44 +0530697}
698
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530699static const struct block_device_operations zram_devops = {
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530700 .swap_slot_free_notify = zram_slot_free_notify,
Nitin Gupta107c1612010-05-17 11:02:44 +0530701 .owner = THIS_MODULE
Nitin Gupta306b0c92009-09-22 10:26:53 +0530702};
703
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530704static int create_device(struct zram *zram, int device_id)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530705{
Nitin Guptade1a21a2010-01-28 21:13:40 +0530706 int ret = 0;
707
Jerome Marchandc5bde232011-06-10 15:28:49 +0200708 init_rwsem(&zram->lock);
Jerome Marchand0900bea2011-09-06 15:02:11 +0200709 init_rwsem(&zram->init_lock);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530710 spin_lock_init(&zram->stat64_lock);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530711
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530712 zram->queue = blk_alloc_queue(GFP_KERNEL);
713 if (!zram->queue) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530714 pr_err("Error allocating disk queue for device %d\n",
715 device_id);
Nitin Guptade1a21a2010-01-28 21:13:40 +0530716 ret = -ENOMEM;
717 goto out;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530718 }
719
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530720 blk_queue_make_request(zram->queue, zram_make_request);
721 zram->queue->queuedata = zram;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530722
723 /* gendisk structure */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530724 zram->disk = alloc_disk(1);
725 if (!zram->disk) {
726 blk_cleanup_queue(zram->queue);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530727 pr_warning("Error allocating disk structure for device %d\n",
728 device_id);
Nitin Guptade1a21a2010-01-28 21:13:40 +0530729 ret = -ENOMEM;
730 goto out;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530731 }
732
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530733 zram->disk->major = zram_major;
734 zram->disk->first_minor = device_id;
735 zram->disk->fops = &zram_devops;
736 zram->disk->queue = zram->queue;
737 zram->disk->private_data = zram;
738 snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530739
Nitin Gupta33863c22010-08-09 22:56:47 +0530740 /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530741 set_capacity(zram->disk, 0);
Nitin Gupta5d83d5a2010-01-28 21:13:39 +0530742
Nitin Guptaa1dd52a2010-06-01 13:31:23 +0530743 /*
744 * To ensure that we always get PAGE_SIZE aligned
745 * and n*PAGE_SIZED sized I/O requests.
746 */
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530747 blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
Robert Jennings7b19b8d2011-01-28 08:58:17 -0600748 blk_queue_logical_block_size(zram->disk->queue,
749 ZRAM_LOGICAL_BLOCK_SIZE);
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530750 blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
751 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
Nitin Gupta5d83d5a2010-01-28 21:13:39 +0530752
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530753 add_disk(zram->disk);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530754
Nitin Gupta33863c22010-08-09 22:56:47 +0530755 ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
756 &zram_disk_attr_group);
757 if (ret < 0) {
758 pr_warning("Error creating sysfs group");
759 goto out;
760 }
Nitin Gupta33863c22010-08-09 22:56:47 +0530761
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530762 zram->init_done = 0;
Nitin Guptade1a21a2010-01-28 21:13:40 +0530763
764out:
765 return ret;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530766}
767
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530768static void destroy_device(struct zram *zram)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530769{
Nitin Gupta33863c22010-08-09 22:56:47 +0530770 sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
771 &zram_disk_attr_group);
Nitin Gupta33863c22010-08-09 22:56:47 +0530772
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530773 if (zram->disk) {
774 del_gendisk(zram->disk);
775 put_disk(zram->disk);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530776 }
777
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530778 if (zram->queue)
779 blk_cleanup_queue(zram->queue);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530780}
781
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500782unsigned int zram_get_num_devices(void)
783{
784 return num_devices;
785}
786
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530787static int __init zram_init(void)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530788{
Nitin Guptade1a21a2010-01-28 21:13:40 +0530789 int ret, dev_id;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530790
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500791 if (num_devices > max_num_devices) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530792 pr_warning("Invalid value for num_devices: %u\n",
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500793 num_devices);
Nitin Guptade1a21a2010-01-28 21:13:40 +0530794 ret = -EINVAL;
795 goto out;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530796 }
797
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530798 zram_major = register_blkdev(0, "zram");
799 if (zram_major <= 0) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530800 pr_warning("Unable to get major number\n");
Nitin Guptade1a21a2010-01-28 21:13:40 +0530801 ret = -EBUSY;
802 goto out;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530803 }
804
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500805 if (!num_devices) {
Nitin Gupta306b0c92009-09-22 10:26:53 +0530806 pr_info("num_devices not specified. Using default: 1\n");
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500807 num_devices = 1;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530808 }
809
810 /* Allocate the device array and initialize each one */
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500811 pr_info("Creating %u devices ...\n", num_devices);
812 zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
Noah Watkins43801f62011-07-20 17:05:57 -0600813 if (!zram_devices) {
Nitin Guptade1a21a2010-01-28 21:13:40 +0530814 ret = -ENOMEM;
815 goto unregister;
816 }
Nitin Gupta306b0c92009-09-22 10:26:53 +0530817
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500818 for (dev_id = 0; dev_id < num_devices; dev_id++) {
Noah Watkins43801f62011-07-20 17:05:57 -0600819 ret = create_device(&zram_devices[dev_id], dev_id);
Nitin Guptade1a21a2010-01-28 21:13:40 +0530820 if (ret)
Minchan Kim3bf040c2010-01-11 16:15:53 +0900821 goto free_devices;
Nitin Guptade1a21a2010-01-28 21:13:40 +0530822 }
823
Nitin Gupta306b0c92009-09-22 10:26:53 +0530824 return 0;
Nitin Guptade1a21a2010-01-28 21:13:40 +0530825
Minchan Kim3bf040c2010-01-11 16:15:53 +0900826free_devices:
Nitin Guptade1a21a2010-01-28 21:13:40 +0530827 while (dev_id)
Noah Watkins43801f62011-07-20 17:05:57 -0600828 destroy_device(&zram_devices[--dev_id]);
829 kfree(zram_devices);
Nitin Guptade1a21a2010-01-28 21:13:40 +0530830unregister:
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530831 unregister_blkdev(zram_major, "zram");
Nitin Guptade1a21a2010-01-28 21:13:40 +0530832out:
Nitin Gupta306b0c92009-09-22 10:26:53 +0530833 return ret;
834}
835
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530836static void __exit zram_exit(void)
Nitin Gupta306b0c92009-09-22 10:26:53 +0530837{
838 int i;
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530839 struct zram *zram;
Nitin Gupta306b0c92009-09-22 10:26:53 +0530840
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500841 for (i = 0; i < num_devices; i++) {
Noah Watkins43801f62011-07-20 17:05:57 -0600842 zram = &zram_devices[i];
Nitin Gupta306b0c92009-09-22 10:26:53 +0530843
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530844 destroy_device(zram);
845 if (zram->init_done)
Nitin Gupta33863c22010-08-09 22:56:47 +0530846 zram_reset_device(zram);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530847 }
848
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530849 unregister_blkdev(zram_major, "zram");
Nitin Gupta306b0c92009-09-22 10:26:53 +0530850
Noah Watkins43801f62011-07-20 17:05:57 -0600851 kfree(zram_devices);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530852 pr_debug("Cleanup done!\n");
853}
854
Nitin Gupta5fa5a902012-02-12 23:04:45 -0500855module_param(num_devices, uint, 0);
856MODULE_PARM_DESC(num_devices, "Number of zram devices");
Nitin Gupta306b0c92009-09-22 10:26:53 +0530857
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530858module_init(zram_init);
859module_exit(zram_exit);
Nitin Gupta306b0c92009-09-22 10:26:53 +0530860
861MODULE_LICENSE("Dual BSD/GPL");
862MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
Nitin Guptaf1e3cff2010-06-01 13:31:25 +0530863MODULE_DESCRIPTION("Compressed RAM Block Device");