blob: 3dc0552cddfbb614d82342ff154bafc10370d2da [file] [log] [blame]
Rafael J. Wysocki61159a32006-03-23 03:00:00 -08001/*
2 * linux/kernel/power/swap.c
3 *
4 * This file provides functions for reading the suspend image from
5 * and writing it to a swap partition.
6 *
Pavel Macheka2531292010-07-18 14:27:13 +02007 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
Rafael J. Wysocki61159a32006-03-23 03:00:00 -08008 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9 *
10 * This file is released under the GPLv2.
11 *
12 */
13
14#include <linux/module.h>
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080015#include <linux/file.h>
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080016#include <linux/delay.h>
17#include <linux/bitops.h>
18#include <linux/genhd.h>
19#include <linux/device.h>
20#include <linux/buffer_head.h>
21#include <linux/bio.h>
Andrew Morton546e0d22006-09-25 23:32:44 -070022#include <linux/blkdev.h>
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080023#include <linux/swap.h>
24#include <linux/swapops.h>
25#include <linux/pm.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090026#include <linux/slab.h>
Bojan Smojverf996fc92010-09-09 23:06:23 +020027#include <linux/lzo.h>
28#include <linux/vmalloc.h>
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080029
30#include "power.h"
31
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080032#define SWSUSP_SIG "S1SUSPEND"
33
Jiri Slaby51fb3522010-05-01 23:53:02 +020034/*
35 * The swap map is a data structure used for keeping track of each page
36 * written to a swap partition. It consists of many swap_map_page
Cesar Eduardo Barros90133672010-06-07 22:23:12 +020037 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
Jiri Slaby51fb3522010-05-01 23:53:02 +020038 * These structures are stored on the swap and linked together with the
39 * help of the .next_swap member.
40 *
41 * The swap map is created during suspend. The swap map pages are
42 * allocated and populated one at a time, so we only need one memory
43 * page to set up the entire structure.
44 *
45 * During resume we also only need to use one swap_map_page structure
46 * at a time.
47 */
48
49#define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
50
51struct swap_map_page {
52 sector_t entries[MAP_PAGE_ENTRIES];
53 sector_t next_swap;
54};
55
56/**
57 * The swap_map_handle structure is used for handling swap in
58 * a file-alike way
59 */
60
61struct swap_map_handle {
62 struct swap_map_page *cur;
63 sector_t cur_swap;
64 sector_t first_sector;
65 unsigned int k;
66};
67
Vivek Goyal1b29c162007-05-02 19:27:07 +020068struct swsusp_header {
Rafael J. Wysockia634cc12007-07-19 01:47:30 -070069 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int)];
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -080070 sector_t image;
Rafael J. Wysockia634cc12007-07-19 01:47:30 -070071 unsigned int flags; /* Flags to pass to the "boot" kernel */
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080072 char orig_sig[10];
73 char sig[10];
Vivek Goyal1b29c162007-05-02 19:27:07 +020074} __attribute__((packed));
75
76static struct swsusp_header *swsusp_header;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -080077
Nigel Cunningham0414f2e2009-12-06 16:15:53 +010078/**
79 * The following functions are used for tracing the allocated
80 * swap pages, so that they can be freed in case of an error.
81 */
82
83struct swsusp_extent {
84 struct rb_node node;
85 unsigned long start;
86 unsigned long end;
87};
88
89static struct rb_root swsusp_extents = RB_ROOT;
90
91static int swsusp_extents_insert(unsigned long swap_offset)
92{
93 struct rb_node **new = &(swsusp_extents.rb_node);
94 struct rb_node *parent = NULL;
95 struct swsusp_extent *ext;
96
97 /* Figure out where to put the new node */
98 while (*new) {
99 ext = container_of(*new, struct swsusp_extent, node);
100 parent = *new;
101 if (swap_offset < ext->start) {
102 /* Try to merge */
103 if (swap_offset == ext->start - 1) {
104 ext->start--;
105 return 0;
106 }
107 new = &((*new)->rb_left);
108 } else if (swap_offset > ext->end) {
109 /* Try to merge */
110 if (swap_offset == ext->end + 1) {
111 ext->end++;
112 return 0;
113 }
114 new = &((*new)->rb_right);
115 } else {
116 /* It already is in the tree */
117 return -EINVAL;
118 }
119 }
120 /* Add the new node and rebalance the tree. */
121 ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
122 if (!ext)
123 return -ENOMEM;
124
125 ext->start = swap_offset;
126 ext->end = swap_offset;
127 rb_link_node(&ext->node, parent, new);
128 rb_insert_color(&ext->node, &swsusp_extents);
129 return 0;
130}
131
132/**
133 * alloc_swapdev_block - allocate a swap page and register that it has
134 * been allocated, so that it can be freed in case of an error.
135 */
136
137sector_t alloc_swapdev_block(int swap)
138{
139 unsigned long offset;
140
Hugh Dickins910321e2010-09-09 16:38:07 -0700141 offset = swp_offset(get_swap_page_of_type(swap));
Nigel Cunningham0414f2e2009-12-06 16:15:53 +0100142 if (offset) {
143 if (swsusp_extents_insert(offset))
Hugh Dickins910321e2010-09-09 16:38:07 -0700144 swap_free(swp_entry(swap, offset));
Nigel Cunningham0414f2e2009-12-06 16:15:53 +0100145 else
146 return swapdev_block(swap, offset);
147 }
148 return 0;
149}
150
151/**
152 * free_all_swap_pages - free swap pages allocated for saving image data.
Cesar Eduardo Barros90133672010-06-07 22:23:12 +0200153 * It also frees the extents used to register which swap entries had been
Nigel Cunningham0414f2e2009-12-06 16:15:53 +0100154 * allocated.
155 */
156
157void free_all_swap_pages(int swap)
158{
159 struct rb_node *node;
160
161 while ((node = swsusp_extents.rb_node)) {
162 struct swsusp_extent *ext;
163 unsigned long offset;
164
165 ext = container_of(node, struct swsusp_extent, node);
166 rb_erase(node, &swsusp_extents);
167 for (offset = ext->start; offset <= ext->end; offset++)
Hugh Dickins910321e2010-09-09 16:38:07 -0700168 swap_free(swp_entry(swap, offset));
Nigel Cunningham0414f2e2009-12-06 16:15:53 +0100169
170 kfree(ext);
171 }
172}
173
174int swsusp_swap_in_use(void)
175{
176 return (swsusp_extents.rb_node != NULL);
177}
178
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800179/*
Rafael J. Wysocki3fc6b342006-12-06 20:34:09 -0800180 * General things
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800181 */
182
183static unsigned short root_swap = 0xffff;
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200184struct block_device *hib_resume_bdev;
Rafael J. Wysocki3fc6b342006-12-06 20:34:09 -0800185
Rafael J. Wysocki3fc6b342006-12-06 20:34:09 -0800186/*
187 * Saving part
188 */
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800189
Jiri Slaby51fb3522010-05-01 23:53:02 +0200190static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800191{
192 int error;
193
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200194 hib_bio_read_page(swsusp_resume_block, swsusp_header, NULL);
Vivek Goyal1b29c162007-05-02 19:27:07 +0200195 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
196 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
197 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
198 memcpy(swsusp_header->sig,SWSUSP_SIG, 10);
Jiri Slaby51fb3522010-05-01 23:53:02 +0200199 swsusp_header->image = handle->first_sector;
Rafael J. Wysockia634cc12007-07-19 01:47:30 -0700200 swsusp_header->flags = flags;
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200201 error = hib_bio_write_page(swsusp_resume_block,
Vivek Goyal1b29c162007-05-02 19:27:07 +0200202 swsusp_header, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800203 } else {
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100204 printk(KERN_ERR "PM: Swap header not found!\n");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800205 error = -ENODEV;
206 }
207 return error;
208}
209
210/**
211 * swsusp_swap_check - check if the resume device is a swap device
212 * and get its index (if so)
Jiri Slaby6f612af2010-05-01 23:54:02 +0200213 *
214 * This is called before saving image
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800215 */
Jiri Slaby6f612af2010-05-01 23:54:02 +0200216static int swsusp_swap_check(void)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800217{
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800218 int res;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800219
Rafael J. Wysocki7bf23682007-01-05 16:36:28 -0800220 res = swap_type_of(swsusp_resume_device, swsusp_resume_block,
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200221 &hib_resume_bdev);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800222 if (res < 0)
223 return res;
224
225 root_swap = res;
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200226 res = blkdev_get(hib_resume_bdev, FMODE_WRITE);
Rafael J. Wysocki7bf23682007-01-05 16:36:28 -0800227 if (res)
228 return res;
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800229
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200230 res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800231 if (res < 0)
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200232 blkdev_put(hib_resume_bdev, FMODE_WRITE);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800233
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800234 return res;
235}
236
237/**
238 * write_page - Write one page to given swap location.
239 * @buf: Address we're writing.
240 * @offset: Offset of the swap page we're writing to.
Andrew Mortonab954162006-09-25 23:32:42 -0700241 * @bio_chain: Link the next write BIO here
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800242 */
243
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800244static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800245{
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800246 void *src;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800247
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800248 if (!offset)
249 return -ENOSPC;
Andrew Mortonab954162006-09-25 23:32:42 -0700250
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800251 if (bio_chain) {
Rafael J. Wysocki85949122006-12-06 20:34:19 -0800252 src = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800253 if (src) {
254 memcpy(src, buf, PAGE_SIZE);
255 } else {
256 WARN_ON_ONCE(1);
257 bio_chain = NULL; /* Go synchronous */
258 src = buf;
Andrew Mortonab954162006-09-25 23:32:42 -0700259 }
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800260 } else {
261 src = buf;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800262 }
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200263 return hib_bio_write_page(offset, src, bio_chain);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800264}
265
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800266static void release_swap_writer(struct swap_map_handle *handle)
267{
268 if (handle->cur)
269 free_page((unsigned long)handle->cur);
270 handle->cur = NULL;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800271}
272
273static int get_swap_writer(struct swap_map_handle *handle)
274{
Jiri Slaby6f612af2010-05-01 23:54:02 +0200275 int ret;
276
277 ret = swsusp_swap_check();
278 if (ret) {
279 if (ret != -ENOSPC)
280 printk(KERN_ERR "PM: Cannot find swap device, try "
281 "swapon -a.\n");
282 return ret;
283 }
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800284 handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
Jiri Slaby6f612af2010-05-01 23:54:02 +0200285 if (!handle->cur) {
286 ret = -ENOMEM;
287 goto err_close;
288 }
Rafael J. Wysockid1d241c2007-05-06 14:50:47 -0700289 handle->cur_swap = alloc_swapdev_block(root_swap);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800290 if (!handle->cur_swap) {
Jiri Slaby6f612af2010-05-01 23:54:02 +0200291 ret = -ENOSPC;
292 goto err_rel;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800293 }
294 handle->k = 0;
Jiri Slaby51fb3522010-05-01 23:53:02 +0200295 handle->first_sector = handle->cur_swap;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800296 return 0;
Jiri Slaby6f612af2010-05-01 23:54:02 +0200297err_rel:
298 release_swap_writer(handle);
299err_close:
300 swsusp_close(FMODE_WRITE);
301 return ret;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800302}
303
Andrew Mortonab954162006-09-25 23:32:42 -0700304static int swap_write_page(struct swap_map_handle *handle, void *buf,
305 struct bio **bio_chain)
306{
307 int error = 0;
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800308 sector_t offset;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800309
310 if (!handle->cur)
311 return -EINVAL;
Rafael J. Wysockid1d241c2007-05-06 14:50:47 -0700312 offset = alloc_swapdev_block(root_swap);
Andrew Mortonab954162006-09-25 23:32:42 -0700313 error = write_page(buf, offset, bio_chain);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800314 if (error)
315 return error;
316 handle->cur->entries[handle->k++] = offset;
317 if (handle->k >= MAP_PAGE_ENTRIES) {
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200318 error = hib_wait_on_bio_chain(bio_chain);
Andrew Mortonab954162006-09-25 23:32:42 -0700319 if (error)
320 goto out;
Rafael J. Wysockid1d241c2007-05-06 14:50:47 -0700321 offset = alloc_swapdev_block(root_swap);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800322 if (!offset)
323 return -ENOSPC;
324 handle->cur->next_swap = offset;
Andrew Mortonab954162006-09-25 23:32:42 -0700325 error = write_page(handle->cur, handle->cur_swap, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800326 if (error)
Andrew Mortonab954162006-09-25 23:32:42 -0700327 goto out;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800328 memset(handle->cur, 0, PAGE_SIZE);
329 handle->cur_swap = offset;
330 handle->k = 0;
331 }
Rafael J. Wysocki59a493352006-12-06 20:34:44 -0800332 out:
Andrew Mortonab954162006-09-25 23:32:42 -0700333 return error;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800334}
335
336static int flush_swap_writer(struct swap_map_handle *handle)
337{
338 if (handle->cur && handle->cur_swap)
Andrew Mortonab954162006-09-25 23:32:42 -0700339 return write_page(handle->cur, handle->cur_swap, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800340 else
341 return -EINVAL;
342}
343
Jiri Slaby6f612af2010-05-01 23:54:02 +0200344static int swap_writer_finish(struct swap_map_handle *handle,
345 unsigned int flags, int error)
346{
347 if (!error) {
348 flush_swap_writer(handle);
349 printk(KERN_INFO "PM: S");
350 error = mark_swapfiles(handle, flags);
351 printk("|\n");
352 }
353
354 if (error)
355 free_all_swap_pages(root_swap);
356 release_swap_writer(handle);
357 swsusp_close(FMODE_WRITE);
358
359 return error;
360}
361
Bojan Smojverf996fc92010-09-09 23:06:23 +0200362/* We need to remember how much compressed data we need to read. */
363#define LZO_HEADER sizeof(size_t)
364
365/* Number of pages/bytes we'll compress at one time. */
366#define LZO_UNC_PAGES 32
367#define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE)
368
369/* Number of pages/bytes we need for compressed data (worst case). */
370#define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
371 LZO_HEADER, PAGE_SIZE)
372#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
373
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800374/**
375 * save_image - save the suspend image data
376 */
377
378static int save_image(struct swap_map_handle *handle,
379 struct snapshot_handle *snapshot,
Andrew Morton3a4f7572006-09-25 23:32:41 -0700380 unsigned int nr_to_write)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800381{
382 unsigned int m;
383 int ret;
Andrew Morton3a4f7572006-09-25 23:32:41 -0700384 int nr_pages;
Andrew Mortonab954162006-09-25 23:32:42 -0700385 int err2;
386 struct bio *bio;
Andrew Morton3a4f7572006-09-25 23:32:41 -0700387 struct timeval start;
388 struct timeval stop;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800389
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100390 printk(KERN_INFO "PM: Saving image data pages (%u pages) ... ",
391 nr_to_write);
Andrew Morton3a4f7572006-09-25 23:32:41 -0700392 m = nr_to_write / 100;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800393 if (!m)
394 m = 1;
395 nr_pages = 0;
Andrew Mortonab954162006-09-25 23:32:42 -0700396 bio = NULL;
Andrew Morton3a4f7572006-09-25 23:32:41 -0700397 do_gettimeofday(&start);
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100398 while (1) {
Jiri Slabyd3c1b242010-05-01 23:52:02 +0200399 ret = snapshot_read_next(snapshot);
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100400 if (ret <= 0)
401 break;
402 ret = swap_write_page(handle, data_of(*snapshot), &bio);
403 if (ret)
404 break;
405 if (!(nr_pages % m))
Jiri Slaby66d0ae42009-12-06 16:16:24 +0100406 printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m);
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100407 nr_pages++;
408 }
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200409 err2 = hib_wait_on_bio_chain(&bio);
Andrew Morton3a4f7572006-09-25 23:32:41 -0700410 do_gettimeofday(&stop);
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100411 if (!ret)
412 ret = err2;
413 if (!ret)
Jiri Slaby66d0ae42009-12-06 16:16:24 +0100414 printk(KERN_CONT "\b\b\b\bdone\n");
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100415 else
Jiri Slaby66d0ae42009-12-06 16:16:24 +0100416 printk(KERN_CONT "\n");
Rafael J. Wysocki0d3a9ab2006-12-06 20:34:32 -0800417 swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
Jiri Slaby4ff277f2009-10-28 22:55:33 +0100418 return ret;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800419}
420
Bojan Smojverf996fc92010-09-09 23:06:23 +0200421
422/**
423 * save_image_lzo - Save the suspend image data compressed with LZO.
424 * @handle: Swap mam handle to use for saving the image.
425 * @snapshot: Image to read data from.
426 * @nr_to_write: Number of pages to save.
427 */
428static int save_image_lzo(struct swap_map_handle *handle,
429 struct snapshot_handle *snapshot,
430 unsigned int nr_to_write)
431{
432 unsigned int m;
433 int ret = 0;
434 int nr_pages;
435 int err2;
436 struct bio *bio;
437 struct timeval start;
438 struct timeval stop;
439 size_t off, unc_len, cmp_len;
440 unsigned char *unc, *cmp, *wrk, *page;
441
442 page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
443 if (!page) {
444 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
445 return -ENOMEM;
446 }
447
448 wrk = vmalloc(LZO1X_1_MEM_COMPRESS);
449 if (!wrk) {
450 printk(KERN_ERR "PM: Failed to allocate LZO workspace\n");
451 free_page((unsigned long)page);
452 return -ENOMEM;
453 }
454
455 unc = vmalloc(LZO_UNC_SIZE);
456 if (!unc) {
457 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n");
458 vfree(wrk);
459 free_page((unsigned long)page);
460 return -ENOMEM;
461 }
462
463 cmp = vmalloc(LZO_CMP_SIZE);
464 if (!cmp) {
465 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n");
466 vfree(unc);
467 vfree(wrk);
468 free_page((unsigned long)page);
469 return -ENOMEM;
470 }
471
472 printk(KERN_INFO
473 "PM: Compressing and saving image data (%u pages) ... ",
474 nr_to_write);
475 m = nr_to_write / 100;
476 if (!m)
477 m = 1;
478 nr_pages = 0;
479 bio = NULL;
480 do_gettimeofday(&start);
481 for (;;) {
482 for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) {
483 ret = snapshot_read_next(snapshot);
484 if (ret < 0)
485 goto out_finish;
486
487 if (!ret)
488 break;
489
490 memcpy(unc + off, data_of(*snapshot), PAGE_SIZE);
491
492 if (!(nr_pages % m))
493 printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m);
494 nr_pages++;
495 }
496
497 if (!off)
498 break;
499
500 unc_len = off;
501 ret = lzo1x_1_compress(unc, unc_len,
502 cmp + LZO_HEADER, &cmp_len, wrk);
503 if (ret < 0) {
504 printk(KERN_ERR "PM: LZO compression failed\n");
505 break;
506 }
507
508 if (unlikely(!cmp_len ||
509 cmp_len > lzo1x_worst_compress(unc_len))) {
510 printk(KERN_ERR "PM: Invalid LZO compressed length\n");
511 ret = -1;
512 break;
513 }
514
515 *(size_t *)cmp = cmp_len;
516
517 /*
518 * Given we are writing one page at a time to disk, we copy
519 * that much from the buffer, although the last bit will likely
520 * be smaller than full page. This is OK - we saved the length
521 * of the compressed data, so any garbage at the end will be
522 * discarded when we read it.
523 */
524 for (off = 0; off < LZO_HEADER + cmp_len; off += PAGE_SIZE) {
525 memcpy(page, cmp + off, PAGE_SIZE);
526
527 ret = swap_write_page(handle, page, &bio);
528 if (ret)
529 goto out_finish;
530 }
531 }
532
533out_finish:
534 err2 = hib_wait_on_bio_chain(&bio);
535 do_gettimeofday(&stop);
536 if (!ret)
537 ret = err2;
538 if (!ret)
539 printk(KERN_CONT "\b\b\b\bdone\n");
540 else
541 printk(KERN_CONT "\n");
542 swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
543
544 vfree(cmp);
545 vfree(unc);
546 vfree(wrk);
547 free_page((unsigned long)page);
548
549 return ret;
550}
551
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800552/**
553 * enough_swap - Make sure we have enough swap to save the image.
554 *
555 * Returns TRUE or FALSE after checking the total amount of swap
556 * space avaiable from the resume partition.
557 */
558
Bojan Smojverf996fc92010-09-09 23:06:23 +0200559static int enough_swap(unsigned int nr_pages, unsigned int flags)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800560{
561 unsigned int free_swap = count_swap_pages(root_swap, 1);
Bojan Smojverf996fc92010-09-09 23:06:23 +0200562 unsigned int required;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800563
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100564 pr_debug("PM: Free swap pages: %u\n", free_swap);
Bojan Smojverf996fc92010-09-09 23:06:23 +0200565
566 required = PAGES_FOR_IO + ((flags & SF_NOCOMPRESS_MODE) ?
567 nr_pages : (nr_pages * LZO_CMP_PAGES) / LZO_UNC_PAGES + 1);
568 return free_swap > required;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800569}
570
571/**
572 * swsusp_write - Write entire image and metadata.
Rafael J. Wysockia634cc12007-07-19 01:47:30 -0700573 * @flags: flags to pass to the "boot" kernel in the image header
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800574 *
575 * It is important _NOT_ to umount filesystems at this point. We want
576 * them synced (in case something goes wrong) but we DO not want to mark
577 * filesystem clean: it is not. (And it does not matter, if we resume
578 * correctly, we'll mark system clean, anyway.)
579 */
580
Rafael J. Wysockia634cc12007-07-19 01:47:30 -0700581int swsusp_write(unsigned int flags)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800582{
583 struct swap_map_handle handle;
584 struct snapshot_handle snapshot;
585 struct swsusp_info *header;
Jiri Slaby6f612af2010-05-01 23:54:02 +0200586 unsigned long pages;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800587 int error;
588
Jiri Slaby6f612af2010-05-01 23:54:02 +0200589 pages = snapshot_get_image_size();
590 error = get_swap_writer(&handle);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800591 if (error) {
Jiri Slaby6f612af2010-05-01 23:54:02 +0200592 printk(KERN_ERR "PM: Cannot get swap writer\n");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800593 return error;
594 }
Bojan Smojverf996fc92010-09-09 23:06:23 +0200595 if (!enough_swap(pages, flags)) {
Jiri Slaby6f612af2010-05-01 23:54:02 +0200596 printk(KERN_ERR "PM: Not enough free swap\n");
597 error = -ENOSPC;
598 goto out_finish;
599 }
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800600 memset(&snapshot, 0, sizeof(struct snapshot_handle));
Jiri Slabyd3c1b242010-05-01 23:52:02 +0200601 error = snapshot_read_next(&snapshot);
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800602 if (error < PAGE_SIZE) {
603 if (error >= 0)
604 error = -EFAULT;
605
Jiri Slaby6f612af2010-05-01 23:54:02 +0200606 goto out_finish;
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800607 }
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800608 header = (struct swsusp_info *)data_of(snapshot);
Jiri Slaby6f612af2010-05-01 23:54:02 +0200609 error = swap_write_page(&handle, header, NULL);
Bojan Smojverf996fc92010-09-09 23:06:23 +0200610 if (!error) {
611 error = (flags & SF_NOCOMPRESS_MODE) ?
612 save_image(&handle, &snapshot, pages - 1) :
613 save_image_lzo(&handle, &snapshot, pages - 1);
614 }
Jiri Slaby6f612af2010-05-01 23:54:02 +0200615out_finish:
616 error = swap_writer_finish(&handle, flags, error);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800617 return error;
618}
619
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800620/**
621 * The following functions allow us to read data using a swap map
622 * in a file-alike way
623 */
624
625static void release_swap_reader(struct swap_map_handle *handle)
626{
627 if (handle->cur)
628 free_page((unsigned long)handle->cur);
629 handle->cur = NULL;
630}
631
Jiri Slaby6f612af2010-05-01 23:54:02 +0200632static int get_swap_reader(struct swap_map_handle *handle,
633 unsigned int *flags_p)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800634{
635 int error;
636
Jiri Slaby6f612af2010-05-01 23:54:02 +0200637 *flags_p = swsusp_header->flags;
638
639 if (!swsusp_header->image) /* how can this happen? */
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800640 return -EINVAL;
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800641
Rafael J. Wysocki85949122006-12-06 20:34:19 -0800642 handle->cur = (struct swap_map_page *)get_zeroed_page(__GFP_WAIT | __GFP_HIGH);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800643 if (!handle->cur)
644 return -ENOMEM;
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800645
Jiri Slaby6f612af2010-05-01 23:54:02 +0200646 error = hib_bio_read_page(swsusp_header->image, handle->cur, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800647 if (error) {
648 release_swap_reader(handle);
649 return error;
650 }
651 handle->k = 0;
652 return 0;
653}
654
Andrew Morton546e0d22006-09-25 23:32:44 -0700655static int swap_read_page(struct swap_map_handle *handle, void *buf,
656 struct bio **bio_chain)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800657{
Rafael J. Wysocki3aef83e2006-12-06 20:34:10 -0800658 sector_t offset;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800659 int error;
660
661 if (!handle->cur)
662 return -EINVAL;
663 offset = handle->cur->entries[handle->k];
664 if (!offset)
665 return -EFAULT;
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200666 error = hib_bio_read_page(offset, buf, bio_chain);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800667 if (error)
668 return error;
669 if (++handle->k >= MAP_PAGE_ENTRIES) {
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200670 error = hib_wait_on_bio_chain(bio_chain);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800671 handle->k = 0;
672 offset = handle->cur->next_swap;
673 if (!offset)
674 release_swap_reader(handle);
Andrew Morton546e0d22006-09-25 23:32:44 -0700675 else if (!error)
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200676 error = hib_bio_read_page(offset, handle->cur, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800677 }
678 return error;
679}
680
Jiri Slaby6f612af2010-05-01 23:54:02 +0200681static int swap_reader_finish(struct swap_map_handle *handle)
682{
683 release_swap_reader(handle);
684
685 return 0;
686}
687
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800688/**
689 * load_image - load the image using the swap map handle
690 * @handle and the snapshot handle @snapshot
691 * (assume there are @nr_pages pages to load)
692 */
693
694static int load_image(struct swap_map_handle *handle,
695 struct snapshot_handle *snapshot,
Andrew Morton546e0d22006-09-25 23:32:44 -0700696 unsigned int nr_to_read)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800697{
698 unsigned int m;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800699 int error = 0;
Andrew Morton8c002492006-09-25 23:32:43 -0700700 struct timeval start;
701 struct timeval stop;
Andrew Morton546e0d22006-09-25 23:32:44 -0700702 struct bio *bio;
703 int err2;
704 unsigned nr_pages;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800705
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100706 printk(KERN_INFO "PM: Loading image data pages (%u pages) ... ",
707 nr_to_read);
Andrew Morton546e0d22006-09-25 23:32:44 -0700708 m = nr_to_read / 100;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800709 if (!m)
710 m = 1;
711 nr_pages = 0;
Andrew Morton546e0d22006-09-25 23:32:44 -0700712 bio = NULL;
Andrew Morton8c002492006-09-25 23:32:43 -0700713 do_gettimeofday(&start);
Andrew Morton546e0d22006-09-25 23:32:44 -0700714 for ( ; ; ) {
Jiri Slabyd3c1b242010-05-01 23:52:02 +0200715 error = snapshot_write_next(snapshot);
Andrew Morton546e0d22006-09-25 23:32:44 -0700716 if (error <= 0)
717 break;
718 error = swap_read_page(handle, data_of(*snapshot), &bio);
719 if (error)
720 break;
721 if (snapshot->sync_read)
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200722 error = hib_wait_on_bio_chain(&bio);
Andrew Morton546e0d22006-09-25 23:32:44 -0700723 if (error)
724 break;
725 if (!(nr_pages % m))
726 printk("\b\b\b\b%3d%%", nr_pages / m);
727 nr_pages++;
728 }
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200729 err2 = hib_wait_on_bio_chain(&bio);
Andrew Morton8c002492006-09-25 23:32:43 -0700730 do_gettimeofday(&stop);
Andrew Morton546e0d22006-09-25 23:32:44 -0700731 if (!error)
732 error = err2;
Con Kolivase655a252006-03-26 01:37:11 -0800733 if (!error) {
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800734 printk("\b\b\b\bdone\n");
Rafael J. Wysocki83573762006-12-06 20:34:18 -0800735 snapshot_write_finalize(snapshot);
Con Kolivase655a252006-03-26 01:37:11 -0800736 if (!snapshot_image_loaded(snapshot))
737 error = -ENODATA;
Jiri Slabybf9fd672009-10-28 22:55:42 +0100738 } else
739 printk("\n");
Rafael J. Wysocki0d3a9ab2006-12-06 20:34:32 -0800740 swsusp_show_speed(&start, &stop, nr_to_read, "Read");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800741 return error;
742}
743
Rafael J. Wysockia634cc12007-07-19 01:47:30 -0700744/**
Bojan Smojverf996fc92010-09-09 23:06:23 +0200745 * load_image_lzo - Load compressed image data and decompress them with LZO.
746 * @handle: Swap map handle to use for loading data.
747 * @snapshot: Image to copy uncompressed data into.
748 * @nr_to_read: Number of pages to load.
749 */
750static int load_image_lzo(struct swap_map_handle *handle,
751 struct snapshot_handle *snapshot,
752 unsigned int nr_to_read)
753{
754 unsigned int m;
755 int error = 0;
756 struct timeval start;
757 struct timeval stop;
758 unsigned nr_pages;
759 size_t off, unc_len, cmp_len;
760 unsigned char *unc, *cmp, *page;
761
762 page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
763 if (!page) {
764 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
765 return -ENOMEM;
766 }
767
768 unc = vmalloc(LZO_UNC_SIZE);
769 if (!unc) {
770 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n");
771 free_page((unsigned long)page);
772 return -ENOMEM;
773 }
774
775 cmp = vmalloc(LZO_CMP_SIZE);
776 if (!cmp) {
777 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n");
778 vfree(unc);
779 free_page((unsigned long)page);
780 return -ENOMEM;
781 }
782
783 printk(KERN_INFO
784 "PM: Loading and decompressing image data (%u pages) ... ",
785 nr_to_read);
786 m = nr_to_read / 100;
787 if (!m)
788 m = 1;
789 nr_pages = 0;
790 do_gettimeofday(&start);
791
792 error = snapshot_write_next(snapshot);
793 if (error <= 0)
794 goto out_finish;
795
796 for (;;) {
797 error = swap_read_page(handle, page, NULL); /* sync */
798 if (error)
799 break;
800
801 cmp_len = *(size_t *)page;
802 if (unlikely(!cmp_len ||
803 cmp_len > lzo1x_worst_compress(LZO_UNC_SIZE))) {
804 printk(KERN_ERR "PM: Invalid LZO compressed length\n");
805 error = -1;
806 break;
807 }
808
809 memcpy(cmp, page, PAGE_SIZE);
810 for (off = PAGE_SIZE; off < LZO_HEADER + cmp_len; off += PAGE_SIZE) {
811 error = swap_read_page(handle, page, NULL); /* sync */
812 if (error)
813 goto out_finish;
814
815 memcpy(cmp + off, page, PAGE_SIZE);
816 }
817
818 unc_len = LZO_UNC_SIZE;
819 error = lzo1x_decompress_safe(cmp + LZO_HEADER, cmp_len,
820 unc, &unc_len);
821 if (error < 0) {
822 printk(KERN_ERR "PM: LZO decompression failed\n");
823 break;
824 }
825
826 if (unlikely(!unc_len ||
827 unc_len > LZO_UNC_SIZE ||
828 unc_len & (PAGE_SIZE - 1))) {
829 printk(KERN_ERR "PM: Invalid LZO uncompressed length\n");
830 error = -1;
831 break;
832 }
833
834 for (off = 0; off < unc_len; off += PAGE_SIZE) {
835 memcpy(data_of(*snapshot), unc + off, PAGE_SIZE);
836
837 if (!(nr_pages % m))
838 printk("\b\b\b\b%3d%%", nr_pages / m);
839 nr_pages++;
840
841 error = snapshot_write_next(snapshot);
842 if (error <= 0)
843 goto out_finish;
844 }
845 }
846
847out_finish:
848 do_gettimeofday(&stop);
849 if (!error) {
850 printk("\b\b\b\bdone\n");
851 snapshot_write_finalize(snapshot);
852 if (!snapshot_image_loaded(snapshot))
853 error = -ENODATA;
854 } else
855 printk("\n");
856 swsusp_show_speed(&start, &stop, nr_to_read, "Read");
857
858 vfree(cmp);
859 vfree(unc);
860 free_page((unsigned long)page);
861
862 return error;
863}
864
865/**
Rafael J. Wysockia634cc12007-07-19 01:47:30 -0700866 * swsusp_read - read the hibernation image.
867 * @flags_p: flags passed by the "frozen" kernel in the image header should
868 * be written into this memeory location
869 */
870
871int swsusp_read(unsigned int *flags_p)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800872{
873 int error;
874 struct swap_map_handle handle;
875 struct snapshot_handle snapshot;
876 struct swsusp_info *header;
877
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800878 memset(&snapshot, 0, sizeof(struct snapshot_handle));
Jiri Slabyd3c1b242010-05-01 23:52:02 +0200879 error = snapshot_write_next(&snapshot);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800880 if (error < PAGE_SIZE)
881 return error < 0 ? error : -EFAULT;
882 header = (struct swsusp_info *)data_of(snapshot);
Jiri Slaby6f612af2010-05-01 23:54:02 +0200883 error = get_swap_reader(&handle, flags_p);
884 if (error)
885 goto end;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800886 if (!error)
Andrew Morton546e0d22006-09-25 23:32:44 -0700887 error = swap_read_page(&handle, header, NULL);
Bojan Smojverf996fc92010-09-09 23:06:23 +0200888 if (!error) {
889 error = (*flags_p & SF_NOCOMPRESS_MODE) ?
890 load_image(&handle, &snapshot, header->pages - 1) :
891 load_image_lzo(&handle, &snapshot, header->pages - 1);
892 }
Jiri Slaby6f612af2010-05-01 23:54:02 +0200893 swap_reader_finish(&handle);
894end:
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800895 if (!error)
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100896 pr_debug("PM: Image successfully loaded\n");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800897 else
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100898 pr_debug("PM: Error %d resuming\n", error);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800899 return error;
900}
901
902/**
903 * swsusp_check - Check for swsusp signature in the resume device
904 */
905
906int swsusp_check(void)
907{
908 int error;
909
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200910 hib_resume_bdev = open_by_devnum(swsusp_resume_device, FMODE_READ);
911 if (!IS_ERR(hib_resume_bdev)) {
912 set_blocksize(hib_resume_bdev, PAGE_SIZE);
OGAWA Hirofumi6373da12007-05-23 13:58:00 -0700913 memset(swsusp_header, 0, PAGE_SIZE);
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200914 error = hib_bio_read_page(swsusp_resume_block,
Vivek Goyal1b29c162007-05-02 19:27:07 +0200915 swsusp_header, NULL);
Rafael J. Wysocki9a154d92006-12-06 20:34:12 -0800916 if (error)
Jiri Slaby76b57e62009-10-07 22:37:35 +0200917 goto put;
Rafael J. Wysocki9a154d92006-12-06 20:34:12 -0800918
Vivek Goyal1b29c162007-05-02 19:27:07 +0200919 if (!memcmp(SWSUSP_SIG, swsusp_header->sig, 10)) {
920 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800921 /* Reset swap signature now */
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200922 error = hib_bio_write_page(swsusp_resume_block,
Vivek Goyal1b29c162007-05-02 19:27:07 +0200923 swsusp_header, NULL);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800924 } else {
Jiri Slaby76b57e62009-10-07 22:37:35 +0200925 error = -EINVAL;
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800926 }
Jiri Slaby76b57e62009-10-07 22:37:35 +0200927
928put:
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800929 if (error)
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200930 blkdev_put(hib_resume_bdev, FMODE_READ);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800931 else
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100932 pr_debug("PM: Signature found, resuming\n");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800933 } else {
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200934 error = PTR_ERR(hib_resume_bdev);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800935 }
936
937 if (error)
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100938 pr_debug("PM: Error %d checking image file\n", error);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800939
940 return error;
941}
942
943/**
944 * swsusp_close - close swap device.
945 */
946
Al Viroc2dd0da2007-10-08 13:21:10 -0400947void swsusp_close(fmode_t mode)
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800948{
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200949 if (IS_ERR(hib_resume_bdev)) {
Rafael J. Wysocki23976722007-12-08 02:09:43 +0100950 pr_debug("PM: Image device not initialised\n");
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800951 return;
952 }
953
Jiri Slaby8a0d6132010-05-01 23:52:34 +0200954 blkdev_put(hib_resume_bdev, mode);
Rafael J. Wysocki61159a32006-03-23 03:00:00 -0800955}
Vivek Goyal1b29c162007-05-02 19:27:07 +0200956
957static int swsusp_header_init(void)
958{
959 swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
960 if (!swsusp_header)
961 panic("Could not allocate memory for swsusp_header\n");
962 return 0;
963}
964
965core_initcall(swsusp_header_init);