blob: 86b6b037fa442bb3b893c712e9bba532c710a331 [file] [log] [blame]
NeilBrown32a76272005-06-21 17:17:14 -07001/*
2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
3 *
4 * bitmap_create - sets up the bitmap structure
5 * bitmap_destroy - destroys the bitmap structure
6 *
7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8 * - added disk storage for bitmap
9 * - changes to allow various bitmap chunk sizes
10 * - added bitmap daemon (to asynchronously clear bitmap bits from disk)
11 */
12
13/*
14 * Still to do:
15 *
16 * flush after percent set rather than just time based. (maybe both).
17 * wait if count gets too high, wake when it drops to half.
18 * allow bitmap to be mirrored with superblock (before or after...)
19 * allow hot-add to re-instate a current device.
20 * allow hot-add of bitmap after quiessing device
21 */
22
23#include <linux/module.h>
24#include <linux/version.h>
25#include <linux/errno.h>
26#include <linux/slab.h>
27#include <linux/init.h>
28#include <linux/config.h>
29#include <linux/timer.h>
30#include <linux/sched.h>
31#include <linux/list.h>
32#include <linux/file.h>
33#include <linux/mount.h>
34#include <linux/buffer_head.h>
35#include <linux/raid/md.h>
36#include <linux/raid/bitmap.h>
37
38/* debug macros */
39
40#define DEBUG 0
41
42#if DEBUG
43/* these are for debugging purposes only! */
44
45/* define one and only one of these */
46#define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
47#define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
48#define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
49#define INJECT_FAULTS_4 0 /* undef */
50#define INJECT_FAULTS_5 0 /* undef */
51#define INJECT_FAULTS_6 0
52
53/* if these are defined, the driver will fail! debug only */
54#define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
55#define INJECT_FATAL_FAULT_2 0 /* undef */
56#define INJECT_FATAL_FAULT_3 0 /* undef */
57#endif
58
59//#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
60#define DPRINTK(x...) do { } while(0)
61
62#ifndef PRINTK
63# if DEBUG > 0
64# define PRINTK(x...) printk(KERN_DEBUG x)
65# else
66# define PRINTK(x...)
67# endif
68#endif
69
70static inline char * bmname(struct bitmap *bitmap)
71{
72 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
73}
74
75
76/*
77 * test if the bitmap is active
78 */
79int bitmap_active(struct bitmap *bitmap)
80{
81 unsigned long flags;
82 int res = 0;
83
84 if (!bitmap)
85 return res;
86 spin_lock_irqsave(&bitmap->lock, flags);
87 res = bitmap->flags & BITMAP_ACTIVE;
88 spin_unlock_irqrestore(&bitmap->lock, flags);
89 return res;
90}
91
92#define WRITE_POOL_SIZE 256
93/* mempool for queueing pending writes on the bitmap file */
94static void *write_pool_alloc(unsigned int gfp_flags, void *data)
95{
96 return kmalloc(sizeof(struct page_list), gfp_flags);
97}
98
99static void write_pool_free(void *ptr, void *data)
100{
101 kfree(ptr);
102}
103
104/*
105 * just a placeholder - calls kmalloc for bitmap pages
106 */
107static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
108{
109 unsigned char *page;
110
111#if INJECT_FAULTS_1
112 page = NULL;
113#else
114 page = kmalloc(PAGE_SIZE, GFP_NOIO);
115#endif
116 if (!page)
117 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
118 else
119 printk("%s: bitmap_alloc_page: allocated page at %p\n",
120 bmname(bitmap), page);
121 return page;
122}
123
124/*
125 * for now just a placeholder -- just calls kfree for bitmap pages
126 */
127static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
128{
129 PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap), page);
130 kfree(page);
131}
132
133/*
134 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
135 *
136 * 1) check to see if this page is allocated, if it's not then try to alloc
137 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
138 * page pointer directly as a counter
139 *
140 * if we find our page, we increment the page's refcount so that it stays
141 * allocated while we're using it
142 */
143static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create)
144{
145 unsigned char *mappage;
146
147 if (page >= bitmap->pages) {
148 printk(KERN_ALERT
149 "%s: invalid bitmap page request: %lu (> %lu)\n",
150 bmname(bitmap), page, bitmap->pages-1);
151 return -EINVAL;
152 }
153
154
155 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
156 return 0;
157
158 if (bitmap->bp[page].map) /* page is already allocated, just return */
159 return 0;
160
161 if (!create)
162 return -ENOENT;
163
164 spin_unlock_irq(&bitmap->lock);
165
166 /* this page has not been allocated yet */
167
168 if ((mappage = bitmap_alloc_page(bitmap)) == NULL) {
169 PRINTK("%s: bitmap map page allocation failed, hijacking\n",
170 bmname(bitmap));
171 /* failed - set the hijacked flag so that we can use the
172 * pointer as a counter */
173 spin_lock_irq(&bitmap->lock);
174 if (!bitmap->bp[page].map)
175 bitmap->bp[page].hijacked = 1;
176 goto out;
177 }
178
179 /* got a page */
180
181 spin_lock_irq(&bitmap->lock);
182
183 /* recheck the page */
184
185 if (bitmap->bp[page].map || bitmap->bp[page].hijacked) {
186 /* somebody beat us to getting the page */
187 bitmap_free_page(bitmap, mappage);
188 return 0;
189 }
190
191 /* no page was in place and we have one, so install it */
192
193 memset(mappage, 0, PAGE_SIZE);
194 bitmap->bp[page].map = mappage;
195 bitmap->missing_pages--;
196out:
197 return 0;
198}
199
200
201/* if page is completely empty, put it back on the free list, or dealloc it */
202/* if page was hijacked, unmark the flag so it might get alloced next time */
203/* Note: lock should be held when calling this */
204static inline void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
205{
206 char *ptr;
207
208 if (bitmap->bp[page].count) /* page is still busy */
209 return;
210
211 /* page is no longer in use, it can be released */
212
213 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
214 bitmap->bp[page].hijacked = 0;
215 bitmap->bp[page].map = NULL;
216 return;
217 }
218
219 /* normal case, free the page */
220
221#if 0
222/* actually ... let's not. We will probably need the page again exactly when
223 * memory is tight and we are flusing to disk
224 */
225 return;
226#else
227 ptr = bitmap->bp[page].map;
228 bitmap->bp[page].map = NULL;
229 bitmap->missing_pages++;
230 bitmap_free_page(bitmap, ptr);
231 return;
232#endif
233}
234
235
236/*
237 * bitmap file handling - read and write the bitmap file and its superblock
238 */
239
240/* copy the pathname of a file to a buffer */
241char *file_path(struct file *file, char *buf, int count)
242{
243 struct dentry *d;
244 struct vfsmount *v;
245
246 if (!buf)
247 return NULL;
248
249 d = file->f_dentry;
250 v = file->f_vfsmnt;
251
252 buf = d_path(d, v, buf, count);
253
254 return IS_ERR(buf) ? NULL : buf;
255}
256
257/*
258 * basic page I/O operations
259 */
260
261/*
262 * write out a page
263 */
NeilBrown77ad4bc2005-06-21 17:17:21 -0700264static int write_page(struct bitmap *bitmap, struct page *page, int wait)
NeilBrown32a76272005-06-21 17:17:14 -0700265{
266 int ret = -ENOMEM;
267
268 lock_page(page);
269
NeilBrown32a76272005-06-21 17:17:14 -0700270 ret = page->mapping->a_ops->prepare_write(NULL, page, 0, PAGE_SIZE);
271 if (!ret)
272 ret = page->mapping->a_ops->commit_write(NULL, page, 0,
273 PAGE_SIZE);
274 if (ret) {
NeilBrown32a76272005-06-21 17:17:14 -0700275 unlock_page(page);
276 return ret;
277 }
278
279 set_page_dirty(page); /* force it to be written out */
NeilBrown77ad4bc2005-06-21 17:17:21 -0700280
281 if (!wait) {
282 /* add to list to be waited for by daemon */
283 struct page_list *item = mempool_alloc(bitmap->write_pool, GFP_NOIO);
284 item->page = page;
285 page_cache_get(page);
286 spin_lock(&bitmap->write_lock);
287 list_add(&item->list, &bitmap->complete_pages);
288 spin_unlock(&bitmap->write_lock);
289 md_wakeup_thread(bitmap->writeback_daemon);
290 }
NeilBrown32a76272005-06-21 17:17:14 -0700291 return write_one_page(page, wait);
292}
293
294/* read a page from a file, pinning it into cache, and return bytes_read */
295static struct page *read_page(struct file *file, unsigned long index,
296 unsigned long *bytes_read)
297{
298 struct inode *inode = file->f_mapping->host;
299 struct page *page = NULL;
300 loff_t isize = i_size_read(inode);
301 unsigned long end_index = isize >> PAGE_CACHE_SHIFT;
302
303 PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_CACHE_SIZE,
304 (unsigned long long)index << PAGE_CACHE_SHIFT);
305
306 page = read_cache_page(inode->i_mapping, index,
307 (filler_t *)inode->i_mapping->a_ops->readpage, file);
308 if (IS_ERR(page))
309 goto out;
310 wait_on_page_locked(page);
311 if (!PageUptodate(page) || PageError(page)) {
312 page_cache_release(page);
313 page = ERR_PTR(-EIO);
314 goto out;
315 }
316
317 if (index > end_index) /* we have read beyond EOF */
318 *bytes_read = 0;
319 else if (index == end_index) /* possible short read */
320 *bytes_read = isize & ~PAGE_CACHE_MASK;
321 else
322 *bytes_read = PAGE_CACHE_SIZE; /* got a full page */
323out:
324 if (IS_ERR(page))
325 printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n",
326 (int)PAGE_CACHE_SIZE,
327 (unsigned long long)index << PAGE_CACHE_SHIFT,
328 PTR_ERR(page));
329 return page;
330}
331
332/*
333 * bitmap file superblock operations
334 */
335
336/* update the event counter and sync the superblock to disk */
337int bitmap_update_sb(struct bitmap *bitmap)
338{
339 bitmap_super_t *sb;
340 unsigned long flags;
341
342 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
343 return 0;
344 spin_lock_irqsave(&bitmap->lock, flags);
345 if (!bitmap->sb_page) { /* no superblock */
346 spin_unlock_irqrestore(&bitmap->lock, flags);
347 return 0;
348 }
NeilBrown32a76272005-06-21 17:17:14 -0700349 spin_unlock_irqrestore(&bitmap->lock, flags);
350 sb = (bitmap_super_t *)kmap(bitmap->sb_page);
351 sb->events = cpu_to_le64(bitmap->mddev->events);
352 if (!bitmap->mddev->degraded)
353 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
354 kunmap(bitmap->sb_page);
NeilBrown77ad4bc2005-06-21 17:17:21 -0700355 return write_page(bitmap, bitmap->sb_page, 0);
NeilBrown32a76272005-06-21 17:17:14 -0700356}
357
358/* print out the bitmap file superblock */
359void bitmap_print_sb(struct bitmap *bitmap)
360{
361 bitmap_super_t *sb;
362
363 if (!bitmap || !bitmap->sb_page)
364 return;
365 sb = (bitmap_super_t *)kmap(bitmap->sb_page);
366 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
NeilBrowna2cff262005-06-21 17:17:20 -0700367 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
368 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
369 printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
NeilBrown32a76272005-06-21 17:17:14 -0700370 *(__u32 *)(sb->uuid+0),
371 *(__u32 *)(sb->uuid+4),
372 *(__u32 *)(sb->uuid+8),
373 *(__u32 *)(sb->uuid+12));
NeilBrowna2cff262005-06-21 17:17:20 -0700374 printk(KERN_DEBUG " events: %llu\n",
NeilBrown32a76272005-06-21 17:17:14 -0700375 (unsigned long long) le64_to_cpu(sb->events));
NeilBrowna2cff262005-06-21 17:17:20 -0700376 printk(KERN_DEBUG "events cleared: %llu\n",
NeilBrown32a76272005-06-21 17:17:14 -0700377 (unsigned long long) le64_to_cpu(sb->events_cleared));
NeilBrowna2cff262005-06-21 17:17:20 -0700378 printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
379 printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
380 printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
381 printk(KERN_DEBUG " sync size: %llu KB\n",
382 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
NeilBrown32a76272005-06-21 17:17:14 -0700383 kunmap(bitmap->sb_page);
384}
385
386/* read the superblock from the bitmap file and initialize some bitmap fields */
387static int bitmap_read_sb(struct bitmap *bitmap)
388{
389 char *reason = NULL;
390 bitmap_super_t *sb;
391 unsigned long chunksize, daemon_sleep;
392 unsigned long bytes_read;
393 unsigned long long events;
394 int err = -EINVAL;
395
396 /* page 0 is the superblock, read it... */
397 bitmap->sb_page = read_page(bitmap->file, 0, &bytes_read);
398 if (IS_ERR(bitmap->sb_page)) {
399 err = PTR_ERR(bitmap->sb_page);
400 bitmap->sb_page = NULL;
401 return err;
402 }
403
404 sb = (bitmap_super_t *)kmap(bitmap->sb_page);
405
406 if (bytes_read < sizeof(*sb)) { /* short read */
407 printk(KERN_INFO "%s: bitmap file superblock truncated\n",
408 bmname(bitmap));
409 err = -ENOSPC;
410 goto out;
411 }
412
413 chunksize = le32_to_cpu(sb->chunksize);
414 daemon_sleep = le32_to_cpu(sb->daemon_sleep);
415
416 /* verify that the bitmap-specific fields are valid */
417 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
418 reason = "bad magic";
419 else if (sb->version != cpu_to_le32(BITMAP_MAJOR))
420 reason = "unrecognized superblock version";
421 else if (chunksize < 512 || chunksize > (1024 * 1024 * 4))
422 reason = "bitmap chunksize out of range (512B - 4MB)";
423 else if ((1 << ffz(~chunksize)) != chunksize)
424 reason = "bitmap chunksize not a power of 2";
425 else if (daemon_sleep < 1 || daemon_sleep > 15)
426 reason = "daemon sleep period out of range";
427 if (reason) {
428 printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
429 bmname(bitmap), reason);
430 goto out;
431 }
432
433 /* keep the array size field of the bitmap superblock up to date */
434 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
435
436 if (!bitmap->mddev->persistent)
437 goto success;
438
439 /*
440 * if we have a persistent array superblock, compare the
441 * bitmap's UUID and event counter to the mddev's
442 */
443 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
444 printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n",
445 bmname(bitmap));
446 goto out;
447 }
448 events = le64_to_cpu(sb->events);
449 if (events < bitmap->mddev->events) {
450 printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) "
451 "-- forcing full recovery\n", bmname(bitmap), events,
452 (unsigned long long) bitmap->mddev->events);
453 sb->state |= BITMAP_STALE;
454 }
455success:
456 /* assign fields using values from superblock */
457 bitmap->chunksize = chunksize;
458 bitmap->daemon_sleep = daemon_sleep;
459 bitmap->flags |= sb->state;
460 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
461 err = 0;
462out:
463 kunmap(bitmap->sb_page);
464 if (err)
465 bitmap_print_sb(bitmap);
466 return err;
467}
468
469enum bitmap_mask_op {
470 MASK_SET,
471 MASK_UNSET
472};
473
474/* record the state of the bitmap in the superblock */
475static void bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
476 enum bitmap_mask_op op)
477{
478 bitmap_super_t *sb;
479 unsigned long flags;
480
481 spin_lock_irqsave(&bitmap->lock, flags);
482 if (!bitmap || !bitmap->sb_page) { /* can't set the state */
483 spin_unlock_irqrestore(&bitmap->lock, flags);
484 return;
485 }
486 page_cache_get(bitmap->sb_page);
487 spin_unlock_irqrestore(&bitmap->lock, flags);
488 sb = (bitmap_super_t *)kmap(bitmap->sb_page);
489 switch (op) {
490 case MASK_SET: sb->state |= bits;
491 break;
492 case MASK_UNSET: sb->state &= ~bits;
493 break;
494 default: BUG();
495 }
496 kunmap(bitmap->sb_page);
497 page_cache_release(bitmap->sb_page);
498}
499
500/*
501 * general bitmap file operations
502 */
503
504/* calculate the index of the page that contains this bit */
505static inline unsigned long file_page_index(unsigned long chunk)
506{
507 return CHUNK_BIT_OFFSET(chunk) >> PAGE_BIT_SHIFT;
508}
509
510/* calculate the (bit) offset of this bit within a page */
511static inline unsigned long file_page_offset(unsigned long chunk)
512{
513 return CHUNK_BIT_OFFSET(chunk) & (PAGE_BITS - 1);
514}
515
516/*
517 * return a pointer to the page in the filemap that contains the given bit
518 *
519 * this lookup is complicated by the fact that the bitmap sb might be exactly
520 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
521 * 0 or page 1
522 */
523static inline struct page *filemap_get_page(struct bitmap *bitmap,
524 unsigned long chunk)
525{
526 return bitmap->filemap[file_page_index(chunk) - file_page_index(0)];
527}
528
529
530static void bitmap_file_unmap(struct bitmap *bitmap)
531{
532 struct page **map, *sb_page;
533 unsigned long *attr;
534 int pages;
535 unsigned long flags;
536
537 spin_lock_irqsave(&bitmap->lock, flags);
538 map = bitmap->filemap;
539 bitmap->filemap = NULL;
540 attr = bitmap->filemap_attr;
541 bitmap->filemap_attr = NULL;
542 pages = bitmap->file_pages;
543 bitmap->file_pages = 0;
544 sb_page = bitmap->sb_page;
545 bitmap->sb_page = NULL;
546 spin_unlock_irqrestore(&bitmap->lock, flags);
547
548 while (pages--)
549 if (map[pages]->index != 0) /* 0 is sb_page, release it below */
550 page_cache_release(map[pages]);
551 kfree(map);
552 kfree(attr);
553
554 if (sb_page)
555 page_cache_release(sb_page);
556}
557
558static void bitmap_stop_daemons(struct bitmap *bitmap);
559
560/* dequeue the next item in a page list -- don't call from irq context */
NeilBrown77ad4bc2005-06-21 17:17:21 -0700561static struct page_list *dequeue_page(struct bitmap *bitmap)
NeilBrown32a76272005-06-21 17:17:14 -0700562{
563 struct page_list *item = NULL;
NeilBrown77ad4bc2005-06-21 17:17:21 -0700564 struct list_head *head = &bitmap->complete_pages;
NeilBrown32a76272005-06-21 17:17:14 -0700565
566 spin_lock(&bitmap->write_lock);
567 if (list_empty(head))
568 goto out;
569 item = list_entry(head->prev, struct page_list, list);
570 list_del(head->prev);
571out:
572 spin_unlock(&bitmap->write_lock);
573 return item;
574}
575
576static void drain_write_queues(struct bitmap *bitmap)
577{
NeilBrown32a76272005-06-21 17:17:14 -0700578 struct page_list *item;
NeilBrown32a76272005-06-21 17:17:14 -0700579
NeilBrown77ad4bc2005-06-21 17:17:21 -0700580 while ((item = dequeue_page(bitmap))) {
581 /* don't bother to wait */
582 page_cache_release(item->page);
583 mempool_free(item, bitmap->write_pool);
NeilBrown32a76272005-06-21 17:17:14 -0700584 }
585
NeilBrown32a76272005-06-21 17:17:14 -0700586 wake_up(&bitmap->write_wait);
NeilBrown32a76272005-06-21 17:17:14 -0700587}
588
589static void bitmap_file_put(struct bitmap *bitmap)
590{
591 struct file *file;
592 struct inode *inode;
593 unsigned long flags;
594
595 spin_lock_irqsave(&bitmap->lock, flags);
596 file = bitmap->file;
597 bitmap->file = NULL;
598 spin_unlock_irqrestore(&bitmap->lock, flags);
599
600 bitmap_stop_daemons(bitmap);
601
602 drain_write_queues(bitmap);
603
604 bitmap_file_unmap(bitmap);
605
606 if (file) {
607 inode = file->f_mapping->host;
608 spin_lock(&inode->i_lock);
609 atomic_set(&inode->i_writecount, 1); /* allow writes again */
610 spin_unlock(&inode->i_lock);
611 fput(file);
612 }
613}
614
615
616/*
617 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
618 * then it is no longer reliable, so we stop using it and we mark the file
619 * as failed in the superblock
620 */
621static void bitmap_file_kick(struct bitmap *bitmap)
622{
623 char *path, *ptr = NULL;
624
625 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET);
626 bitmap_update_sb(bitmap);
627
628 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
629 if (path)
630 ptr = file_path(bitmap->file, path, PAGE_SIZE);
631
632 printk(KERN_ALERT "%s: kicking failed bitmap file %s from array!\n",
633 bmname(bitmap), ptr ? ptr : "");
634
635 kfree(path);
636
637 bitmap_file_put(bitmap);
638
639 return;
640}
641
642enum bitmap_page_attr {
643 BITMAP_PAGE_DIRTY = 1, // there are set bits that need to be synced
644 BITMAP_PAGE_CLEAN = 2, // there are bits that might need to be cleared
645 BITMAP_PAGE_NEEDWRITE=4, // there are cleared bits that need to be synced
646};
647
648static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
649 enum bitmap_page_attr attr)
650{
651 bitmap->filemap_attr[page->index] |= attr;
652}
653
654static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
655 enum bitmap_page_attr attr)
656{
657 bitmap->filemap_attr[page->index] &= ~attr;
658}
659
660static inline unsigned long get_page_attr(struct bitmap *bitmap, struct page *page)
661{
662 return bitmap->filemap_attr[page->index];
663}
664
665/*
666 * bitmap_file_set_bit -- called before performing a write to the md device
667 * to set (and eventually sync) a particular bit in the bitmap file
668 *
669 * we set the bit immediately, then we record the page number so that
670 * when an unplug occurs, we can flush the dirty pages out to disk
671 */
672static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
673{
674 unsigned long bit;
675 struct page *page;
676 void *kaddr;
677 unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
678
679 if (!bitmap->file || !bitmap->filemap) {
680 return;
681 }
682
683 page = filemap_get_page(bitmap, chunk);
684 bit = file_page_offset(chunk);
685
686
687 /* make sure the page stays cached until it gets written out */
688 if (! (get_page_attr(bitmap, page) & BITMAP_PAGE_DIRTY))
689 page_cache_get(page);
690
691 /* set the bit */
692 kaddr = kmap_atomic(page, KM_USER0);
693 set_bit(bit, kaddr);
694 kunmap_atomic(kaddr, KM_USER0);
695 PRINTK("set file bit %lu page %lu\n", bit, page->index);
696
697 /* record page number so it gets flushed to disk when unplug occurs */
698 set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
699
700}
701
702/* this gets called when the md device is ready to unplug its underlying
703 * (slave) device queues -- before we let any writes go down, we need to
704 * sync the dirty pages of the bitmap file to disk */
705int bitmap_unplug(struct bitmap *bitmap)
706{
707 unsigned long i, attr, flags;
708 struct page *page;
709 int wait = 0;
710
711 if (!bitmap)
712 return 0;
713
714 /* look at each page to see if there are any set bits that need to be
715 * flushed out to disk */
716 for (i = 0; i < bitmap->file_pages; i++) {
717 spin_lock_irqsave(&bitmap->lock, flags);
718 if (!bitmap->file || !bitmap->filemap) {
719 spin_unlock_irqrestore(&bitmap->lock, flags);
720 return 0;
721 }
722 page = bitmap->filemap[i];
723 attr = get_page_attr(bitmap, page);
724 clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
725 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
726 if ((attr & BITMAP_PAGE_DIRTY))
727 wait = 1;
728 spin_unlock_irqrestore(&bitmap->lock, flags);
729
730 if (attr & (BITMAP_PAGE_DIRTY | BITMAP_PAGE_NEEDWRITE))
NeilBrown77ad4bc2005-06-21 17:17:21 -0700731 if (write_page(bitmap, page, 0))
NeilBrownbfb39fb2005-06-21 17:17:20 -0700732 return 1;
NeilBrown32a76272005-06-21 17:17:14 -0700733 }
734 if (wait) { /* if any writes were performed, we need to wait on them */
735 spin_lock_irq(&bitmap->write_lock);
736 wait_event_lock_irq(bitmap->write_wait,
NeilBrown77ad4bc2005-06-21 17:17:21 -0700737 list_empty(&bitmap->complete_pages), bitmap->write_lock,
NeilBrown32a76272005-06-21 17:17:14 -0700738 wake_up_process(bitmap->writeback_daemon->tsk));
739 spin_unlock_irq(&bitmap->write_lock);
740 }
741 return 0;
742}
743
744static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset,
NeilBrowncdbb4cc2005-06-21 17:17:18 -0700745 unsigned long sectors, int in_sync);
NeilBrown32a76272005-06-21 17:17:14 -0700746/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
747 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
748 * memory mapping of the bitmap file
749 * Special cases:
750 * if there's no bitmap file, or if the bitmap file had been
751 * previously kicked from the array, we mark all the bits as
752 * 1's in order to cause a full resync.
753 */
NeilBrowncdbb4cc2005-06-21 17:17:18 -0700754static int bitmap_init_from_disk(struct bitmap *bitmap, int in_sync)
NeilBrown32a76272005-06-21 17:17:14 -0700755{
756 unsigned long i, chunks, index, oldindex, bit;
757 struct page *page = NULL, *oldpage = NULL;
758 unsigned long num_pages, bit_cnt = 0;
759 struct file *file;
760 unsigned long bytes, offset, dummy;
761 int outofdate;
762 int ret = -ENOSPC;
763
764 chunks = bitmap->chunks;
765 file = bitmap->file;
766
NeilBrown78d742d2005-06-21 17:17:15 -0700767 BUG_ON(!file);
NeilBrown32a76272005-06-21 17:17:14 -0700768
769#if INJECT_FAULTS_3
770 outofdate = 1;
771#else
772 outofdate = bitmap->flags & BITMAP_STALE;
773#endif
774 if (outofdate)
775 printk(KERN_INFO "%s: bitmap file is out of date, doing full "
776 "recovery\n", bmname(bitmap));
777
778 bytes = (chunks + 7) / 8;
NeilBrownbc7f77d2005-06-21 17:17:17 -0700779
NeilBrowncdbb4cc2005-06-21 17:17:18 -0700780 num_pages = (bytes + sizeof(bitmap_super_t) + PAGE_SIZE - 1) / PAGE_SIZE;
NeilBrownbc7f77d2005-06-21 17:17:17 -0700781
NeilBrown32a76272005-06-21 17:17:14 -0700782 if (i_size_read(file->f_mapping->host) < bytes + sizeof(bitmap_super_t)) {
783 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
784 bmname(bitmap),
785 (unsigned long) i_size_read(file->f_mapping->host),
786 bytes + sizeof(bitmap_super_t));
787 goto out;
788 }
NeilBrownbc7f77d2005-06-21 17:17:17 -0700789
790 ret = -ENOMEM;
791
NeilBrown32a76272005-06-21 17:17:14 -0700792 bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
NeilBrownbc7f77d2005-06-21 17:17:17 -0700793 if (!bitmap->filemap)
NeilBrown32a76272005-06-21 17:17:14 -0700794 goto out;
NeilBrown32a76272005-06-21 17:17:14 -0700795
796 bitmap->filemap_attr = kmalloc(sizeof(long) * num_pages, GFP_KERNEL);
NeilBrownbc7f77d2005-06-21 17:17:17 -0700797 if (!bitmap->filemap_attr)
NeilBrown32a76272005-06-21 17:17:14 -0700798 goto out;
NeilBrown32a76272005-06-21 17:17:14 -0700799
800 memset(bitmap->filemap_attr, 0, sizeof(long) * num_pages);
801
802 oldindex = ~0L;
803
804 for (i = 0; i < chunks; i++) {
805 index = file_page_index(i);
806 bit = file_page_offset(i);
807 if (index != oldindex) { /* this is a new page, read it in */
808 /* unmap the old page, we're done with it */
809 if (oldpage != NULL)
810 kunmap(oldpage);
811 if (index == 0) {
812 /*
813 * if we're here then the superblock page
814 * contains some bits (PAGE_SIZE != sizeof sb)
815 * we've already read it in, so just use it
816 */
817 page = bitmap->sb_page;
818 offset = sizeof(bitmap_super_t);
819 } else {
820 page = read_page(file, index, &dummy);
821 if (IS_ERR(page)) { /* read error */
822 ret = PTR_ERR(page);
823 goto out;
824 }
825 offset = 0;
826 }
827 oldindex = index;
828 oldpage = page;
829 kmap(page);
830
831 if (outofdate) {
832 /*
833 * if bitmap is out of date, dirty the
834 * whole page and write it out
835 */
836 memset(page_address(page) + offset, 0xff,
837 PAGE_SIZE - offset);
NeilBrown77ad4bc2005-06-21 17:17:21 -0700838 ret = write_page(bitmap, page, 1);
NeilBrown32a76272005-06-21 17:17:14 -0700839 if (ret) {
840 kunmap(page);
841 /* release, page not in filemap yet */
842 page_cache_release(page);
843 goto out;
844 }
845 }
846
847 bitmap->filemap[bitmap->file_pages++] = page;
848 }
849 if (test_bit(bit, page_address(page))) {
850 /* if the disk bit is set, set the memory bit */
851 bitmap_set_memory_bits(bitmap,
NeilBrowncdbb4cc2005-06-21 17:17:18 -0700852 i << CHUNK_BLOCK_SHIFT(bitmap), 1, in_sync);
NeilBrown32a76272005-06-21 17:17:14 -0700853 bit_cnt++;
854 }
NeilBrown32a76272005-06-21 17:17:14 -0700855 }
856
857 /* everything went OK */
858 ret = 0;
859 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
860
861 if (page) /* unmap the last page */
862 kunmap(page);
863
864 if (bit_cnt) { /* Kick recovery if any bits were set */
865 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
866 md_wakeup_thread(bitmap->mddev->thread);
867 }
868
869out:
870 printk(KERN_INFO "%s: bitmap initialized from disk: "
871 "read %lu/%lu pages, set %lu bits, status: %d\n",
872 bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt, ret);
873
874 return ret;
875}
876
877
878static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
879{
880 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
881 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
882 bitmap->bp[page].count += inc;
883/*
884 if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
885 (unsigned long long)offset, inc, bitmap->bp[page].count);
886*/
887 bitmap_checkfree(bitmap, page);
888}
889static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
890 sector_t offset, int *blocks,
891 int create);
892
893/*
894 * bitmap daemon -- periodically wakes up to clean bits and flush pages
895 * out to disk
896 */
897
898int bitmap_daemon_work(struct bitmap *bitmap)
899{
900 unsigned long bit, j;
901 unsigned long flags;
902 struct page *page = NULL, *lastpage = NULL;
903 int err = 0;
904 int blocks;
905 int attr;
906
907 if (bitmap == NULL)
908 return 0;
909 if (time_before(jiffies, bitmap->daemon_lastrun + bitmap->daemon_sleep*HZ))
910 return 0;
911 bitmap->daemon_lastrun = jiffies;
912
913 for (j = 0; j < bitmap->chunks; j++) {
914 bitmap_counter_t *bmc;
915 spin_lock_irqsave(&bitmap->lock, flags);
916 if (!bitmap->file || !bitmap->filemap) {
917 /* error or shutdown */
918 spin_unlock_irqrestore(&bitmap->lock, flags);
919 break;
920 }
921
922 page = filemap_get_page(bitmap, j);
923 /* skip this page unless it's marked as needing cleaning */
924 if (!((attr=get_page_attr(bitmap, page)) & BITMAP_PAGE_CLEAN)) {
925 if (attr & BITMAP_PAGE_NEEDWRITE) {
926 page_cache_get(page);
927 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
928 }
929 spin_unlock_irqrestore(&bitmap->lock, flags);
930 if (attr & BITMAP_PAGE_NEEDWRITE) {
NeilBrown77ad4bc2005-06-21 17:17:21 -0700931 if (write_page(bitmap, page, 0))
NeilBrown32a76272005-06-21 17:17:14 -0700932 bitmap_file_kick(bitmap);
933 page_cache_release(page);
934 }
935 continue;
936 }
937
938 bit = file_page_offset(j);
939
940 if (page != lastpage) {
941 /* grab the new page, sync and release the old */
942 page_cache_get(page);
943 if (lastpage != NULL) {
944 if (get_page_attr(bitmap, lastpage) & BITMAP_PAGE_NEEDWRITE) {
945 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
946 spin_unlock_irqrestore(&bitmap->lock, flags);
NeilBrown77ad4bc2005-06-21 17:17:21 -0700947 err = write_page(bitmap, lastpage, 0);
NeilBrown32a76272005-06-21 17:17:14 -0700948 } else {
949 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
950 spin_unlock_irqrestore(&bitmap->lock, flags);
951 }
952 kunmap(lastpage);
953 page_cache_release(lastpage);
954 if (err)
955 bitmap_file_kick(bitmap);
956 } else
957 spin_unlock_irqrestore(&bitmap->lock, flags);
958 lastpage = page;
959 kmap(page);
960/*
961 printk("bitmap clean at page %lu\n", j);
962*/
963 spin_lock_irqsave(&bitmap->lock, flags);
964 clear_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
965 }
966 bmc = bitmap_get_counter(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
967 &blocks, 0);
968 if (bmc) {
969/*
970 if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
971*/
972 if (*bmc == 2) {
973 *bmc=1; /* maybe clear the bit next time */
974 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
975 } else if (*bmc == 1) {
976 /* we can clear the bit */
977 *bmc = 0;
978 bitmap_count_page(bitmap, j << CHUNK_BLOCK_SHIFT(bitmap),
979 -1);
980
981 /* clear the bit */
982 clear_bit(bit, page_address(page));
983 }
984 }
985 spin_unlock_irqrestore(&bitmap->lock, flags);
986 }
987
988 /* now sync the final page */
989 if (lastpage != NULL) {
990 kunmap(lastpage);
991 spin_lock_irqsave(&bitmap->lock, flags);
992 if (get_page_attr(bitmap, lastpage) &BITMAP_PAGE_NEEDWRITE) {
993 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
994 spin_unlock_irqrestore(&bitmap->lock, flags);
NeilBrown77ad4bc2005-06-21 17:17:21 -0700995 err = write_page(bitmap, lastpage, 0);
NeilBrown32a76272005-06-21 17:17:14 -0700996 } else {
997 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
998 spin_unlock_irqrestore(&bitmap->lock, flags);
999 }
1000
1001 page_cache_release(lastpage);
1002 }
1003
1004 return err;
1005}
1006
1007static void daemon_exit(struct bitmap *bitmap, mdk_thread_t **daemon)
1008{
1009 mdk_thread_t *dmn;
1010 unsigned long flags;
1011
1012 /* if no one is waiting on us, we'll free the md thread struct
1013 * and exit, otherwise we let the waiter clean things up */
1014 spin_lock_irqsave(&bitmap->lock, flags);
1015 if ((dmn = *daemon)) { /* no one is waiting, cleanup and exit */
1016 *daemon = NULL;
1017 spin_unlock_irqrestore(&bitmap->lock, flags);
1018 kfree(dmn);
1019 complete_and_exit(NULL, 0); /* do_exit not exported */
1020 }
1021 spin_unlock_irqrestore(&bitmap->lock, flags);
1022}
1023
1024static void bitmap_writeback_daemon(mddev_t *mddev)
1025{
1026 struct bitmap *bitmap = mddev->bitmap;
1027 struct page *page;
1028 struct page_list *item;
1029 int err = 0;
1030
NeilBrown77ad4bc2005-06-21 17:17:21 -07001031 if (signal_pending(current)) {
1032 printk(KERN_INFO
1033 "%s: bitmap writeback daemon got signal, exiting...\n",
1034 bmname(bitmap));
1035 err = -EINTR;
1036 goto out;
1037 }
NeilBrown32a76272005-06-21 17:17:14 -07001038
NeilBrown77ad4bc2005-06-21 17:17:21 -07001039 PRINTK("%s: bitmap writeback daemon woke up...\n", bmname(bitmap));
1040 /* wait on bitmap page writebacks */
1041 while ((item = dequeue_page(bitmap))) {
1042 page = item->page;
1043 mempool_free(item, bitmap->write_pool);
1044 PRINTK("wait on page writeback: %p\n", page);
1045 wait_on_page_writeback(page);
1046 PRINTK("finished page writeback: %p\n", page);
1047
1048 err = PageError(page);
1049 page_cache_release(page);
1050 if (err) {
1051 printk(KERN_WARNING "%s: bitmap file writeback "
1052 "failed (page %lu): %d\n",
1053 bmname(bitmap), page->index, err);
1054 bitmap_file_kick(bitmap);
1055 goto out;
NeilBrown32a76272005-06-21 17:17:14 -07001056 }
1057 }
NeilBrown77ad4bc2005-06-21 17:17:21 -07001058 out:
1059 wake_up(&bitmap->write_wait);
NeilBrown32a76272005-06-21 17:17:14 -07001060 if (err) {
1061 printk(KERN_INFO "%s: bitmap writeback daemon exiting (%d)\n",
NeilBrown77ad4bc2005-06-21 17:17:21 -07001062 bmname(bitmap), err);
NeilBrown32a76272005-06-21 17:17:14 -07001063 daemon_exit(bitmap, &bitmap->writeback_daemon);
1064 }
NeilBrown32a76272005-06-21 17:17:14 -07001065}
1066
1067static int bitmap_start_daemon(struct bitmap *bitmap, mdk_thread_t **ptr,
1068 void (*func)(mddev_t *), char *name)
1069{
1070 mdk_thread_t *daemon;
1071 unsigned long flags;
1072 char namebuf[32];
1073
1074 spin_lock_irqsave(&bitmap->lock, flags);
1075 *ptr = NULL;
1076 if (!bitmap->file) /* no need for daemon if there's no backing file */
1077 goto out_unlock;
1078
1079 spin_unlock_irqrestore(&bitmap->lock, flags);
1080
1081#if INJECT_FATAL_FAULT_2
1082 daemon = NULL;
1083#else
1084 sprintf(namebuf, "%%s_%s", name);
1085 daemon = md_register_thread(func, bitmap->mddev, namebuf);
1086#endif
1087 if (!daemon) {
1088 printk(KERN_ERR "%s: failed to start bitmap daemon\n",
1089 bmname(bitmap));
1090 return -ECHILD;
1091 }
1092
1093 spin_lock_irqsave(&bitmap->lock, flags);
1094 *ptr = daemon;
1095
1096 md_wakeup_thread(daemon); /* start it running */
1097
1098 PRINTK("%s: %s daemon (pid %d) started...\n",
NeilBrownd80a1382005-06-21 17:17:17 -07001099 bmname(bitmap), name, daemon->tsk->pid);
NeilBrown32a76272005-06-21 17:17:14 -07001100out_unlock:
1101 spin_unlock_irqrestore(&bitmap->lock, flags);
1102 return 0;
1103}
1104
1105static int bitmap_start_daemons(struct bitmap *bitmap)
1106{
1107 int err = bitmap_start_daemon(bitmap, &bitmap->writeback_daemon,
1108 bitmap_writeback_daemon, "bitmap_wb");
1109 return err;
1110}
1111
1112static void bitmap_stop_daemon(struct bitmap *bitmap, mdk_thread_t **ptr)
1113{
1114 mdk_thread_t *daemon;
1115 unsigned long flags;
1116
1117 spin_lock_irqsave(&bitmap->lock, flags);
1118 daemon = *ptr;
1119 *ptr = NULL;
1120 spin_unlock_irqrestore(&bitmap->lock, flags);
1121 if (daemon)
1122 md_unregister_thread(daemon); /* destroy the thread */
1123}
1124
1125static void bitmap_stop_daemons(struct bitmap *bitmap)
1126{
1127 /* the daemons can't stop themselves... they'll just exit instead... */
1128 if (bitmap->writeback_daemon &&
1129 current->pid != bitmap->writeback_daemon->tsk->pid)
1130 bitmap_stop_daemon(bitmap, &bitmap->writeback_daemon);
1131}
1132
1133static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
1134 sector_t offset, int *blocks,
1135 int create)
1136{
1137 /* If 'create', we might release the lock and reclaim it.
1138 * The lock must have been taken with interrupts enabled.
1139 * If !create, we don't release the lock.
1140 */
1141 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
1142 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1143 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1144 sector_t csize;
1145
1146 if (bitmap_checkpage(bitmap, page, create) < 0) {
1147 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
1148 *blocks = csize - (offset & (csize- 1));
1149 return NULL;
1150 }
1151 /* now locked ... */
1152
1153 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1154 /* should we use the first or second counter field
1155 * of the hijacked pointer? */
1156 int hi = (pageoff > PAGE_COUNTER_MASK);
1157 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
1158 PAGE_COUNTER_SHIFT - 1);
1159 *blocks = csize - (offset & (csize- 1));
1160 return &((bitmap_counter_t *)
1161 &bitmap->bp[page].map)[hi];
1162 } else { /* page is allocated */
1163 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
1164 *blocks = csize - (offset & (csize- 1));
1165 return (bitmap_counter_t *)
1166 &(bitmap->bp[page].map[pageoff]);
1167 }
1168}
1169
1170int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors)
1171{
1172 if (!bitmap) return 0;
1173 while (sectors) {
1174 int blocks;
1175 bitmap_counter_t *bmc;
1176
1177 spin_lock_irq(&bitmap->lock);
1178 bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
1179 if (!bmc) {
1180 spin_unlock_irq(&bitmap->lock);
1181 return 0;
1182 }
1183
1184 switch(*bmc) {
1185 case 0:
1186 bitmap_file_set_bit(bitmap, offset);
1187 bitmap_count_page(bitmap,offset, 1);
1188 blk_plug_device(bitmap->mddev->queue);
1189 /* fall through */
1190 case 1:
1191 *bmc = 2;
1192 }
1193 if ((*bmc & COUNTER_MAX) == COUNTER_MAX) BUG();
1194 (*bmc)++;
1195
1196 spin_unlock_irq(&bitmap->lock);
1197
1198 offset += blocks;
1199 if (sectors > blocks)
1200 sectors -= blocks;
1201 else sectors = 0;
1202 }
1203 return 0;
1204}
1205
1206void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1207 int success)
1208{
1209 if (!bitmap) return;
1210 while (sectors) {
1211 int blocks;
1212 unsigned long flags;
1213 bitmap_counter_t *bmc;
1214
1215 spin_lock_irqsave(&bitmap->lock, flags);
1216 bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
1217 if (!bmc) {
1218 spin_unlock_irqrestore(&bitmap->lock, flags);
1219 return;
1220 }
1221
1222 if (!success && ! (*bmc & NEEDED_MASK))
1223 *bmc |= NEEDED_MASK;
1224
1225 (*bmc)--;
1226 if (*bmc <= 2) {
1227 set_page_attr(bitmap,
1228 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1229 BITMAP_PAGE_CLEAN);
1230 }
1231 spin_unlock_irqrestore(&bitmap->lock, flags);
1232 offset += blocks;
1233 if (sectors > blocks)
1234 sectors -= blocks;
1235 else sectors = 0;
1236 }
1237}
1238
1239int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks)
1240{
1241 bitmap_counter_t *bmc;
1242 int rv;
1243 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1244 *blocks = 1024;
1245 return 1; /* always resync if no bitmap */
1246 }
1247 spin_lock_irq(&bitmap->lock);
1248 bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
1249 rv = 0;
1250 if (bmc) {
1251 /* locked */
1252 if (RESYNC(*bmc))
1253 rv = 1;
1254 else if (NEEDED(*bmc)) {
1255 rv = 1;
1256 *bmc |= RESYNC_MASK;
1257 *bmc &= ~NEEDED_MASK;
1258 }
1259 }
1260 spin_unlock_irq(&bitmap->lock);
1261 return rv;
1262}
1263
1264void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int aborted)
1265{
1266 bitmap_counter_t *bmc;
1267 unsigned long flags;
1268/*
1269 if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
1270*/ if (bitmap == NULL) {
1271 *blocks = 1024;
1272 return;
1273 }
1274 spin_lock_irqsave(&bitmap->lock, flags);
1275 bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
1276 if (bmc == NULL)
1277 goto unlock;
1278 /* locked */
1279/*
1280 if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
1281*/
1282 if (RESYNC(*bmc)) {
1283 *bmc &= ~RESYNC_MASK;
1284
1285 if (!NEEDED(*bmc) && aborted)
1286 *bmc |= NEEDED_MASK;
1287 else {
1288 if (*bmc <= 2) {
1289 set_page_attr(bitmap,
1290 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1291 BITMAP_PAGE_CLEAN);
1292 }
1293 }
1294 }
1295 unlock:
1296 spin_unlock_irqrestore(&bitmap->lock, flags);
1297}
1298
1299void bitmap_close_sync(struct bitmap *bitmap)
1300{
1301 /* Sync has finished, and any bitmap chunks that weren't synced
1302 * properly have been aborted. It remains to us to clear the
1303 * RESYNC bit wherever it is still on
1304 */
1305 sector_t sector = 0;
1306 int blocks;
1307 if (!bitmap) return;
1308 while (sector < bitmap->mddev->resync_max_sectors) {
1309 bitmap_end_sync(bitmap, sector, &blocks, 0);
1310/*
1311 if (sector < 500) printk("bitmap_close_sync: sec %llu blks %d\n",
1312 (unsigned long long)sector, blocks);
1313*/ sector += blocks;
1314 }
1315}
1316
1317static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset,
NeilBrowncdbb4cc2005-06-21 17:17:18 -07001318 unsigned long sectors, int in_sync)
NeilBrown32a76272005-06-21 17:17:14 -07001319{
1320 /* For each chunk covered by any of these sectors, set the
NeilBrowncdbb4cc2005-06-21 17:17:18 -07001321 * counter to 1 and set resync_needed unless in_sync. They should all
NeilBrown32a76272005-06-21 17:17:14 -07001322 * be 0 at this point
1323 */
1324 while (sectors) {
1325 int secs;
1326 bitmap_counter_t *bmc;
1327 spin_lock_irq(&bitmap->lock);
1328 bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
1329 if (!bmc) {
1330 spin_unlock_irq(&bitmap->lock);
1331 return;
1332 }
NeilBrowncdbb4cc2005-06-21 17:17:18 -07001333 if (! *bmc) {
1334 struct page *page;
1335 *bmc = 1 | (in_sync? 0 : NEEDED_MASK);
NeilBrown32a76272005-06-21 17:17:14 -07001336 bitmap_count_page(bitmap, offset, 1);
NeilBrowncdbb4cc2005-06-21 17:17:18 -07001337 page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
1338 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
NeilBrown32a76272005-06-21 17:17:14 -07001339 }
1340 spin_unlock_irq(&bitmap->lock);
1341 if (sectors > secs)
1342 sectors -= secs;
1343 else
1344 sectors = 0;
1345 }
1346}
1347
1348/* dirty the entire bitmap */
1349int bitmap_setallbits(struct bitmap *bitmap)
1350{
1351 unsigned long flags;
1352 unsigned long j;
1353
1354 /* dirty the in-memory bitmap */
1355 bitmap_set_memory_bits(bitmap, 0, bitmap->chunks << CHUNK_BLOCK_SHIFT(bitmap), 1);
1356
1357 /* dirty the bitmap file */
1358 for (j = 0; j < bitmap->file_pages; j++) {
1359 struct page *page = bitmap->filemap[j];
1360
1361 spin_lock_irqsave(&bitmap->lock, flags);
1362 page_cache_get(page);
1363 spin_unlock_irqrestore(&bitmap->lock, flags);
1364 memset(kmap(page), 0xff, PAGE_SIZE);
1365 kunmap(page);
NeilBrown77ad4bc2005-06-21 17:17:21 -07001366 if (write_page(bitmap, page, 0))
NeilBrownbfb39fb2005-06-21 17:17:20 -07001367 return 1;
NeilBrown32a76272005-06-21 17:17:14 -07001368 }
1369
1370 return 0;
1371}
1372
1373/*
1374 * free memory that was allocated
1375 */
1376void bitmap_destroy(mddev_t *mddev)
1377{
1378 unsigned long k, pages;
1379 struct bitmap_page *bp;
1380 struct bitmap *bitmap = mddev->bitmap;
1381
1382 if (!bitmap) /* there was no bitmap */
1383 return;
1384
1385 mddev->bitmap = NULL; /* disconnect from the md device */
1386
1387 /* release the bitmap file and kill the daemon */
1388 bitmap_file_put(bitmap);
1389
1390 bp = bitmap->bp;
1391 pages = bitmap->pages;
1392
1393 /* free all allocated memory */
1394
1395 mempool_destroy(bitmap->write_pool);
1396
1397 if (bp) /* deallocate the page memory */
1398 for (k = 0; k < pages; k++)
1399 if (bp[k].map && !bp[k].hijacked)
1400 kfree(bp[k].map);
1401 kfree(bp);
1402 kfree(bitmap);
1403}
1404
1405/*
1406 * initialize the bitmap structure
1407 * if this returns an error, bitmap_destroy must be called to do clean up
1408 */
1409int bitmap_create(mddev_t *mddev)
1410{
1411 struct bitmap *bitmap;
1412 unsigned long blocks = mddev->resync_max_sectors;
1413 unsigned long chunks;
1414 unsigned long pages;
1415 struct file *file = mddev->bitmap_file;
1416 int err;
1417
1418 BUG_ON(sizeof(bitmap_super_t) != 256);
1419
1420 if (!file) /* bitmap disabled, nothing to do */
1421 return 0;
1422
1423 bitmap = kmalloc(sizeof(*bitmap), GFP_KERNEL);
1424 if (!bitmap)
1425 return -ENOMEM;
1426
1427 memset(bitmap, 0, sizeof(*bitmap));
1428
1429 spin_lock_init(&bitmap->lock);
1430 bitmap->mddev = mddev;
1431 mddev->bitmap = bitmap;
1432
1433 spin_lock_init(&bitmap->write_lock);
NeilBrown32a76272005-06-21 17:17:14 -07001434 INIT_LIST_HEAD(&bitmap->complete_pages);
1435 init_waitqueue_head(&bitmap->write_wait);
1436 bitmap->write_pool = mempool_create(WRITE_POOL_SIZE, write_pool_alloc,
1437 write_pool_free, NULL);
1438 if (!bitmap->write_pool)
1439 return -ENOMEM;
1440
1441 bitmap->file = file;
1442 get_file(file);
1443 /* read superblock from bitmap file (this sets bitmap->chunksize) */
1444 err = bitmap_read_sb(bitmap);
1445 if (err)
1446 return err;
1447
1448 bitmap->chunkshift = find_first_bit(&bitmap->chunksize,
1449 sizeof(bitmap->chunksize));
1450
1451 /* now that chunksize and chunkshift are set, we can use these macros */
1452 chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) /
1453 CHUNK_BLOCK_RATIO(bitmap);
1454 pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
1455
1456 BUG_ON(!pages);
1457
1458 bitmap->chunks = chunks;
1459 bitmap->pages = pages;
1460 bitmap->missing_pages = pages;
1461 bitmap->counter_bits = COUNTER_BITS;
1462
1463 bitmap->syncchunk = ~0UL;
1464
1465#if INJECT_FATAL_FAULT_1
1466 bitmap->bp = NULL;
1467#else
1468 bitmap->bp = kmalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
1469#endif
1470 if (!bitmap->bp)
1471 return -ENOMEM;
1472 memset(bitmap->bp, 0, pages * sizeof(*bitmap->bp));
1473
1474 bitmap->flags |= BITMAP_ACTIVE;
1475
1476 /* now that we have some pages available, initialize the in-memory
1477 * bitmap from the on-disk bitmap */
NeilBrowncdbb4cc2005-06-21 17:17:18 -07001478 err = bitmap_init_from_disk(bitmap, mddev->recovery_cp == MaxSector);
NeilBrown32a76272005-06-21 17:17:14 -07001479 if (err)
1480 return err;
1481
1482 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1483 pages, bmname(bitmap));
1484
1485 /* kick off the bitmap daemons */
1486 err = bitmap_start_daemons(bitmap);
1487 if (err)
1488 return err;
1489 return bitmap_update_sb(bitmap);
1490}
1491
1492/* the bitmap API -- for raid personalities */
1493EXPORT_SYMBOL(bitmap_startwrite);
1494EXPORT_SYMBOL(bitmap_endwrite);
1495EXPORT_SYMBOL(bitmap_start_sync);
1496EXPORT_SYMBOL(bitmap_end_sync);
1497EXPORT_SYMBOL(bitmap_unplug);
1498EXPORT_SYMBOL(bitmap_close_sync);
1499EXPORT_SYMBOL(bitmap_daemon_work);