blob: 9390e9526786e893b1e67ecf3bb43597784437a5 [file] [log] [blame]
Philipp Reisnerb411b362009-09-25 16:07:19 -07001/*
2 drbd_bitmap.c
3
4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
5
6 Copyright (C) 2004-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 2004-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2004-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
9
10 drbd is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 2, or (at your option)
13 any later version.
14
15 drbd is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with drbd; see the file COPYING. If not, write to
22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 */
24
25#include <linux/bitops.h>
26#include <linux/vmalloc.h>
27#include <linux/string.h>
28#include <linux/drbd.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090029#include <linux/slab.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070030#include <asm/kmap_types.h>
31#include "drbd_int.h"
32
33/* OPAQUE outside this file!
34 * interface defined in drbd_int.h
35
36 * convention:
37 * function name drbd_bm_... => used elsewhere, "public".
38 * function name bm_... => internal to implementation, "private".
39
40 * Note that since find_first_bit returns int, at the current granularity of
41 * the bitmap (4KB per byte), this implementation "only" supports up to
42 * 1<<(32+12) == 16 TB...
43 */
44
45/*
46 * NOTE
47 * Access to the *bm_pages is protected by bm_lock.
48 * It is safe to read the other members within the lock.
49 *
50 * drbd_bm_set_bits is called from bio_endio callbacks,
51 * We may be called with irq already disabled,
52 * so we need spin_lock_irqsave().
53 * And we need the kmap_atomic.
54 */
55struct drbd_bitmap {
56 struct page **bm_pages;
57 spinlock_t bm_lock;
58 /* WARNING unsigned long bm_*:
59 * 32bit number of bit offset is just enough for 512 MB bitmap.
60 * it will blow up if we make the bitmap bigger...
61 * not that it makes much sense to have a bitmap that large,
62 * rather change the granularity to 16k or 64k or something.
63 * (that implies other problems, however...)
64 */
65 unsigned long bm_set; /* nr of set bits; THINK maybe atomic_t? */
66 unsigned long bm_bits;
67 size_t bm_words;
68 size_t bm_number_of_pages;
69 sector_t bm_dev_capacity;
Thomas Gleixner8a03ae22010-01-29 20:39:07 +000070 struct mutex bm_change; /* serializes resize operations */
Philipp Reisnerb411b362009-09-25 16:07:19 -070071
72 atomic_t bm_async_io;
73 wait_queue_head_t bm_io_wait;
74
75 unsigned long bm_flags;
76
77 /* debugging aid, in case we are still racy somewhere */
78 char *bm_why;
79 struct task_struct *bm_task;
80};
81
82/* definition of bits in bm_flags */
83#define BM_LOCKED 0
84#define BM_MD_IO_ERROR 1
85#define BM_P_VMALLOCED 2
86
Philipp Reisnerb4ee79d2010-04-01 09:57:40 +020087static int __bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
Philipp Reisnerfd764382010-04-01 09:57:40 +020088 unsigned long e, int val, const enum km_type km);
89
Philipp Reisnerb411b362009-09-25 16:07:19 -070090static int bm_is_locked(struct drbd_bitmap *b)
91{
92 return test_bit(BM_LOCKED, &b->bm_flags);
93}
94
95#define bm_print_lock_info(m) __bm_print_lock_info(m, __func__)
96static void __bm_print_lock_info(struct drbd_conf *mdev, const char *func)
97{
98 struct drbd_bitmap *b = mdev->bitmap;
99 if (!__ratelimit(&drbd_ratelimit_state))
100 return;
101 dev_err(DEV, "FIXME %s in %s, bitmap locked for '%s' by %s\n",
102 current == mdev->receiver.task ? "receiver" :
103 current == mdev->asender.task ? "asender" :
104 current == mdev->worker.task ? "worker" : current->comm,
105 func, b->bm_why ?: "?",
106 b->bm_task == mdev->receiver.task ? "receiver" :
107 b->bm_task == mdev->asender.task ? "asender" :
108 b->bm_task == mdev->worker.task ? "worker" : "?");
109}
110
111void drbd_bm_lock(struct drbd_conf *mdev, char *why)
112{
113 struct drbd_bitmap *b = mdev->bitmap;
114 int trylock_failed;
115
116 if (!b) {
117 dev_err(DEV, "FIXME no bitmap in drbd_bm_lock!?\n");
118 return;
119 }
120
Thomas Gleixner8a03ae22010-01-29 20:39:07 +0000121 trylock_failed = !mutex_trylock(&b->bm_change);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700122
123 if (trylock_failed) {
124 dev_warn(DEV, "%s going to '%s' but bitmap already locked for '%s' by %s\n",
125 current == mdev->receiver.task ? "receiver" :
126 current == mdev->asender.task ? "asender" :
127 current == mdev->worker.task ? "worker" : current->comm,
128 why, b->bm_why ?: "?",
129 b->bm_task == mdev->receiver.task ? "receiver" :
130 b->bm_task == mdev->asender.task ? "asender" :
131 b->bm_task == mdev->worker.task ? "worker" : "?");
Thomas Gleixner8a03ae22010-01-29 20:39:07 +0000132 mutex_lock(&b->bm_change);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700133 }
134 if (__test_and_set_bit(BM_LOCKED, &b->bm_flags))
135 dev_err(DEV, "FIXME bitmap already locked in bm_lock\n");
136
137 b->bm_why = why;
138 b->bm_task = current;
139}
140
141void drbd_bm_unlock(struct drbd_conf *mdev)
142{
143 struct drbd_bitmap *b = mdev->bitmap;
144 if (!b) {
145 dev_err(DEV, "FIXME no bitmap in drbd_bm_unlock!?\n");
146 return;
147 }
148
149 if (!__test_and_clear_bit(BM_LOCKED, &mdev->bitmap->bm_flags))
150 dev_err(DEV, "FIXME bitmap not locked in bm_unlock\n");
151
152 b->bm_why = NULL;
153 b->bm_task = NULL;
Thomas Gleixner8a03ae22010-01-29 20:39:07 +0000154 mutex_unlock(&b->bm_change);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700155}
156
157/* word offset to long pointer */
158static unsigned long *__bm_map_paddr(struct drbd_bitmap *b, unsigned long offset, const enum km_type km)
159{
160 struct page *page;
161 unsigned long page_nr;
162
163 /* page_nr = (word*sizeof(long)) >> PAGE_SHIFT; */
164 page_nr = offset >> (PAGE_SHIFT - LN2_BPL + 3);
165 BUG_ON(page_nr >= b->bm_number_of_pages);
166 page = b->bm_pages[page_nr];
167
168 return (unsigned long *) kmap_atomic(page, km);
169}
170
171static unsigned long * bm_map_paddr(struct drbd_bitmap *b, unsigned long offset)
172{
173 return __bm_map_paddr(b, offset, KM_IRQ1);
174}
175
176static void __bm_unmap(unsigned long *p_addr, const enum km_type km)
177{
178 kunmap_atomic(p_addr, km);
179};
180
181static void bm_unmap(unsigned long *p_addr)
182{
183 return __bm_unmap(p_addr, KM_IRQ1);
184}
185
186/* long word offset of _bitmap_ sector */
187#define S2W(s) ((s)<<(BM_EXT_SHIFT-BM_BLOCK_SHIFT-LN2_BPL))
188/* word offset from start of bitmap to word number _in_page_
189 * modulo longs per page
190#define MLPP(X) ((X) % (PAGE_SIZE/sizeof(long))
191 hm, well, Philipp thinks gcc might not optimze the % into & (... - 1)
192 so do it explicitly:
193 */
194#define MLPP(X) ((X) & ((PAGE_SIZE/sizeof(long))-1))
195
196/* Long words per page */
197#define LWPP (PAGE_SIZE/sizeof(long))
198
199/*
200 * actually most functions herein should take a struct drbd_bitmap*, not a
201 * struct drbd_conf*, but for the debug macros I like to have the mdev around
202 * to be able to report device specific.
203 */
204
205static void bm_free_pages(struct page **pages, unsigned long number)
206{
207 unsigned long i;
208 if (!pages)
209 return;
210
211 for (i = 0; i < number; i++) {
212 if (!pages[i]) {
213 printk(KERN_ALERT "drbd: bm_free_pages tried to free "
214 "a NULL pointer; i=%lu n=%lu\n",
215 i, number);
216 continue;
217 }
218 __free_page(pages[i]);
219 pages[i] = NULL;
220 }
221}
222
223static void bm_vk_free(void *ptr, int v)
224{
225 if (v)
226 vfree(ptr);
227 else
228 kfree(ptr);
229}
230
231/*
232 * "have" and "want" are NUMBER OF PAGES.
233 */
234static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want)
235{
236 struct page **old_pages = b->bm_pages;
237 struct page **new_pages, *page;
238 unsigned int i, bytes, vmalloced = 0;
239 unsigned long have = b->bm_number_of_pages;
240
241 BUG_ON(have == 0 && old_pages != NULL);
242 BUG_ON(have != 0 && old_pages == NULL);
243
244 if (have == want)
245 return old_pages;
246
247 /* Trying kmalloc first, falling back to vmalloc.
248 * GFP_KERNEL is ok, as this is done when a lower level disk is
249 * "attached" to the drbd. Context is receiver thread or cqueue
250 * thread. As we have no disk yet, we are not in the IO path,
251 * not even the IO path of the peer. */
252 bytes = sizeof(struct page *)*want;
253 new_pages = kmalloc(bytes, GFP_KERNEL);
254 if (!new_pages) {
255 new_pages = vmalloc(bytes);
256 if (!new_pages)
257 return NULL;
258 vmalloced = 1;
259 }
260
261 memset(new_pages, 0, bytes);
262 if (want >= have) {
263 for (i = 0; i < have; i++)
264 new_pages[i] = old_pages[i];
265 for (; i < want; i++) {
266 page = alloc_page(GFP_HIGHUSER);
267 if (!page) {
268 bm_free_pages(new_pages + have, i - have);
269 bm_vk_free(new_pages, vmalloced);
270 return NULL;
271 }
272 new_pages[i] = page;
273 }
274 } else {
275 for (i = 0; i < want; i++)
276 new_pages[i] = old_pages[i];
277 /* NOT HERE, we are outside the spinlock!
278 bm_free_pages(old_pages + want, have - want);
279 */
280 }
281
282 if (vmalloced)
283 set_bit(BM_P_VMALLOCED, &b->bm_flags);
284 else
285 clear_bit(BM_P_VMALLOCED, &b->bm_flags);
286
287 return new_pages;
288}
289
290/*
291 * called on driver init only. TODO call when a device is created.
292 * allocates the drbd_bitmap, and stores it in mdev->bitmap.
293 */
294int drbd_bm_init(struct drbd_conf *mdev)
295{
296 struct drbd_bitmap *b = mdev->bitmap;
297 WARN_ON(b != NULL);
298 b = kzalloc(sizeof(struct drbd_bitmap), GFP_KERNEL);
299 if (!b)
300 return -ENOMEM;
301 spin_lock_init(&b->bm_lock);
Thomas Gleixner8a03ae22010-01-29 20:39:07 +0000302 mutex_init(&b->bm_change);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700303 init_waitqueue_head(&b->bm_io_wait);
304
305 mdev->bitmap = b;
306
307 return 0;
308}
309
310sector_t drbd_bm_capacity(struct drbd_conf *mdev)
311{
312 ERR_IF(!mdev->bitmap) return 0;
313 return mdev->bitmap->bm_dev_capacity;
314}
315
316/* called on driver unload. TODO: call when a device is destroyed.
317 */
318void drbd_bm_cleanup(struct drbd_conf *mdev)
319{
320 ERR_IF (!mdev->bitmap) return;
321 bm_free_pages(mdev->bitmap->bm_pages, mdev->bitmap->bm_number_of_pages);
322 bm_vk_free(mdev->bitmap->bm_pages, test_bit(BM_P_VMALLOCED, &mdev->bitmap->bm_flags));
323 kfree(mdev->bitmap);
324 mdev->bitmap = NULL;
325}
326
327/*
328 * since (b->bm_bits % BITS_PER_LONG) != 0,
329 * this masks out the remaining bits.
330 * Returns the number of bits cleared.
331 */
332static int bm_clear_surplus(struct drbd_bitmap *b)
333{
334 const unsigned long mask = (1UL << (b->bm_bits & (BITS_PER_LONG-1))) - 1;
335 size_t w = b->bm_bits >> LN2_BPL;
336 int cleared = 0;
337 unsigned long *p_addr, *bm;
338
339 p_addr = bm_map_paddr(b, w);
340 bm = p_addr + MLPP(w);
341 if (w < b->bm_words) {
342 cleared = hweight_long(*bm & ~mask);
343 *bm &= mask;
344 w++; bm++;
345 }
346
347 if (w < b->bm_words) {
348 cleared += hweight_long(*bm);
349 *bm = 0;
350 }
351 bm_unmap(p_addr);
352 return cleared;
353}
354
355static void bm_set_surplus(struct drbd_bitmap *b)
356{
357 const unsigned long mask = (1UL << (b->bm_bits & (BITS_PER_LONG-1))) - 1;
358 size_t w = b->bm_bits >> LN2_BPL;
359 unsigned long *p_addr, *bm;
360
361 p_addr = bm_map_paddr(b, w);
362 bm = p_addr + MLPP(w);
363 if (w < b->bm_words) {
364 *bm |= ~mask;
365 bm++; w++;
366 }
367
368 if (w < b->bm_words) {
369 *bm = ~(0UL);
370 }
371 bm_unmap(p_addr);
372}
373
374static unsigned long __bm_count_bits(struct drbd_bitmap *b, const int swap_endian)
375{
376 unsigned long *p_addr, *bm, offset = 0;
377 unsigned long bits = 0;
378 unsigned long i, do_now;
379
380 while (offset < b->bm_words) {
381 i = do_now = min_t(size_t, b->bm_words-offset, LWPP);
382 p_addr = __bm_map_paddr(b, offset, KM_USER0);
383 bm = p_addr + MLPP(offset);
384 while (i--) {
385#ifndef __LITTLE_ENDIAN
386 if (swap_endian)
387 *bm = lel_to_cpu(*bm);
388#endif
389 bits += hweight_long(*bm++);
390 }
391 __bm_unmap(p_addr, KM_USER0);
392 offset += do_now;
393 cond_resched();
394 }
395
396 return bits;
397}
398
399static unsigned long bm_count_bits(struct drbd_bitmap *b)
400{
401 return __bm_count_bits(b, 0);
402}
403
404static unsigned long bm_count_bits_swap_endian(struct drbd_bitmap *b)
405{
406 return __bm_count_bits(b, 1);
407}
408
409/* offset and len in long words.*/
410static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)
411{
412 unsigned long *p_addr, *bm;
413 size_t do_now, end;
414
415#define BM_SECTORS_PER_BIT (BM_BLOCK_SIZE/512)
416
417 end = offset + len;
418
419 if (end > b->bm_words) {
420 printk(KERN_ALERT "drbd: bm_memset end > bm_words\n");
421 return;
422 }
423
424 while (offset < end) {
425 do_now = min_t(size_t, ALIGN(offset + 1, LWPP), end) - offset;
426 p_addr = bm_map_paddr(b, offset);
427 bm = p_addr + MLPP(offset);
428 if (bm+do_now > p_addr + LWPP) {
429 printk(KERN_ALERT "drbd: BUG BUG BUG! p_addr:%p bm:%p do_now:%d\n",
430 p_addr, bm, (int)do_now);
431 break; /* breaks to after catch_oob_access_end() only! */
432 }
433 memset(bm, c, do_now * sizeof(long));
434 bm_unmap(p_addr);
435 offset += do_now;
436 }
437}
438
439/*
440 * make sure the bitmap has enough room for the attached storage,
441 * if necessary, resize.
442 * called whenever we may have changed the device size.
443 * returns -ENOMEM if we could not allocate enough memory, 0 on success.
444 * In case this is actually a resize, we copy the old bitmap into the new one.
445 * Otherwise, the bitmap is initialized to all bits set.
446 */
Philipp Reisner02d9a942010-03-24 16:23:03 +0100447int drbd_bm_resize(struct drbd_conf *mdev, sector_t capacity, int set_new_bits)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700448{
449 struct drbd_bitmap *b = mdev->bitmap;
450 unsigned long bits, words, owords, obits, *p_addr, *bm;
451 unsigned long want, have, onpages; /* number of pages */
452 struct page **npages, **opages = NULL;
453 int err = 0, growing;
454 int opages_vmalloced;
455
456 ERR_IF(!b) return -ENOMEM;
457
458 drbd_bm_lock(mdev, "resize");
459
460 dev_info(DEV, "drbd_bm_resize called with capacity == %llu\n",
461 (unsigned long long)capacity);
462
463 if (capacity == b->bm_dev_capacity)
464 goto out;
465
466 opages_vmalloced = test_bit(BM_P_VMALLOCED, &b->bm_flags);
467
468 if (capacity == 0) {
469 spin_lock_irq(&b->bm_lock);
470 opages = b->bm_pages;
471 onpages = b->bm_number_of_pages;
472 owords = b->bm_words;
473 b->bm_pages = NULL;
474 b->bm_number_of_pages =
475 b->bm_set =
476 b->bm_bits =
477 b->bm_words =
478 b->bm_dev_capacity = 0;
479 spin_unlock_irq(&b->bm_lock);
480 bm_free_pages(opages, onpages);
481 bm_vk_free(opages, opages_vmalloced);
482 goto out;
483 }
484 bits = BM_SECT_TO_BIT(ALIGN(capacity, BM_SECT_PER_BIT));
485
486 /* if we would use
487 words = ALIGN(bits,BITS_PER_LONG) >> LN2_BPL;
488 a 32bit host could present the wrong number of words
489 to a 64bit host.
490 */
491 words = ALIGN(bits, 64) >> LN2_BPL;
492
493 if (get_ldev(mdev)) {
494 D_ASSERT((u64)bits <= (((u64)mdev->ldev->md.md_size_sect-MD_BM_OFFSET) << 12));
495 put_ldev(mdev);
496 }
497
498 /* one extra long to catch off by one errors */
499 want = ALIGN((words+1)*sizeof(long), PAGE_SIZE) >> PAGE_SHIFT;
500 have = b->bm_number_of_pages;
501 if (want == have) {
502 D_ASSERT(b->bm_pages != NULL);
503 npages = b->bm_pages;
504 } else {
Andreas Gruenbacher0cf9d272010-12-07 10:43:29 +0100505 if (drbd_insert_fault(mdev, DRBD_FAULT_BM_ALLOC))
Philipp Reisnerb411b362009-09-25 16:07:19 -0700506 npages = NULL;
507 else
508 npages = bm_realloc_pages(b, want);
509 }
510
511 if (!npages) {
512 err = -ENOMEM;
513 goto out;
514 }
515
516 spin_lock_irq(&b->bm_lock);
517 opages = b->bm_pages;
518 owords = b->bm_words;
519 obits = b->bm_bits;
520
521 growing = bits > obits;
Philipp Reisner52236712010-04-28 14:46:57 +0200522 if (opages && growing && set_new_bits)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700523 bm_set_surplus(b);
524
525 b->bm_pages = npages;
526 b->bm_number_of_pages = want;
527 b->bm_bits = bits;
528 b->bm_words = words;
529 b->bm_dev_capacity = capacity;
530
531 if (growing) {
Philipp Reisner02d9a942010-03-24 16:23:03 +0100532 if (set_new_bits) {
533 bm_memset(b, owords, 0xff, words-owords);
534 b->bm_set += bits - obits;
535 } else
536 bm_memset(b, owords, 0x00, words-owords);
537
Philipp Reisnerb411b362009-09-25 16:07:19 -0700538 }
539
540 if (want < have) {
541 /* implicit: (opages != NULL) && (opages != npages) */
542 bm_free_pages(opages + want, have - want);
543 }
544
545 p_addr = bm_map_paddr(b, words);
546 bm = p_addr + MLPP(words);
547 *bm = DRBD_MAGIC;
548 bm_unmap(p_addr);
549
550 (void)bm_clear_surplus(b);
551
552 spin_unlock_irq(&b->bm_lock);
553 if (opages != npages)
554 bm_vk_free(opages, opages_vmalloced);
555 if (!growing)
556 b->bm_set = bm_count_bits(b);
557 dev_info(DEV, "resync bitmap: bits=%lu words=%lu\n", bits, words);
558
559 out:
560 drbd_bm_unlock(mdev);
561 return err;
562}
563
564/* inherently racy:
565 * if not protected by other means, return value may be out of date when
566 * leaving this function...
567 * we still need to lock it, since it is important that this returns
568 * bm_set == 0 precisely.
569 *
570 * maybe bm_set should be atomic_t ?
571 */
Philipp Reisner07782862010-08-31 12:00:50 +0200572unsigned long _drbd_bm_total_weight(struct drbd_conf *mdev)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700573{
574 struct drbd_bitmap *b = mdev->bitmap;
575 unsigned long s;
576 unsigned long flags;
577
578 ERR_IF(!b) return 0;
579 ERR_IF(!b->bm_pages) return 0;
580
581 spin_lock_irqsave(&b->bm_lock, flags);
582 s = b->bm_set;
583 spin_unlock_irqrestore(&b->bm_lock, flags);
584
585 return s;
586}
587
588unsigned long drbd_bm_total_weight(struct drbd_conf *mdev)
589{
590 unsigned long s;
591 /* if I don't have a disk, I don't know about out-of-sync status */
592 if (!get_ldev_if_state(mdev, D_NEGOTIATING))
593 return 0;
594 s = _drbd_bm_total_weight(mdev);
595 put_ldev(mdev);
596 return s;
597}
598
599size_t drbd_bm_words(struct drbd_conf *mdev)
600{
601 struct drbd_bitmap *b = mdev->bitmap;
602 ERR_IF(!b) return 0;
603 ERR_IF(!b->bm_pages) return 0;
604
605 return b->bm_words;
606}
607
608unsigned long drbd_bm_bits(struct drbd_conf *mdev)
609{
610 struct drbd_bitmap *b = mdev->bitmap;
611 ERR_IF(!b) return 0;
612
613 return b->bm_bits;
614}
615
616/* merge number words from buffer into the bitmap starting at offset.
617 * buffer[i] is expected to be little endian unsigned long.
618 * bitmap must be locked by drbd_bm_lock.
619 * currently only used from receive_bitmap.
620 */
621void drbd_bm_merge_lel(struct drbd_conf *mdev, size_t offset, size_t number,
622 unsigned long *buffer)
623{
624 struct drbd_bitmap *b = mdev->bitmap;
625 unsigned long *p_addr, *bm;
626 unsigned long word, bits;
627 size_t end, do_now;
628
629 end = offset + number;
630
631 ERR_IF(!b) return;
632 ERR_IF(!b->bm_pages) return;
633 if (number == 0)
634 return;
635 WARN_ON(offset >= b->bm_words);
636 WARN_ON(end > b->bm_words);
637
638 spin_lock_irq(&b->bm_lock);
639 while (offset < end) {
640 do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
641 p_addr = bm_map_paddr(b, offset);
642 bm = p_addr + MLPP(offset);
643 offset += do_now;
644 while (do_now--) {
645 bits = hweight_long(*bm);
646 word = *bm | lel_to_cpu(*buffer++);
647 *bm++ = word;
648 b->bm_set += hweight_long(word) - bits;
649 }
650 bm_unmap(p_addr);
651 }
652 /* with 32bit <-> 64bit cross-platform connect
653 * this is only correct for current usage,
654 * where we _know_ that we are 64 bit aligned,
655 * and know that this function is used in this way, too...
656 */
657 if (end == b->bm_words)
658 b->bm_set -= bm_clear_surplus(b);
659
660 spin_unlock_irq(&b->bm_lock);
661}
662
663/* copy number words from the bitmap starting at offset into the buffer.
664 * buffer[i] will be little endian unsigned long.
665 */
666void drbd_bm_get_lel(struct drbd_conf *mdev, size_t offset, size_t number,
667 unsigned long *buffer)
668{
669 struct drbd_bitmap *b = mdev->bitmap;
670 unsigned long *p_addr, *bm;
671 size_t end, do_now;
672
673 end = offset + number;
674
675 ERR_IF(!b) return;
676 ERR_IF(!b->bm_pages) return;
677
678 spin_lock_irq(&b->bm_lock);
679 if ((offset >= b->bm_words) ||
680 (end > b->bm_words) ||
681 (number <= 0))
682 dev_err(DEV, "offset=%lu number=%lu bm_words=%lu\n",
683 (unsigned long) offset,
684 (unsigned long) number,
685 (unsigned long) b->bm_words);
686 else {
687 while (offset < end) {
688 do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset;
689 p_addr = bm_map_paddr(b, offset);
690 bm = p_addr + MLPP(offset);
691 offset += do_now;
692 while (do_now--)
693 *buffer++ = cpu_to_lel(*bm++);
694 bm_unmap(p_addr);
695 }
696 }
697 spin_unlock_irq(&b->bm_lock);
698}
699
700/* set all bits in the bitmap */
701void drbd_bm_set_all(struct drbd_conf *mdev)
702{
703 struct drbd_bitmap *b = mdev->bitmap;
704 ERR_IF(!b) return;
705 ERR_IF(!b->bm_pages) return;
706
707 spin_lock_irq(&b->bm_lock);
708 bm_memset(b, 0, 0xff, b->bm_words);
709 (void)bm_clear_surplus(b);
710 b->bm_set = b->bm_bits;
711 spin_unlock_irq(&b->bm_lock);
712}
713
714/* clear all bits in the bitmap */
715void drbd_bm_clear_all(struct drbd_conf *mdev)
716{
717 struct drbd_bitmap *b = mdev->bitmap;
718 ERR_IF(!b) return;
719 ERR_IF(!b->bm_pages) return;
720
721 spin_lock_irq(&b->bm_lock);
722 bm_memset(b, 0, 0, b->bm_words);
723 b->bm_set = 0;
724 spin_unlock_irq(&b->bm_lock);
725}
726
727static void bm_async_io_complete(struct bio *bio, int error)
728{
729 struct drbd_bitmap *b = bio->bi_private;
730 int uptodate = bio_flagged(bio, BIO_UPTODATE);
731
732
733 /* strange behavior of some lower level drivers...
734 * fail the request by clearing the uptodate flag,
735 * but do not return any error?!
736 * do we want to WARN() on this? */
737 if (!error && !uptodate)
738 error = -EIO;
739
740 if (error) {
741 /* doh. what now?
742 * for now, set all bits, and flag MD_IO_ERROR */
743 __set_bit(BM_MD_IO_ERROR, &b->bm_flags);
744 }
745 if (atomic_dec_and_test(&b->bm_async_io))
746 wake_up(&b->bm_io_wait);
747
748 bio_put(bio);
749}
750
751static void bm_page_io_async(struct drbd_conf *mdev, struct drbd_bitmap *b, int page_nr, int rw) __must_hold(local)
752{
753 /* we are process context. we always get a bio */
754 struct bio *bio = bio_alloc(GFP_KERNEL, 1);
755 unsigned int len;
756 sector_t on_disk_sector =
757 mdev->ldev->md.md_offset + mdev->ldev->md.bm_offset;
758 on_disk_sector += ((sector_t)page_nr) << (PAGE_SHIFT-9);
759
760 /* this might happen with very small
761 * flexible external meta data device */
762 len = min_t(unsigned int, PAGE_SIZE,
763 (drbd_md_last_sector(mdev->ldev) - on_disk_sector + 1)<<9);
764
765 bio->bi_bdev = mdev->ldev->md_bdev;
766 bio->bi_sector = on_disk_sector;
767 bio_add_page(bio, b->bm_pages[page_nr], len, 0);
768 bio->bi_private = b;
769 bio->bi_end_io = bm_async_io_complete;
770
Andreas Gruenbacher0cf9d272010-12-07 10:43:29 +0100771 if (drbd_insert_fault(mdev, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -0700772 bio->bi_rw |= rw;
773 bio_endio(bio, -EIO);
774 } else {
775 submit_bio(rw, bio);
776 }
777}
778
779# if defined(__LITTLE_ENDIAN)
780 /* nothing to do, on disk == in memory */
781# define bm_cpu_to_lel(x) ((void)0)
782# else
Philipp Reisnerb4ee79d2010-04-01 09:57:40 +0200783static void bm_cpu_to_lel(struct drbd_bitmap *b)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700784{
785 /* need to cpu_to_lel all the pages ...
786 * this may be optimized by using
787 * cpu_to_lel(-1) == -1 and cpu_to_lel(0) == 0;
788 * the following is still not optimal, but better than nothing */
789 unsigned int i;
790 unsigned long *p_addr, *bm;
791 if (b->bm_set == 0) {
792 /* no page at all; avoid swap if all is 0 */
793 i = b->bm_number_of_pages;
794 } else if (b->bm_set == b->bm_bits) {
795 /* only the last page */
796 i = b->bm_number_of_pages - 1;
797 } else {
798 /* all pages */
799 i = 0;
800 }
801 for (; i < b->bm_number_of_pages; i++) {
802 p_addr = kmap_atomic(b->bm_pages[i], KM_USER0);
803 for (bm = p_addr; bm < p_addr + PAGE_SIZE/sizeof(long); bm++)
804 *bm = cpu_to_lel(*bm);
805 kunmap_atomic(p_addr, KM_USER0);
806 }
807}
808# endif
809/* lel_to_cpu == cpu_to_lel */
810# define bm_lel_to_cpu(x) bm_cpu_to_lel(x)
811
812/*
813 * bm_rw: read/write the whole bitmap from/to its on disk location.
814 */
815static int bm_rw(struct drbd_conf *mdev, int rw) __must_hold(local)
816{
817 struct drbd_bitmap *b = mdev->bitmap;
818 /* sector_t sector; */
819 int bm_words, num_pages, i;
820 unsigned long now;
821 char ppb[10];
822 int err = 0;
823
824 WARN_ON(!bm_is_locked(b));
825
826 /* no spinlock here, the drbd_bm_lock should be enough! */
827
828 bm_words = drbd_bm_words(mdev);
829 num_pages = (bm_words*sizeof(long) + PAGE_SIZE-1) >> PAGE_SHIFT;
830
831 /* on disk bitmap is little endian */
832 if (rw == WRITE)
833 bm_cpu_to_lel(b);
834
835 now = jiffies;
836 atomic_set(&b->bm_async_io, num_pages);
837 __clear_bit(BM_MD_IO_ERROR, &b->bm_flags);
838
839 /* let the layers below us try to merge these bios... */
840 for (i = 0; i < num_pages; i++)
841 bm_page_io_async(mdev, b, i, rw);
842
Philipp Reisnerb411b362009-09-25 16:07:19 -0700843 wait_event(b->bm_io_wait, atomic_read(&b->bm_async_io) == 0);
844
845 if (test_bit(BM_MD_IO_ERROR, &b->bm_flags)) {
846 dev_alert(DEV, "we had at least one MD IO ERROR during bitmap IO\n");
Andreas Gruenbacher81e84652010-12-09 15:03:57 +0100847 drbd_chk_io_error(mdev, 1, true);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700848 err = -EIO;
849 }
850
851 now = jiffies;
852 if (rw == WRITE) {
853 /* swap back endianness */
854 bm_lel_to_cpu(b);
855 /* flush bitmap to stable storage */
856 drbd_md_flush(mdev);
857 } else /* rw == READ */ {
858 /* just read, if necessary adjust endianness */
859 b->bm_set = bm_count_bits_swap_endian(b);
860 dev_info(DEV, "recounting of set bits took additional %lu jiffies\n",
861 jiffies - now);
862 }
863 now = b->bm_set;
864
865 dev_info(DEV, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n",
866 ppsize(ppb, now << (BM_BLOCK_SHIFT-10)), now);
867
868 return err;
869}
870
871/**
872 * drbd_bm_read() - Read the whole bitmap from its on disk location.
873 * @mdev: DRBD device.
874 */
875int drbd_bm_read(struct drbd_conf *mdev) __must_hold(local)
876{
877 return bm_rw(mdev, READ);
878}
879
880/**
881 * drbd_bm_write() - Write the whole bitmap to its on disk location.
882 * @mdev: DRBD device.
883 */
884int drbd_bm_write(struct drbd_conf *mdev) __must_hold(local)
885{
886 return bm_rw(mdev, WRITE);
887}
888
889/**
890 * drbd_bm_write_sect: Writes a 512 (MD_SECTOR_SIZE) byte piece of the bitmap
891 * @mdev: DRBD device.
892 * @enr: Extent number in the resync lru (happens to be sector offset)
893 *
894 * The BM_EXT_SIZE is on purpose exactly the amount of the bitmap covered
895 * by a single sector write. Therefore enr == sector offset from the
896 * start of the bitmap.
897 */
898int drbd_bm_write_sect(struct drbd_conf *mdev, unsigned long enr) __must_hold(local)
899{
900 sector_t on_disk_sector = enr + mdev->ldev->md.md_offset
901 + mdev->ldev->md.bm_offset;
902 int bm_words, num_words, offset;
903 int err = 0;
904
905 mutex_lock(&mdev->md_io_mutex);
906 bm_words = drbd_bm_words(mdev);
907 offset = S2W(enr); /* word offset into bitmap */
908 num_words = min(S2W(1), bm_words - offset);
909 if (num_words < S2W(1))
910 memset(page_address(mdev->md_io_page), 0, MD_SECTOR_SIZE);
911 drbd_bm_get_lel(mdev, offset, num_words,
912 page_address(mdev->md_io_page));
913 if (!drbd_md_sync_page_io(mdev, mdev->ldev, on_disk_sector, WRITE)) {
914 int i;
915 err = -EIO;
916 dev_err(DEV, "IO ERROR writing bitmap sector %lu "
917 "(meta-disk sector %llus)\n",
918 enr, (unsigned long long)on_disk_sector);
Andreas Gruenbacher81e84652010-12-09 15:03:57 +0100919 drbd_chk_io_error(mdev, 1, true);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700920 for (i = 0; i < AL_EXT_PER_BM_SECT; i++)
921 drbd_bm_ALe_set_all(mdev, enr*AL_EXT_PER_BM_SECT+i);
922 }
923 mdev->bm_writ_cnt++;
924 mutex_unlock(&mdev->md_io_mutex);
925 return err;
926}
927
928/* NOTE
929 * find_first_bit returns int, we return unsigned long.
930 * should not make much difference anyways, but ...
931 *
932 * this returns a bit number, NOT a sector!
933 */
934#define BPP_MASK ((1UL << (PAGE_SHIFT+3)) - 1)
935static unsigned long __bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo,
936 const int find_zero_bit, const enum km_type km)
937{
938 struct drbd_bitmap *b = mdev->bitmap;
939 unsigned long i = -1UL;
940 unsigned long *p_addr;
941 unsigned long bit_offset; /* bit offset of the mapped page. */
942
943 if (bm_fo > b->bm_bits) {
944 dev_err(DEV, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);
945 } else {
946 while (bm_fo < b->bm_bits) {
947 unsigned long offset;
948 bit_offset = bm_fo & ~BPP_MASK; /* bit offset of the page */
949 offset = bit_offset >> LN2_BPL; /* word offset of the page */
950 p_addr = __bm_map_paddr(b, offset, km);
951
952 if (find_zero_bit)
953 i = find_next_zero_bit(p_addr, PAGE_SIZE*8, bm_fo & BPP_MASK);
954 else
955 i = find_next_bit(p_addr, PAGE_SIZE*8, bm_fo & BPP_MASK);
956
957 __bm_unmap(p_addr, km);
958 if (i < PAGE_SIZE*8) {
959 i = bit_offset + i;
960 if (i >= b->bm_bits)
961 break;
962 goto found;
963 }
964 bm_fo = bit_offset + PAGE_SIZE*8;
965 }
966 i = -1UL;
967 }
968 found:
969 return i;
970}
971
972static unsigned long bm_find_next(struct drbd_conf *mdev,
973 unsigned long bm_fo, const int find_zero_bit)
974{
975 struct drbd_bitmap *b = mdev->bitmap;
976 unsigned long i = -1UL;
977
978 ERR_IF(!b) return i;
979 ERR_IF(!b->bm_pages) return i;
980
981 spin_lock_irq(&b->bm_lock);
982 if (bm_is_locked(b))
983 bm_print_lock_info(mdev);
984
985 i = __bm_find_next(mdev, bm_fo, find_zero_bit, KM_IRQ1);
986
987 spin_unlock_irq(&b->bm_lock);
988 return i;
989}
990
991unsigned long drbd_bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo)
992{
993 return bm_find_next(mdev, bm_fo, 0);
994}
995
996#if 0
997/* not yet needed for anything. */
998unsigned long drbd_bm_find_next_zero(struct drbd_conf *mdev, unsigned long bm_fo)
999{
1000 return bm_find_next(mdev, bm_fo, 1);
1001}
1002#endif
1003
1004/* does not spin_lock_irqsave.
1005 * you must take drbd_bm_lock() first */
1006unsigned long _drbd_bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo)
1007{
1008 /* WARN_ON(!bm_is_locked(mdev)); */
1009 return __bm_find_next(mdev, bm_fo, 0, KM_USER1);
1010}
1011
1012unsigned long _drbd_bm_find_next_zero(struct drbd_conf *mdev, unsigned long bm_fo)
1013{
1014 /* WARN_ON(!bm_is_locked(mdev)); */
1015 return __bm_find_next(mdev, bm_fo, 1, KM_USER1);
1016}
1017
1018/* returns number of bits actually changed.
1019 * for val != 0, we change 0 -> 1, return code positive
1020 * for val == 0, we change 1 -> 0, return code negative
1021 * wants bitnr, not sector.
1022 * expected to be called for only a few bits (e - s about BITS_PER_LONG).
1023 * Must hold bitmap lock already. */
Philipp Reisnerb4ee79d2010-04-01 09:57:40 +02001024static int __bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
Philipp Reisnerb411b362009-09-25 16:07:19 -07001025 unsigned long e, int val, const enum km_type km)
1026{
1027 struct drbd_bitmap *b = mdev->bitmap;
1028 unsigned long *p_addr = NULL;
1029 unsigned long bitnr;
1030 unsigned long last_page_nr = -1UL;
1031 int c = 0;
1032
1033 if (e >= b->bm_bits) {
1034 dev_err(DEV, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n",
1035 s, e, b->bm_bits);
1036 e = b->bm_bits ? b->bm_bits -1 : 0;
1037 }
1038 for (bitnr = s; bitnr <= e; bitnr++) {
1039 unsigned long offset = bitnr>>LN2_BPL;
1040 unsigned long page_nr = offset >> (PAGE_SHIFT - LN2_BPL + 3);
1041 if (page_nr != last_page_nr) {
1042 if (p_addr)
1043 __bm_unmap(p_addr, km);
1044 p_addr = __bm_map_paddr(b, offset, km);
1045 last_page_nr = page_nr;
1046 }
1047 if (val)
1048 c += (0 == __test_and_set_bit(bitnr & BPP_MASK, p_addr));
1049 else
1050 c -= (0 != __test_and_clear_bit(bitnr & BPP_MASK, p_addr));
1051 }
1052 if (p_addr)
1053 __bm_unmap(p_addr, km);
1054 b->bm_set += c;
1055 return c;
1056}
1057
1058/* returns number of bits actually changed.
1059 * for val != 0, we change 0 -> 1, return code positive
1060 * for val == 0, we change 1 -> 0, return code negative
1061 * wants bitnr, not sector */
Philipp Reisnerb4ee79d2010-04-01 09:57:40 +02001062static int bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
Philipp Reisnerb411b362009-09-25 16:07:19 -07001063 const unsigned long e, int val)
1064{
1065 unsigned long flags;
1066 struct drbd_bitmap *b = mdev->bitmap;
1067 int c = 0;
1068
1069 ERR_IF(!b) return 1;
1070 ERR_IF(!b->bm_pages) return 0;
1071
1072 spin_lock_irqsave(&b->bm_lock, flags);
1073 if (bm_is_locked(b))
1074 bm_print_lock_info(mdev);
1075
1076 c = __bm_change_bits_to(mdev, s, e, val, KM_IRQ1);
1077
1078 spin_unlock_irqrestore(&b->bm_lock, flags);
1079 return c;
1080}
1081
1082/* returns number of bits changed 0 -> 1 */
1083int drbd_bm_set_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
1084{
1085 return bm_change_bits_to(mdev, s, e, 1);
1086}
1087
1088/* returns number of bits changed 1 -> 0 */
1089int drbd_bm_clear_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
1090{
1091 return -bm_change_bits_to(mdev, s, e, 0);
1092}
1093
1094/* sets all bits in full words,
1095 * from first_word up to, but not including, last_word */
1096static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b,
1097 int page_nr, int first_word, int last_word)
1098{
1099 int i;
1100 int bits;
1101 unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr], KM_USER0);
1102 for (i = first_word; i < last_word; i++) {
1103 bits = hweight_long(paddr[i]);
1104 paddr[i] = ~0UL;
1105 b->bm_set += BITS_PER_LONG - bits;
1106 }
1107 kunmap_atomic(paddr, KM_USER0);
1108}
1109
1110/* Same thing as drbd_bm_set_bits, but without taking the spin_lock_irqsave.
1111 * You must first drbd_bm_lock().
1112 * Can be called to set the whole bitmap in one go.
1113 * Sets bits from s to e _inclusive_. */
1114void _drbd_bm_set_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
1115{
1116 /* First set_bit from the first bit (s)
1117 * up to the next long boundary (sl),
1118 * then assign full words up to the last long boundary (el),
1119 * then set_bit up to and including the last bit (e).
1120 *
1121 * Do not use memset, because we must account for changes,
1122 * so we need to loop over the words with hweight() anyways.
1123 */
1124 unsigned long sl = ALIGN(s,BITS_PER_LONG);
1125 unsigned long el = (e+1) & ~((unsigned long)BITS_PER_LONG-1);
1126 int first_page;
1127 int last_page;
1128 int page_nr;
1129 int first_word;
1130 int last_word;
1131
1132 if (e - s <= 3*BITS_PER_LONG) {
1133 /* don't bother; el and sl may even be wrong. */
1134 __bm_change_bits_to(mdev, s, e, 1, KM_USER0);
1135 return;
1136 }
1137
1138 /* difference is large enough that we can trust sl and el */
1139
1140 /* bits filling the current long */
1141 if (sl)
1142 __bm_change_bits_to(mdev, s, sl-1, 1, KM_USER0);
1143
1144 first_page = sl >> (3 + PAGE_SHIFT);
1145 last_page = el >> (3 + PAGE_SHIFT);
1146
1147 /* MLPP: modulo longs per page */
1148 /* LWPP: long words per page */
1149 first_word = MLPP(sl >> LN2_BPL);
1150 last_word = LWPP;
1151
1152 /* first and full pages, unless first page == last page */
1153 for (page_nr = first_page; page_nr < last_page; page_nr++) {
1154 bm_set_full_words_within_one_page(mdev->bitmap, page_nr, first_word, last_word);
1155 cond_resched();
1156 first_word = 0;
1157 }
1158
1159 /* last page (respectively only page, for first page == last page) */
1160 last_word = MLPP(el >> LN2_BPL);
1161 bm_set_full_words_within_one_page(mdev->bitmap, last_page, first_word, last_word);
1162
1163 /* possibly trailing bits.
1164 * example: (e & 63) == 63, el will be e+1.
1165 * if that even was the very last bit,
1166 * it would trigger an assert in __bm_change_bits_to()
1167 */
1168 if (el <= e)
1169 __bm_change_bits_to(mdev, el, e, 1, KM_USER0);
1170}
1171
1172/* returns bit state
1173 * wants bitnr, NOT sector.
1174 * inherently racy... area needs to be locked by means of {al,rs}_lru
1175 * 1 ... bit set
1176 * 0 ... bit not set
1177 * -1 ... first out of bounds access, stop testing for bits!
1178 */
1179int drbd_bm_test_bit(struct drbd_conf *mdev, const unsigned long bitnr)
1180{
1181 unsigned long flags;
1182 struct drbd_bitmap *b = mdev->bitmap;
1183 unsigned long *p_addr;
1184 int i;
1185
1186 ERR_IF(!b) return 0;
1187 ERR_IF(!b->bm_pages) return 0;
1188
1189 spin_lock_irqsave(&b->bm_lock, flags);
1190 if (bm_is_locked(b))
1191 bm_print_lock_info(mdev);
1192 if (bitnr < b->bm_bits) {
1193 unsigned long offset = bitnr>>LN2_BPL;
1194 p_addr = bm_map_paddr(b, offset);
1195 i = test_bit(bitnr & BPP_MASK, p_addr) ? 1 : 0;
1196 bm_unmap(p_addr);
1197 } else if (bitnr == b->bm_bits) {
1198 i = -1;
1199 } else { /* (bitnr > b->bm_bits) */
1200 dev_err(DEV, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits);
1201 i = 0;
1202 }
1203
1204 spin_unlock_irqrestore(&b->bm_lock, flags);
1205 return i;
1206}
1207
1208/* returns number of bits set in the range [s, e] */
1209int drbd_bm_count_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e)
1210{
1211 unsigned long flags;
1212 struct drbd_bitmap *b = mdev->bitmap;
1213 unsigned long *p_addr = NULL, page_nr = -1;
1214 unsigned long bitnr;
1215 int c = 0;
1216 size_t w;
1217
1218 /* If this is called without a bitmap, that is a bug. But just to be
1219 * robust in case we screwed up elsewhere, in that case pretend there
1220 * was one dirty bit in the requested area, so we won't try to do a
1221 * local read there (no bitmap probably implies no disk) */
1222 ERR_IF(!b) return 1;
1223 ERR_IF(!b->bm_pages) return 1;
1224
1225 spin_lock_irqsave(&b->bm_lock, flags);
1226 if (bm_is_locked(b))
1227 bm_print_lock_info(mdev);
1228 for (bitnr = s; bitnr <= e; bitnr++) {
1229 w = bitnr >> LN2_BPL;
1230 if (page_nr != w >> (PAGE_SHIFT - LN2_BPL + 3)) {
1231 page_nr = w >> (PAGE_SHIFT - LN2_BPL + 3);
1232 if (p_addr)
1233 bm_unmap(p_addr);
1234 p_addr = bm_map_paddr(b, w);
1235 }
1236 ERR_IF (bitnr >= b->bm_bits) {
1237 dev_err(DEV, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits);
1238 } else {
1239 c += (0 != test_bit(bitnr - (page_nr << (PAGE_SHIFT+3)), p_addr));
1240 }
1241 }
1242 if (p_addr)
1243 bm_unmap(p_addr);
1244 spin_unlock_irqrestore(&b->bm_lock, flags);
1245 return c;
1246}
1247
1248
1249/* inherently racy...
1250 * return value may be already out-of-date when this function returns.
1251 * but the general usage is that this is only use during a cstate when bits are
1252 * only cleared, not set, and typically only care for the case when the return
1253 * value is zero, or we already "locked" this "bitmap extent" by other means.
1254 *
1255 * enr is bm-extent number, since we chose to name one sector (512 bytes)
1256 * worth of the bitmap a "bitmap extent".
1257 *
1258 * TODO
1259 * I think since we use it like a reference count, we should use the real
1260 * reference count of some bitmap extent element from some lru instead...
1261 *
1262 */
1263int drbd_bm_e_weight(struct drbd_conf *mdev, unsigned long enr)
1264{
1265 struct drbd_bitmap *b = mdev->bitmap;
1266 int count, s, e;
1267 unsigned long flags;
1268 unsigned long *p_addr, *bm;
1269
1270 ERR_IF(!b) return 0;
1271 ERR_IF(!b->bm_pages) return 0;
1272
1273 spin_lock_irqsave(&b->bm_lock, flags);
1274 if (bm_is_locked(b))
1275 bm_print_lock_info(mdev);
1276
1277 s = S2W(enr);
1278 e = min((size_t)S2W(enr+1), b->bm_words);
1279 count = 0;
1280 if (s < b->bm_words) {
1281 int n = e-s;
1282 p_addr = bm_map_paddr(b, s);
1283 bm = p_addr + MLPP(s);
1284 while (n--)
1285 count += hweight_long(*bm++);
1286 bm_unmap(p_addr);
1287 } else {
1288 dev_err(DEV, "start offset (%d) too large in drbd_bm_e_weight\n", s);
1289 }
1290 spin_unlock_irqrestore(&b->bm_lock, flags);
1291 return count;
1292}
1293
1294/* set all bits covered by the AL-extent al_enr */
1295unsigned long drbd_bm_ALe_set_all(struct drbd_conf *mdev, unsigned long al_enr)
1296{
1297 struct drbd_bitmap *b = mdev->bitmap;
1298 unsigned long *p_addr, *bm;
1299 unsigned long weight;
1300 int count, s, e, i, do_now;
1301 ERR_IF(!b) return 0;
1302 ERR_IF(!b->bm_pages) return 0;
1303
1304 spin_lock_irq(&b->bm_lock);
1305 if (bm_is_locked(b))
1306 bm_print_lock_info(mdev);
1307 weight = b->bm_set;
1308
1309 s = al_enr * BM_WORDS_PER_AL_EXT;
1310 e = min_t(size_t, s + BM_WORDS_PER_AL_EXT, b->bm_words);
1311 /* assert that s and e are on the same page */
1312 D_ASSERT((e-1) >> (PAGE_SHIFT - LN2_BPL + 3)
1313 == s >> (PAGE_SHIFT - LN2_BPL + 3));
1314 count = 0;
1315 if (s < b->bm_words) {
1316 i = do_now = e-s;
1317 p_addr = bm_map_paddr(b, s);
1318 bm = p_addr + MLPP(s);
1319 while (i--) {
1320 count += hweight_long(*bm);
1321 *bm = -1UL;
1322 bm++;
1323 }
1324 bm_unmap(p_addr);
1325 b->bm_set += do_now*BITS_PER_LONG - count;
1326 if (e == b->bm_words)
1327 b->bm_set -= bm_clear_surplus(b);
1328 } else {
1329 dev_err(DEV, "start offset (%d) too large in drbd_bm_ALe_set_all\n", s);
1330 }
1331 weight = b->bm_set - weight;
1332 spin_unlock_irq(&b->bm_lock);
1333 return weight;
1334}