blob: d26a3fa6368849ce95d59a7847c8853c11861026 [file] [log] [blame]
Philipp Reisnerb411b362009-09-25 16:07:19 -07001/*
2 drbd_actlog.c
3
4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
5
6 Copyright (C) 2003-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 2003-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2003-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
9
10 drbd is free software; you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 2, or (at your option)
13 any later version.
14
15 drbd is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with drbd; see the file COPYING. If not, write to
22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23
24 */
25
26#include <linux/slab.h>
Lars Ellenberg7ad651b2011-02-21 13:21:03 +010027#include <linux/crc32c.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070028#include <linux/drbd.h>
Lars Ellenberg7ad651b2011-02-21 13:21:03 +010029#include <linux/drbd_limits.h>
30#include <linux/dynamic_debug.h>
Philipp Reisnerb411b362009-09-25 16:07:19 -070031#include "drbd_int.h"
Philipp Reisnerb411b362009-09-25 16:07:19 -070032
Lars Ellenberg85f103d2011-03-31 12:06:48 +020033
34enum al_transaction_types {
35 AL_TR_UPDATE = 0,
36 AL_TR_INITIALIZED = 0xffff
37};
Lars Ellenberg7ad651b2011-02-21 13:21:03 +010038/* all fields on disc in big endian */
39struct __packed al_transaction_on_disk {
40 /* don't we all like magic */
41 __be32 magic;
42
43 /* to identify the most recent transaction block
44 * in the on disk ring buffer */
45 __be32 tr_number;
46
47 /* checksum on the full 4k block, with this field set to 0. */
48 __be32 crc32c;
49
50 /* type of transaction, special transaction types like:
Lars Ellenberg85f103d2011-03-31 12:06:48 +020051 * purge-all, set-all-idle, set-all-active, ... to-be-defined
52 * see also enum al_transaction_types */
Lars Ellenberg7ad651b2011-02-21 13:21:03 +010053 __be16 transaction_type;
54
55 /* we currently allow only a few thousand extents,
56 * so 16bit will be enough for the slot number. */
57
58 /* how many updates in this transaction */
59 __be16 n_updates;
60
61 /* maximum slot number, "al-extents" in drbd.conf speak.
62 * Having this in each transaction should make reconfiguration
63 * of that parameter easier. */
64 __be16 context_size;
65
66 /* slot number the context starts with */
67 __be16 context_start_slot_nr;
68
69 /* Some reserved bytes. Expected usage is a 64bit counter of
70 * sectors-written since device creation, and other data generation tag
71 * supporting usage */
72 __be32 __reserved[4];
73
74 /* --- 36 byte used --- */
75
76 /* Reserve space for up to AL_UPDATES_PER_TRANSACTION changes
77 * in one transaction, then use the remaining byte in the 4k block for
78 * context information. "Flexible" number of updates per transaction
79 * does not help, as we have to account for the case when all update
80 * slots are used anyways, so it would only complicate code without
81 * additional benefit.
82 */
83 __be16 update_slot_nr[AL_UPDATES_PER_TRANSACTION];
84
85 /* but the extent number is 32bit, which at an extent size of 4 MiB
86 * allows to cover device sizes of up to 2**54 Byte (16 PiB) */
87 __be32 update_extent_nr[AL_UPDATES_PER_TRANSACTION];
88
89 /* --- 420 bytes used (36 + 64*6) --- */
90
91 /* 4096 - 420 = 3676 = 919 * 4 */
92 __be32 context[AL_CONTEXT_PER_TRANSACTION];
Philipp Reisnerb411b362009-09-25 16:07:19 -070093};
94
Lars Ellenberge37d2432014-04-01 23:53:30 +020095void *drbd_md_get_buffer(struct drbd_device *device, const char *intent)
Philipp Reisnercdfda632011-07-05 15:38:59 +020096{
97 int r;
98
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +020099 wait_event(device->misc_wait,
Lars Ellenberge37d2432014-04-01 23:53:30 +0200100 (r = atomic_cmpxchg(&device->md_io.in_use, 0, 1)) == 0 ||
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200101 device->state.disk <= D_FAILED);
Philipp Reisnercdfda632011-07-05 15:38:59 +0200102
Lars Ellenberge37d2432014-04-01 23:53:30 +0200103 if (r)
104 return NULL;
105
106 device->md_io.current_use = intent;
107 device->md_io.start_jif = jiffies;
108 device->md_io.submit_jif = device->md_io.start_jif - 1;
109 return page_address(device->md_io.page);
Philipp Reisnercdfda632011-07-05 15:38:59 +0200110}
111
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200112void drbd_md_put_buffer(struct drbd_device *device)
Philipp Reisnercdfda632011-07-05 15:38:59 +0200113{
Lars Ellenberge37d2432014-04-01 23:53:30 +0200114 if (atomic_dec_and_test(&device->md_io.in_use))
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200115 wake_up(&device->misc_wait);
Philipp Reisnercdfda632011-07-05 15:38:59 +0200116}
117
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200118void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev,
Philipp Reisner32db80f2012-02-22 11:51:57 +0100119 unsigned int *done)
Philipp Reisnercdfda632011-07-05 15:38:59 +0200120{
Philipp Reisner32db80f2012-02-22 11:51:57 +0100121 long dt;
122
123 rcu_read_lock();
124 dt = rcu_dereference(bdev->disk_conf)->disk_timeout;
125 rcu_read_unlock();
126 dt = dt * HZ / 10;
127 if (dt == 0)
128 dt = MAX_SCHEDULE_TIMEOUT;
129
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200130 dt = wait_event_timeout(device->misc_wait,
131 *done || test_bit(FORCE_DETACH, &device->flags), dt);
Lars Ellenberge34b6772012-09-27 15:07:11 +0200132 if (dt == 0) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200133 drbd_err(device, "meta-data IO operation timed out\n");
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200134 drbd_chk_io_error(device, 1, DRBD_FORCE_DETACH);
Lars Ellenberge34b6772012-09-27 15:07:11 +0200135 }
Philipp Reisnercdfda632011-07-05 15:38:59 +0200136}
137
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200138static int _drbd_md_sync_page_io(struct drbd_device *device,
Philipp Reisnerb411b362009-09-25 16:07:19 -0700139 struct drbd_backing_dev *bdev,
Lars Ellenberg193cb002014-04-02 00:24:18 +0200140 sector_t sector, int rw)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700141{
142 struct bio *bio;
Lars Ellenberg193cb002014-04-02 00:24:18 +0200143 /* we do all our meta data IO in aligned 4k blocks. */
144 const int size = 4096;
Andreas Gruenbacherac29f402010-12-13 02:20:47 +0100145 int err;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700146
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200147 device->md_io.done = 0;
148 device->md_io.error = -ENODEV;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700149
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200150 if ((rw & WRITE) && !test_bit(MD_NO_FUA, &device->flags))
Lars Ellenberg86e1e982011-06-28 13:22:48 +0200151 rw |= REQ_FUA | REQ_FLUSH;
Lars Ellenberg2ed912e2014-02-11 08:56:53 +0100152 rw |= REQ_SYNC | REQ_NOIDLE;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700153
Lars Ellenbergda4a75d2011-02-23 17:02:01 +0100154 bio = bio_alloc_drbd(GFP_NOIO);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700155 bio->bi_bdev = bdev->md_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700156 bio->bi_iter.bi_sector = sector;
Andreas Gruenbacherac29f402010-12-13 02:20:47 +0100157 err = -EIO;
Lars Ellenberg193cb002014-04-02 00:24:18 +0200158 if (bio_add_page(bio, device->md_io.page, size, 0) != size)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700159 goto out;
Lars Ellenberge37d2432014-04-01 23:53:30 +0200160 bio->bi_private = device;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700161 bio->bi_end_io = drbd_md_io_complete;
162 bio->bi_rw = rw;
163
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200164 if (!(rw & WRITE) && device->state.disk == D_DISKLESS && device->ldev == NULL)
Lars Ellenbergc04ccaa2013-03-19 18:16:47 +0100165 /* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */
166 ;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200167 else if (!get_ldev_if_state(device, D_ATTACHING)) {
Lars Ellenbergc04ccaa2013-03-19 18:16:47 +0100168 /* Corresponding put_ldev in drbd_md_io_complete() */
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200169 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in _drbd_md_sync_page_io()\n");
Philipp Reisnercdfda632011-07-05 15:38:59 +0200170 err = -ENODEV;
171 goto out;
172 }
173
174 bio_get(bio); /* one bio_put() is in the completion handler */
Lars Ellenberge37d2432014-04-01 23:53:30 +0200175 atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */
176 device->md_io.submit_jif = jiffies;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200177 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD))
Philipp Reisnerb411b362009-09-25 16:07:19 -0700178 bio_endio(bio, -EIO);
179 else
180 submit_bio(rw, bio);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200181 wait_until_done_or_force_detached(device, bdev, &device->md_io.done);
Andreas Gruenbacherac29f402010-12-13 02:20:47 +0100182 if (bio_flagged(bio, BIO_UPTODATE))
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200183 err = device->md_io.error;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700184
Philipp Reisnerb411b362009-09-25 16:07:19 -0700185 out:
186 bio_put(bio);
Andreas Gruenbacherac29f402010-12-13 02:20:47 +0100187 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700188}
189
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200190int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev,
Philipp Reisnerb411b362009-09-25 16:07:19 -0700191 sector_t sector, int rw)
192{
Andreas Gruenbacher3fbf4d22010-12-13 02:25:41 +0100193 int err;
Lars Ellenberge37d2432014-04-01 23:53:30 +0200194 D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700195
196 BUG_ON(!bdev->md_bdev);
197
Lars Ellenberge4d7d6f2014-04-28 18:43:28 +0200198 dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n",
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100199 current->comm, current->pid, __func__,
Lars Ellenbergc04ccaa2013-03-19 18:16:47 +0100200 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ",
201 (void*)_RET_IP_ );
Philipp Reisnerb411b362009-09-25 16:07:19 -0700202
203 if (sector < drbd_md_first_sector(bdev) ||
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100204 sector + 7 > drbd_md_last_sector(bdev))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200205 drbd_alert(device, "%s [%d]:%s(,%llus,%s) out of range md access!\n",
Philipp Reisnerb411b362009-09-25 16:07:19 -0700206 current->comm, current->pid, __func__,
207 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ");
208
Lars Ellenberg193cb002014-04-02 00:24:18 +0200209 err = _drbd_md_sync_page_io(device, bdev, sector, rw);
Andreas Gruenbacher3fbf4d22010-12-13 02:25:41 +0100210 if (err) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200211 drbd_err(device, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n",
Andreas Gruenbacher935be262011-08-19 13:47:31 +0200212 (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", err);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700213 }
Andreas Gruenbacher3fbf4d22010-12-13 02:25:41 +0100214 return err;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700215}
216
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200217static struct bm_extent *find_active_resync_extent(struct drbd_device *device, unsigned int enr)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700218{
Philipp Reisnerb411b362009-09-25 16:07:19 -0700219 struct lc_element *tmp;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200220 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700221 if (unlikely(tmp != NULL)) {
222 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100223 if (test_bit(BME_NO_WRITES, &bm_ext->flags))
224 return bm_ext;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700225 }
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100226 return NULL;
227}
228
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200229static struct lc_element *_al_get(struct drbd_device *device, unsigned int enr, bool nonblock)
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100230{
231 struct lc_element *al_ext;
232 struct bm_extent *bm_ext;
233 int wake;
234
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200235 spin_lock_irq(&device->al_lock);
236 bm_ext = find_active_resync_extent(device, enr);
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100237 if (bm_ext) {
238 wake = !test_and_set_bit(BME_PRIORITY, &bm_ext->flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200239 spin_unlock_irq(&device->al_lock);
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100240 if (wake)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200241 wake_up(&device->al_wait);
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100242 return NULL;
243 }
244 if (nonblock)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200245 al_ext = lc_try_get(device->act_log, enr);
Lars Ellenberg6c3c43552013-03-19 18:16:53 +0100246 else
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200247 al_ext = lc_get(device->act_log, enr);
248 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700249 return al_ext;
250}
251
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200252bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i)
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100253{
254 /* for bios crossing activity log extent boundaries,
255 * we may need to activate two extents in one go */
256 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
257 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100258
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200259 D_ASSERT(device, (unsigned)(last - first) <= 1);
260 D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100261
262 /* FIXME figure out a fast path for bios crossing AL extent boundaries */
263 if (first != last)
264 return false;
265
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200266 return _al_get(device, first, true);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100267}
268
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200269bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700270{
Lars Ellenberg77265472011-03-31 16:00:51 +0200271 /* for bios crossing activity log extent boundaries,
272 * we may need to activate two extents in one go */
Lars Ellenberge15766e2011-04-01 10:38:30 +0200273 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
Lars Ellenberg81a35372012-07-30 09:00:54 +0200274 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
Lars Ellenberge15766e2011-04-01 10:38:30 +0200275 unsigned enr;
Lars Ellenbergebfd5d82013-03-19 18:16:49 +0100276 bool need_transaction = false;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700277
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200278 D_ASSERT(device, first <= last);
279 D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700280
Lars Ellenbergebfd5d82013-03-19 18:16:49 +0100281 for (enr = first; enr <= last; enr++) {
282 struct lc_element *al_ext;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200283 wait_event(device->al_wait,
284 (al_ext = _al_get(device, enr, false)) != NULL);
Lars Ellenbergebfd5d82013-03-19 18:16:49 +0100285 if (al_ext->lc_number != enr)
286 need_transaction = true;
287 }
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100288 return need_transaction;
289}
Lars Ellenbergebfd5d82013-03-19 18:16:49 +0100290
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100291static int al_write_transaction(struct drbd_device *device);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100292
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100293void drbd_al_begin_io_commit(struct drbd_device *device)
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100294{
295 bool locked = false;
296
Lars Ellenberg7dc1d672011-05-03 16:49:20 +0200297 /* Serialize multiple transactions.
298 * This uses test_and_set_bit, memory barrier is implicit.
299 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200300 wait_event(device->al_wait,
301 device->act_log->pending_changes == 0 ||
302 (locked = lc_try_lock_for_transaction(device->act_log)));
Lars Ellenberg7dc1d672011-05-03 16:49:20 +0200303
304 if (locked) {
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100305 /* Double check: it may have been committed by someone else,
306 * while we have been waiting for the lock. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200307 if (device->act_log->pending_changes) {
Philipp Reisner9a51ab12012-02-20 21:53:28 +0100308 bool write_al_updates;
309
310 rcu_read_lock();
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200311 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
Philipp Reisner9a51ab12012-02-20 21:53:28 +0100312 rcu_read_unlock();
313
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100314 if (write_al_updates)
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100315 al_write_transaction(device);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200316 spin_lock_irq(&device->al_lock);
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100317 /* FIXME
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200318 if (err)
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100319 we need an "lc_cancel" here;
320 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200321 lc_committed(device->act_log);
322 spin_unlock_irq(&device->al_lock);
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100323 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200324 lc_unlock(device->act_log);
325 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700326 }
327}
328
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100329/*
330 * @delegate: delegate activity log I/O to the worker thread
331 */
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100332void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i)
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100333{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200334 if (drbd_al_begin_io_prepare(device, i))
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100335 drbd_al_begin_io_commit(device);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100336}
337
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200338int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i)
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100339{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200340 struct lru_cache *al = device->act_log;
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100341 /* for bios crossing activity log extent boundaries,
342 * we may need to activate two extents in one go */
343 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
344 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
345 unsigned nr_al_extents;
346 unsigned available_update_slots;
347 unsigned enr;
348
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200349 D_ASSERT(device, first <= last);
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100350
351 nr_al_extents = 1 + last - first; /* worst case: all touched extends are cold. */
352 available_update_slots = min(al->nr_elements - al->used,
353 al->max_pending_changes - al->pending_changes);
354
355 /* We want all necessary updates for a given request within the same transaction
356 * We could first check how many updates are *actually* needed,
357 * and use that instead of the worst-case nr_al_extents */
Lars Ellenbergf5b90b62014-05-07 22:41:28 +0200358 if (available_update_slots < nr_al_extents) {
359 /* Too many activity log extents are currently "hot".
360 *
361 * If we have accumulated pending changes already,
362 * we made progress.
363 *
364 * If we cannot get even a single pending change through,
365 * stop the fast path until we made some progress,
366 * or requests to "cold" extents could be starved. */
367 if (!al->pending_changes)
368 __set_bit(__LC_STARVING, &device->act_log->flags);
369 return -ENOBUFS;
370 }
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100371
372 /* Is resync active in this area? */
373 for (enr = first; enr <= last; enr++) {
374 struct lc_element *tmp;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200375 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100376 if (unlikely(tmp != NULL)) {
377 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);
378 if (test_bit(BME_NO_WRITES, &bm_ext->flags)) {
Lars Ellenberg0b6ef412013-03-27 14:08:49 +0100379 if (!test_and_set_bit(BME_PRIORITY, &bm_ext->flags))
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100380 return -EBUSY;
381 return -EWOULDBLOCK;
382 }
383 }
384 }
385
386 /* Checkout the refcounts.
387 * Given that we checked for available elements and update slots above,
388 * this has to be successful. */
389 for (enr = first; enr <= last; enr++) {
390 struct lc_element *al_ext;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200391 al_ext = lc_get_cumulative(device->act_log, enr);
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100392 if (!al_ext)
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200393 drbd_info(device, "LOGIC BUG for enr=%u\n", enr);
Lars Ellenberg08a1dda2013-03-19 18:16:56 +0100394 }
395 return 0;
396}
397
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200398void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700399{
Lars Ellenberge15766e2011-04-01 10:38:30 +0200400 /* for bios crossing activity log extent boundaries,
401 * we may need to activate two extents in one go */
402 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
Lars Ellenberg81a35372012-07-30 09:00:54 +0200403 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
Lars Ellenberge15766e2011-04-01 10:38:30 +0200404 unsigned enr;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700405 struct lc_element *extent;
406 unsigned long flags;
407
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200408 D_ASSERT(device, first <= last);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200409 spin_lock_irqsave(&device->al_lock, flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700410
Lars Ellenberge15766e2011-04-01 10:38:30 +0200411 for (enr = first; enr <= last; enr++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200412 extent = lc_find(device->act_log, enr);
Lars Ellenberge15766e2011-04-01 10:38:30 +0200413 if (!extent) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200414 drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr);
Lars Ellenberge15766e2011-04-01 10:38:30 +0200415 continue;
416 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200417 lc_put(device->act_log, extent);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700418 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200419 spin_unlock_irqrestore(&device->al_lock, flags);
420 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700421}
422
Lars Ellenberg19f843a2010-12-15 08:59:11 +0100423#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
424/* Currently BM_BLOCK_SHIFT, BM_EXT_SHIFT and AL_EXTENT_SHIFT
425 * are still coupled, or assume too much about their relation.
426 * Code below will not work if this is violated.
427 * Will be cleaned up with some followup patch.
428 */
429# error FIXME
430#endif
431
432static unsigned int al_extent_to_bm_page(unsigned int al_enr)
433{
434 return al_enr >>
435 /* bit to page */
436 ((PAGE_SHIFT + 3) -
437 /* al extent number to bit */
438 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
439}
440
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200441static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
Lars Ellenbergae8bf312013-03-19 18:16:43 +0100442{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200443 const unsigned int stripes = device->ldev->md.al_stripes;
444 const unsigned int stripe_size_4kB = device->ldev->md.al_stripe_size_4k;
Lars Ellenbergae8bf312013-03-19 18:16:43 +0100445
446 /* transaction number, modulo on-disk ring buffer wrap around */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200447 unsigned int t = device->al_tr_number % (device->ldev->md.al_size_4k);
Lars Ellenbergae8bf312013-03-19 18:16:43 +0100448
449 /* ... to aligned 4k on disk block */
450 t = ((t % stripes) * stripe_size_4kB) + t/stripes;
451
452 /* ... to 512 byte sector in activity log */
453 t *= 8;
454
455 /* ... plus offset to the on disk position */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200456 return device->ldev->md.md_offset + device->ldev->md.al_offset + t;
Lars Ellenbergae8bf312013-03-19 18:16:43 +0100457}
458
Lars Ellenberg4dd726f2014-02-11 11:15:36 +0100459int al_write_transaction(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700460{
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100461 struct al_transaction_on_disk *buffer;
462 struct lc_element *e;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700463 sector_t sector;
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100464 int i, mx;
465 unsigned extent_nr;
466 unsigned crc = 0;
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200467 int err = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700468
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200469 if (!get_ldev(device)) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200470 drbd_err(device, "disk is %s, cannot start al transaction\n",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200471 drbd_disk_str(device->state.disk));
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200472 return -EIO;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700473 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700474
Lars Ellenberg6719fb02010-10-18 23:04:07 +0200475 /* The bitmap write may have failed, causing a state change. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200476 if (device->state.disk < D_INCONSISTENT) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200477 drbd_err(device,
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100478 "disk is %s, cannot write al transaction\n",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200479 drbd_disk_str(device->state.disk));
480 put_ldev(device);
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200481 return -EIO;
Lars Ellenberg6719fb02010-10-18 23:04:07 +0200482 }
483
Lars Ellenberge37d2432014-04-01 23:53:30 +0200484 /* protects md_io_buffer, al_tr_cycle, ... */
485 buffer = drbd_md_get_buffer(device, __func__);
Philipp Reisnercdfda632011-07-05 15:38:59 +0200486 if (!buffer) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200487 drbd_err(device, "disk failed while waiting for md_io buffer\n");
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200488 put_ldev(device);
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200489 return -ENODEV;
Philipp Reisnercdfda632011-07-05 15:38:59 +0200490 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700491
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100492 memset(buffer, 0, sizeof(*buffer));
493 buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200494 buffer->tr_number = cpu_to_be32(device->al_tr_number);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700495
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100496 i = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700497
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100498 /* Even though no one can start to change this list
499 * once we set the LC_LOCKED -- from drbd_al_begin_io(),
500 * lc_try_lock_for_transaction() --, someone may still
501 * be in the process of changing it. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200502 spin_lock_irq(&device->al_lock);
503 list_for_each_entry(e, &device->act_log->to_be_changed, list) {
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100504 if (i == AL_UPDATES_PER_TRANSACTION) {
505 i++;
506 break;
507 }
508 buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
509 buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
510 if (e->lc_number != LC_FREE)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200511 drbd_bm_mark_for_writeout(device,
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100512 al_extent_to_bm_page(e->lc_number));
513 i++;
514 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200515 spin_unlock_irq(&device->al_lock);
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100516 BUG_ON(i > AL_UPDATES_PER_TRANSACTION);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700517
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100518 buffer->n_updates = cpu_to_be16(i);
519 for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) {
520 buffer->update_slot_nr[i] = cpu_to_be16(-1);
521 buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE);
522 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700523
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200524 buffer->context_size = cpu_to_be16(device->act_log->nr_elements);
525 buffer->context_start_slot_nr = cpu_to_be16(device->al_tr_cycle);
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100526
527 mx = min_t(int, AL_CONTEXT_PER_TRANSACTION,
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200528 device->act_log->nr_elements - device->al_tr_cycle);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700529 for (i = 0; i < mx; i++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200530 unsigned idx = device->al_tr_cycle + i;
531 extent_nr = lc_element_by_index(device->act_log, idx)->lc_number;
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100532 buffer->context[i] = cpu_to_be32(extent_nr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700533 }
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100534 for (; i < AL_CONTEXT_PER_TRANSACTION; i++)
535 buffer->context[i] = cpu_to_be32(LC_FREE);
536
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200537 device->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION;
538 if (device->al_tr_cycle >= device->act_log->nr_elements)
539 device->al_tr_cycle = 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700540
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200541 sector = al_tr_number_to_on_disk_sector(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700542
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100543 crc = crc32c(0, buffer, 4096);
544 buffer->crc32c = cpu_to_be32(crc);
545
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200546 if (drbd_bm_write_hinted(device))
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200547 err = -EIO;
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100548 else {
549 bool write_al_updates;
550 rcu_read_lock();
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200551 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100552 rcu_read_unlock();
553 if (write_al_updates) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200554 if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100555 err = -EIO;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200556 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100557 } else {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200558 device->al_tr_number++;
559 device->al_writ_cnt++;
Lars Ellenbergb5bc8e02013-03-19 18:16:52 +0100560 }
561 }
Lars Ellenberg7ad651b2011-02-21 13:21:03 +0100562 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700563
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200564 drbd_md_put_buffer(device);
565 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700566
Philipp Reisner1b7ab152011-07-15 17:19:02 +0200567 return err;
568}
569
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200570static int _try_lc_del(struct drbd_device *device, struct lc_element *al_ext)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700571{
572 int rv;
573
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200574 spin_lock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700575 rv = (al_ext->refcnt == 0);
576 if (likely(rv))
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200577 lc_del(device->act_log, al_ext);
578 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700579
580 return rv;
581}
582
583/**
584 * drbd_al_shrink() - Removes all active extents form the activity log
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200585 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -0700586 *
587 * Removes all active extents form the activity log, waiting until
588 * the reference count of each entry dropped to 0 first, of course.
589 *
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200590 * You need to lock device->act_log with lc_try_lock() / lc_unlock()
Philipp Reisnerb411b362009-09-25 16:07:19 -0700591 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200592void drbd_al_shrink(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700593{
594 struct lc_element *al_ext;
595 int i;
596
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200597 D_ASSERT(device, test_bit(__LC_LOCKED, &device->act_log->flags));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700598
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200599 for (i = 0; i < device->act_log->nr_elements; i++) {
600 al_ext = lc_element_by_index(device->act_log, i);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700601 if (al_ext->lc_number == LC_FREE)
602 continue;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200603 wait_event(device->al_wait, _try_lc_del(device, al_ext));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700604 }
605
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200606 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700607}
608
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200609int drbd_initialize_al(struct drbd_device *device, void *buffer)
Philipp Reisnerd752b262013-06-25 16:50:08 +0200610{
611 struct al_transaction_on_disk *al = buffer;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200612 struct drbd_md *md = &device->ldev->md;
Philipp Reisnerd752b262013-06-25 16:50:08 +0200613 sector_t al_base = md->md_offset + md->al_offset;
614 int al_size_4k = md->al_stripes * md->al_stripe_size_4k;
615 int i;
616
617 memset(al, 0, 4096);
618 al->magic = cpu_to_be32(DRBD_AL_MAGIC);
619 al->transaction_type = cpu_to_be16(AL_TR_INITIALIZED);
620 al->crc32c = cpu_to_be32(crc32c(0, al, 4096));
621
622 for (i = 0; i < al_size_4k; i++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200623 int err = drbd_md_sync_page_io(device, device->ldev, al_base + i * 8, WRITE);
Philipp Reisnerd752b262013-06-25 16:50:08 +0200624 if (err)
625 return err;
626 }
627 return 0;
628}
629
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100630static const char *drbd_change_sync_fname[] = {
631 [RECORD_RS_FAILED] = "drbd_rs_failed_io",
632 [SET_IN_SYNC] = "drbd_set_in_sync",
633 [SET_OUT_OF_SYNC] = "drbd_set_out_of_sync"
634};
635
Philipp Reisnerb411b362009-09-25 16:07:19 -0700636/* ATTENTION. The AL's extents are 4MB each, while the extents in the
637 * resync LRU-cache are 16MB each.
638 * The caller of this function has to hold an get_ldev() reference.
639 *
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100640 * Adjusts the caching members ->rs_left (success) or ->rs_failed (!success),
641 * potentially pulling in (and recounting the corresponding bits)
642 * this resync extent into the resync extent lru cache.
643 *
644 * Returns whether all bits have been cleared for this resync extent,
645 * precisely: (rs_left <= rs_failed)
646 *
Philipp Reisnerb411b362009-09-25 16:07:19 -0700647 * TODO will be obsoleted once we have a caching lru of the on disk bitmap
648 */
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100649static bool update_rs_extent(struct drbd_device *device,
650 unsigned int enr, int count,
651 enum update_sync_bits_mode mode)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700652{
653 struct lc_element *e;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700654
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +0200655 D_ASSERT(device, atomic_read(&device->local_cnt));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700656
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100657 /* When setting out-of-sync bits,
658 * we don't need it cached (lc_find).
659 * But if it is present in the cache,
660 * we should update the cached bit count.
661 * Otherwise, that extent should be in the resync extent lru cache
662 * already -- or we want to pull it in if necessary -- (lc_get),
663 * then update and check rs_left and rs_failed. */
664 if (mode == SET_OUT_OF_SYNC)
665 e = lc_find(device->resync, enr);
666 else
667 e = lc_get(device->resync, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700668 if (e) {
669 struct bm_extent *ext = lc_entry(e, struct bm_extent, lce);
670 if (ext->lce.lc_number == enr) {
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100671 if (mode == SET_IN_SYNC)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700672 ext->rs_left -= count;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100673 else if (mode == SET_OUT_OF_SYNC)
674 ext->rs_left += count;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700675 else
676 ext->rs_failed += count;
677 if (ext->rs_left < ext->rs_failed) {
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100678 drbd_warn(device, "BAD! enr=%u rs_left=%d "
Philipp Reisner975b2972011-11-17 10:11:47 +0100679 "rs_failed=%d count=%d cstate=%s\n",
Philipp Reisnerb411b362009-09-25 16:07:19 -0700680 ext->lce.lc_number, ext->rs_left,
Philipp Reisner975b2972011-11-17 10:11:47 +0100681 ext->rs_failed, count,
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200682 drbd_conn_str(device->state.conn));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700683
Philipp Reisner975b2972011-11-17 10:11:47 +0100684 /* We don't expect to be able to clear more bits
685 * than have been set when we originally counted
686 * the set bits to cache that value in ext->rs_left.
687 * Whatever the reason (disconnect during resync,
688 * delayed local completion of an application write),
689 * try to fix it up by recounting here. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200690 ext->rs_left = drbd_bm_e_weight(device, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700691 }
692 } else {
693 /* Normally this element should be in the cache,
694 * since drbd_rs_begin_io() pulled it already in.
695 *
696 * But maybe an application write finished, and we set
697 * something outside the resync lru_cache in sync.
698 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200699 int rs_left = drbd_bm_e_weight(device, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700700 if (ext->flags != 0) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200701 drbd_warn(device, "changing resync lce: %d[%u;%02lx]"
Philipp Reisnerb411b362009-09-25 16:07:19 -0700702 " -> %d[%u;00]\n",
703 ext->lce.lc_number, ext->rs_left,
704 ext->flags, enr, rs_left);
705 ext->flags = 0;
706 }
707 if (ext->rs_failed) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200708 drbd_warn(device, "Kicking resync_lru element enr=%u "
Philipp Reisnerb411b362009-09-25 16:07:19 -0700709 "out with rs_failed=%d\n",
710 ext->lce.lc_number, ext->rs_failed);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700711 }
712 ext->rs_left = rs_left;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100713 ext->rs_failed = (mode == RECORD_RS_FAILED) ? count : 0;
Lars Ellenberg46a15bc2011-02-21 13:21:01 +0100714 /* we don't keep a persistent log of the resync lru,
715 * we can commit any change right away. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200716 lc_committed(device->resync);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700717 }
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100718 if (mode != SET_OUT_OF_SYNC)
719 lc_put(device->resync, &ext->lce);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700720 /* no race, we are within the al_lock! */
721
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100722 if (ext->rs_left <= ext->rs_failed) {
Philipp Reisnerb411b362009-09-25 16:07:19 -0700723 ext->rs_failed = 0;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100724 return true;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700725 }
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100726 } else if (mode != SET_OUT_OF_SYNC) {
727 /* be quiet if lc_find() did not find it. */
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200728 drbd_err(device, "lc_get() failed! locked=%d/%d flags=%lu\n",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200729 device->resync_locked,
730 device->resync->nr_elements,
731 device->resync->flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700732 }
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100733 return false;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700734}
735
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200736void drbd_advance_rs_marks(struct drbd_device *device, unsigned long still_to_go)
Lars Ellenbergc6ea14d2010-11-05 09:23:37 +0100737{
738 unsigned long now = jiffies;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200739 unsigned long last = device->rs_mark_time[device->rs_last_mark];
740 int next = (device->rs_last_mark + 1) % DRBD_SYNC_MARKS;
Lars Ellenbergc6ea14d2010-11-05 09:23:37 +0100741 if (time_after_eq(now, last + DRBD_SYNC_MARK_STEP)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200742 if (device->rs_mark_left[device->rs_last_mark] != still_to_go &&
743 device->state.conn != C_PAUSED_SYNC_T &&
744 device->state.conn != C_PAUSED_SYNC_S) {
745 device->rs_mark_time[next] = now;
746 device->rs_mark_left[next] = still_to_go;
747 device->rs_last_mark = next;
Lars Ellenbergc6ea14d2010-11-05 09:23:37 +0100748 }
749 }
750}
751
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100752/* It is called lazy update, so don't do write-out too often. */
753static bool lazy_bitmap_update_due(struct drbd_device *device)
754{
755 return time_after(jiffies, device->rs_last_bcast + 2*HZ);
756}
757
758static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done)
759{
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100760 if (rs_done)
761 set_bit(RS_DONE, &device->flags);
762 /* and also set RS_PROGRESS below */
763 else if (!lazy_bitmap_update_due(device))
764 return;
765
Lars Ellenberge334f552014-02-11 09:30:49 +0100766 drbd_device_post_work(device, RS_PROGRESS);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100767}
768
769static int update_sync_bits(struct drbd_device *device,
770 unsigned long sbnr, unsigned long ebnr,
771 enum update_sync_bits_mode mode)
772{
773 /*
774 * We keep a count of set bits per resync-extent in the ->rs_left
775 * caching member, so we need to loop and work within the resync extent
776 * alignment. Typically this loop will execute exactly once.
777 */
778 unsigned long flags;
779 unsigned long count = 0;
780 unsigned int cleared = 0;
781 while (sbnr <= ebnr) {
782 /* set temporary boundary bit number to last bit number within
783 * the resync extent of the current start bit number,
784 * but cap at provided end bit number */
785 unsigned long tbnr = min(ebnr, sbnr | BM_BLOCKS_PER_BM_EXT_MASK);
786 unsigned long c;
787
788 if (mode == RECORD_RS_FAILED)
789 /* Only called from drbd_rs_failed_io(), bits
790 * supposedly still set. Recount, maybe some
791 * of the bits have been successfully cleared
792 * by application IO meanwhile.
793 */
794 c = drbd_bm_count_bits(device, sbnr, tbnr);
795 else if (mode == SET_IN_SYNC)
796 c = drbd_bm_clear_bits(device, sbnr, tbnr);
797 else /* if (mode == SET_OUT_OF_SYNC) */
798 c = drbd_bm_set_bits(device, sbnr, tbnr);
799
800 if (c) {
801 spin_lock_irqsave(&device->al_lock, flags);
802 cleared += update_rs_extent(device, BM_BIT_TO_EXT(sbnr), c, mode);
803 spin_unlock_irqrestore(&device->al_lock, flags);
804 count += c;
805 }
806 sbnr = tbnr + 1;
807 }
808 if (count) {
809 if (mode == SET_IN_SYNC) {
810 unsigned long still_to_go = drbd_bm_total_weight(device);
811 bool rs_is_done = (still_to_go <= device->rs_failed);
812 drbd_advance_rs_marks(device, still_to_go);
813 if (cleared || rs_is_done)
814 maybe_schedule_on_disk_bitmap_update(device, rs_is_done);
815 } else if (mode == RECORD_RS_FAILED)
816 device->rs_failed += count;
817 wake_up(&device->al_wait);
818 }
819 return count;
820}
821
Philipp Reisnerb411b362009-09-25 16:07:19 -0700822/* clear the bit corresponding to the piece of storage in question:
823 * size byte of data starting from sector. Only clear a bits of the affected
824 * one ore more _aligned_ BM_BLOCK_SIZE blocks.
825 *
826 * called by worker on C_SYNC_TARGET and receiver on SyncSource.
827 *
828 */
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100829int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size,
830 enum update_sync_bits_mode mode,
831 const char *file, const unsigned int line)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700832{
833 /* Is called from worker and receiver context _only_ */
834 unsigned long sbnr, ebnr, lbnr;
835 unsigned long count = 0;
836 sector_t esector, nr_sectors;
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100837
838 /* This would be an empty REQ_FLUSH, be silent. */
839 if ((mode == SET_OUT_OF_SYNC) && size == 0)
840 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700841
Lars Ellenberga0fb3c42014-04-28 18:43:23 +0200842 if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_DISCARD_SIZE) {
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100843 drbd_err(device, "%s: sector=%llus size=%d nonsense!\n",
844 drbd_change_sync_fname[mode],
Philipp Reisnerb411b362009-09-25 16:07:19 -0700845 (unsigned long long)sector, size);
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100846 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700847 }
Philipp Reisner518a4d52012-10-19 14:21:22 +0200848
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200849 if (!get_ldev(device))
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100850 return 0; /* no disk, no metadata, no bitmap to manipulate bits in */
Philipp Reisner518a4d52012-10-19 14:21:22 +0200851
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200852 nr_sectors = drbd_get_capacity(device->this_bdev);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700853 esector = sector + (size >> 9) - 1;
854
Andreas Gruenbacher841ce242010-12-15 19:31:20 +0100855 if (!expect(sector < nr_sectors))
Philipp Reisner518a4d52012-10-19 14:21:22 +0200856 goto out;
Andreas Gruenbacher841ce242010-12-15 19:31:20 +0100857 if (!expect(esector < nr_sectors))
858 esector = nr_sectors - 1;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700859
860 lbnr = BM_SECT_TO_BIT(nr_sectors-1);
861
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100862 if (mode == SET_IN_SYNC) {
863 /* Round up start sector, round down end sector. We make sure
864 * we only clear full, aligned, BM_BLOCK_SIZE blocks. */
865 if (unlikely(esector < BM_SECT_PER_BIT-1))
866 goto out;
867 if (unlikely(esector == (nr_sectors-1)))
868 ebnr = lbnr;
869 else
870 ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1));
871 sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1);
872 } else {
873 /* We set it out of sync, or record resync failure.
874 * Should not round anything here. */
875 sbnr = BM_SECT_TO_BIT(sector);
876 ebnr = BM_SECT_TO_BIT(esector);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700877 }
878
Lars Ellenberg5ab7d2c2014-01-27 15:58:22 +0100879 count = update_sync_bits(device, sbnr, ebnr, mode);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700880out:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200881 put_ldev(device);
Philipp Reisner73a01a12010-10-27 14:33:00 +0200882 return count;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700883}
884
885static
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200886struct bm_extent *_bme_get(struct drbd_device *device, unsigned int enr)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700887{
888 struct lc_element *e;
889 struct bm_extent *bm_ext;
890 int wakeup = 0;
891 unsigned long rs_flags;
892
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200893 spin_lock_irq(&device->al_lock);
894 if (device->resync_locked > device->resync->nr_elements/2) {
895 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700896 return NULL;
897 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200898 e = lc_get(device->resync, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700899 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
900 if (bm_ext) {
901 if (bm_ext->lce.lc_number != enr) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200902 bm_ext->rs_left = drbd_bm_e_weight(device, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700903 bm_ext->rs_failed = 0;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200904 lc_committed(device->resync);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700905 wakeup = 1;
906 }
907 if (bm_ext->lce.refcnt == 1)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200908 device->resync_locked++;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700909 set_bit(BME_NO_WRITES, &bm_ext->flags);
910 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200911 rs_flags = device->resync->flags;
912 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700913 if (wakeup)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200914 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700915
916 if (!bm_ext) {
917 if (rs_flags & LC_STARVING)
Andreas Gruenbacherd0180172011-07-03 17:53:52 +0200918 drbd_warn(device, "Have to wait for element"
Philipp Reisnerb411b362009-09-25 16:07:19 -0700919 " (resync LRU too small?)\n");
Lars Ellenberg46a15bc2011-02-21 13:21:01 +0100920 BUG_ON(rs_flags & LC_LOCKED);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700921 }
922
923 return bm_ext;
924}
925
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200926static int _is_in_al(struct drbd_device *device, unsigned int enr)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700927{
Lars Ellenberg46a15bc2011-02-21 13:21:01 +0100928 int rv;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700929
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200930 spin_lock_irq(&device->al_lock);
931 rv = lc_is_used(device->act_log, enr);
932 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700933
Philipp Reisnerb411b362009-09-25 16:07:19 -0700934 return rv;
935}
936
937/**
938 * drbd_rs_begin_io() - Gets an extent in the resync LRU cache and sets it to BME_LOCKED
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200939 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -0700940 * @sector: The sector number.
941 *
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200942 * This functions sleeps on al_wait. Returns 0 on success, -EINTR if interrupted.
Philipp Reisnerb411b362009-09-25 16:07:19 -0700943 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200944int drbd_rs_begin_io(struct drbd_device *device, sector_t sector)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700945{
946 unsigned int enr = BM_SECT_TO_EXT(sector);
947 struct bm_extent *bm_ext;
948 int i, sig;
Lars Ellenberge8299872014-04-28 18:43:19 +0200949 bool sa;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700950
Philipp Reisnerf91ab622010-11-09 13:59:41 +0100951retry:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200952 sig = wait_event_interruptible(device->al_wait,
953 (bm_ext = _bme_get(device, enr)));
Philipp Reisnerb411b362009-09-25 16:07:19 -0700954 if (sig)
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200955 return -EINTR;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700956
957 if (test_bit(BME_LOCKED, &bm_ext->flags))
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200958 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700959
Lars Ellenberge8299872014-04-28 18:43:19 +0200960 /* step aside only while we are above c-min-rate; unless disabled. */
961 sa = drbd_rs_c_min_rate_throttle(device);
962
Philipp Reisnerb411b362009-09-25 16:07:19 -0700963 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200964 sig = wait_event_interruptible(device->al_wait,
965 !_is_in_al(device, enr * AL_EXT_PER_BM_SECT + i) ||
Lars Ellenberge8299872014-04-28 18:43:19 +0200966 (sa && test_bit(BME_PRIORITY, &bm_ext->flags)));
Philipp Reisnerf91ab622010-11-09 13:59:41 +0100967
Lars Ellenberge8299872014-04-28 18:43:19 +0200968 if (sig || (sa && test_bit(BME_PRIORITY, &bm_ext->flags))) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200969 spin_lock_irq(&device->al_lock);
970 if (lc_put(device->resync, &bm_ext->lce) == 0) {
Philipp Reisnerf91ab622010-11-09 13:59:41 +0100971 bm_ext->flags = 0; /* clears BME_NO_WRITES and eventually BME_PRIORITY */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200972 device->resync_locked--;
973 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -0700974 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200975 spin_unlock_irq(&device->al_lock);
Philipp Reisnerf91ab622010-11-09 13:59:41 +0100976 if (sig)
977 return -EINTR;
978 if (schedule_timeout_interruptible(HZ/10))
979 return -EINTR;
Philipp Reisnerf91ab622010-11-09 13:59:41 +0100980 goto retry;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700981 }
982 }
Philipp Reisnerb411b362009-09-25 16:07:19 -0700983 set_bit(BME_LOCKED, &bm_ext->flags);
Lars Ellenberg80a40e42010-08-11 23:28:00 +0200984 return 0;
Philipp Reisnerb411b362009-09-25 16:07:19 -0700985}
986
987/**
988 * drbd_try_rs_begin_io() - Gets an extent in the resync LRU cache, does not sleep
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200989 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -0700990 * @sector: The sector number.
991 *
992 * Gets an extent in the resync LRU cache, sets it to BME_NO_WRITES, then
993 * tries to set it to BME_LOCKED. Returns 0 upon success, and -EAGAIN
994 * if there is still application IO going on in this area.
995 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +0200996int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector)
Philipp Reisnerb411b362009-09-25 16:07:19 -0700997{
998 unsigned int enr = BM_SECT_TO_EXT(sector);
999 const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT;
1000 struct lc_element *e;
1001 struct bm_extent *bm_ext;
1002 int i;
Lars Ellenbergad3fee72013-12-20 11:22:13 +01001003 bool throttle = drbd_rs_should_slow_down(device, sector, true);
1004
1005 /* If we need to throttle, a half-locked (only marked BME_NO_WRITES,
1006 * not yet BME_LOCKED) extent needs to be kicked out explicitly if we
1007 * need to throttle. There is at most one such half-locked extent,
1008 * which is remembered in resync_wenr. */
1009
1010 if (throttle && device->resync_wenr != enr)
1011 return -EAGAIN;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001012
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001013 spin_lock_irq(&device->al_lock);
1014 if (device->resync_wenr != LC_FREE && device->resync_wenr != enr) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001015 /* in case you have very heavy scattered io, it may
1016 * stall the syncer undefined if we give up the ref count
1017 * when we try again and requeue.
1018 *
1019 * if we don't give up the refcount, but the next time
1020 * we are scheduled this extent has been "synced" by new
1021 * application writes, we'd miss the lc_put on the
1022 * extent we keep the refcount on.
1023 * so we remembered which extent we had to try again, and
1024 * if the next requested one is something else, we do
1025 * the lc_put here...
1026 * we also have to wake_up
1027 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001028 e = lc_find(device->resync, device->resync_wenr);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001029 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1030 if (bm_ext) {
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001031 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1032 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
Philipp Reisnerb411b362009-09-25 16:07:19 -07001033 clear_bit(BME_NO_WRITES, &bm_ext->flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001034 device->resync_wenr = LC_FREE;
Lars Ellenbergad3fee72013-12-20 11:22:13 +01001035 if (lc_put(device->resync, &bm_ext->lce) == 0) {
1036 bm_ext->flags = 0;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001037 device->resync_locked--;
Lars Ellenbergad3fee72013-12-20 11:22:13 +01001038 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001039 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001040 } else {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001041 drbd_alert(device, "LOGIC BUG\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -07001042 }
1043 }
1044 /* TRY. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001045 e = lc_try_get(device->resync, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001046 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1047 if (bm_ext) {
1048 if (test_bit(BME_LOCKED, &bm_ext->flags))
1049 goto proceed;
1050 if (!test_and_set_bit(BME_NO_WRITES, &bm_ext->flags)) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001051 device->resync_locked++;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001052 } else {
1053 /* we did set the BME_NO_WRITES,
1054 * but then could not set BME_LOCKED,
1055 * so we tried again.
1056 * drop the extra reference. */
Philipp Reisnerb411b362009-09-25 16:07:19 -07001057 bm_ext->lce.refcnt--;
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001058 D_ASSERT(device, bm_ext->lce.refcnt > 0);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001059 }
1060 goto check_al;
1061 } else {
1062 /* do we rather want to try later? */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001063 if (device->resync_locked > device->resync->nr_elements-3)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001064 goto try_again;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001065 /* Do or do not. There is no try. -- Yoda */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001066 e = lc_get(device->resync, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001067 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1068 if (!bm_ext) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001069 const unsigned long rs_flags = device->resync->flags;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001070 if (rs_flags & LC_STARVING)
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001071 drbd_warn(device, "Have to wait for element"
Philipp Reisnerb411b362009-09-25 16:07:19 -07001072 " (resync LRU too small?)\n");
Lars Ellenberg46a15bc2011-02-21 13:21:01 +01001073 BUG_ON(rs_flags & LC_LOCKED);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001074 goto try_again;
1075 }
1076 if (bm_ext->lce.lc_number != enr) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001077 bm_ext->rs_left = drbd_bm_e_weight(device, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001078 bm_ext->rs_failed = 0;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001079 lc_committed(device->resync);
1080 wake_up(&device->al_wait);
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001081 D_ASSERT(device, test_bit(BME_LOCKED, &bm_ext->flags) == 0);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001082 }
1083 set_bit(BME_NO_WRITES, &bm_ext->flags);
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001084 D_ASSERT(device, bm_ext->lce.refcnt == 1);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001085 device->resync_locked++;
Philipp Reisnerb411b362009-09-25 16:07:19 -07001086 goto check_al;
1087 }
1088check_al:
Philipp Reisnerb411b362009-09-25 16:07:19 -07001089 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001090 if (lc_is_used(device->act_log, al_enr+i))
Philipp Reisnerb411b362009-09-25 16:07:19 -07001091 goto try_again;
1092 }
1093 set_bit(BME_LOCKED, &bm_ext->flags);
1094proceed:
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001095 device->resync_wenr = LC_FREE;
1096 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001097 return 0;
1098
1099try_again:
Lars Ellenbergad3fee72013-12-20 11:22:13 +01001100 if (bm_ext) {
1101 if (throttle) {
1102 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1103 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
1104 clear_bit(BME_NO_WRITES, &bm_ext->flags);
1105 device->resync_wenr = LC_FREE;
1106 if (lc_put(device->resync, &bm_ext->lce) == 0) {
1107 bm_ext->flags = 0;
1108 device->resync_locked--;
1109 }
1110 wake_up(&device->al_wait);
1111 } else
1112 device->resync_wenr = enr;
1113 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001114 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001115 return -EAGAIN;
1116}
1117
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001118void drbd_rs_complete_io(struct drbd_device *device, sector_t sector)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001119{
1120 unsigned int enr = BM_SECT_TO_EXT(sector);
1121 struct lc_element *e;
1122 struct bm_extent *bm_ext;
1123 unsigned long flags;
1124
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001125 spin_lock_irqsave(&device->al_lock, flags);
1126 e = lc_find(device->resync, enr);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001127 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1128 if (!bm_ext) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001129 spin_unlock_irqrestore(&device->al_lock, flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001130 if (__ratelimit(&drbd_ratelimit_state))
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001131 drbd_err(device, "drbd_rs_complete_io() called, but extent not found\n");
Philipp Reisnerb411b362009-09-25 16:07:19 -07001132 return;
1133 }
1134
1135 if (bm_ext->lce.refcnt == 0) {
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001136 spin_unlock_irqrestore(&device->al_lock, flags);
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001137 drbd_err(device, "drbd_rs_complete_io(,%llu [=%u]) called, "
Philipp Reisnerb411b362009-09-25 16:07:19 -07001138 "but refcnt is 0!?\n",
1139 (unsigned long long)sector, enr);
1140 return;
1141 }
1142
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001143 if (lc_put(device->resync, &bm_ext->lce) == 0) {
Philipp Reisnere3555d82010-11-07 15:56:29 +01001144 bm_ext->flags = 0; /* clear BME_LOCKED, BME_NO_WRITES and BME_PRIORITY */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001145 device->resync_locked--;
1146 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001147 }
1148
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001149 spin_unlock_irqrestore(&device->al_lock, flags);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001150}
1151
1152/**
1153 * drbd_rs_cancel_all() - Removes all extents from the resync LRU (even BME_LOCKED)
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001154 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -07001155 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001156void drbd_rs_cancel_all(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001157{
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001158 spin_lock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001159
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001160 if (get_ldev_if_state(device, D_FAILED)) { /* Makes sure ->resync is there. */
1161 lc_reset(device->resync);
1162 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001163 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001164 device->resync_locked = 0;
1165 device->resync_wenr = LC_FREE;
1166 spin_unlock_irq(&device->al_lock);
1167 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001168}
1169
1170/**
1171 * drbd_rs_del_all() - Gracefully remove all extents from the resync LRU
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001172 * @device: DRBD device.
Philipp Reisnerb411b362009-09-25 16:07:19 -07001173 *
1174 * Returns 0 upon success, -EAGAIN if at least one reference count was
1175 * not zero.
1176 */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001177int drbd_rs_del_all(struct drbd_device *device)
Philipp Reisnerb411b362009-09-25 16:07:19 -07001178{
1179 struct lc_element *e;
1180 struct bm_extent *bm_ext;
1181 int i;
1182
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001183 spin_lock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001184
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001185 if (get_ldev_if_state(device, D_FAILED)) {
Philipp Reisnerb411b362009-09-25 16:07:19 -07001186 /* ok, ->resync is there. */
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001187 for (i = 0; i < device->resync->nr_elements; i++) {
1188 e = lc_element_by_index(device->resync, i);
Philipp Reisnerb2b163d2010-04-02 08:40:33 +02001189 bm_ext = lc_entry(e, struct bm_extent, lce);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001190 if (bm_ext->lce.lc_number == LC_FREE)
1191 continue;
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001192 if (bm_ext->lce.lc_number == device->resync_wenr) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001193 drbd_info(device, "dropping %u in drbd_rs_del_all, apparently"
Philipp Reisnerb411b362009-09-25 16:07:19 -07001194 " got 'synced' by application io\n",
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001195 device->resync_wenr);
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001196 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1197 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
Philipp Reisnerb411b362009-09-25 16:07:19 -07001198 clear_bit(BME_NO_WRITES, &bm_ext->flags);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001199 device->resync_wenr = LC_FREE;
1200 lc_put(device->resync, &bm_ext->lce);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001201 }
1202 if (bm_ext->lce.refcnt != 0) {
Andreas Gruenbacherd0180172011-07-03 17:53:52 +02001203 drbd_info(device, "Retrying drbd_rs_del_all() later. "
Philipp Reisnerb411b362009-09-25 16:07:19 -07001204 "refcnt=%d\n", bm_ext->lce.refcnt);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001205 put_ldev(device);
1206 spin_unlock_irq(&device->al_lock);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001207 return -EAGAIN;
1208 }
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001209 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1210 D_ASSERT(device, !test_bit(BME_NO_WRITES, &bm_ext->flags));
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001211 lc_del(device->resync, &bm_ext->lce);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001212 }
Andreas Gruenbacher0b0ba1e2011-06-27 16:23:33 +02001213 D_ASSERT(device, device->resync->used == 0);
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001214 put_ldev(device);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001215 }
Andreas Gruenbacherb30ab792011-07-03 13:26:43 +02001216 spin_unlock_irq(&device->al_lock);
1217 wake_up(&device->al_wait);
Philipp Reisnerb411b362009-09-25 16:07:19 -07001218
1219 return 0;
1220}