blob: cd7490311e518b9139db7d95818111685d52f0fa [file] [log] [blame]
Kent Overstreetcafe5632013-03-23 16:11:31 -07001/*
2 * Moving/copying garbage collector
3 *
4 * Copyright 2012 Google, Inc.
5 */
6
7#include "bcache.h"
8#include "btree.h"
9#include "debug.h"
10#include "request.h"
11
Kent Overstreetc37511b2013-04-26 15:39:55 -070012#include <trace/events/bcache.h>
13
Kent Overstreetcafe5632013-03-23 16:11:31 -070014struct moving_io {
Kent Overstreet220bb382013-09-10 19:02:45 -070015 struct closure cl;
Kent Overstreetcafe5632013-03-23 16:11:31 -070016 struct keybuf_key *w;
Kent Overstreet220bb382013-09-10 19:02:45 -070017 struct data_insert_op op;
Kent Overstreetcafe5632013-03-23 16:11:31 -070018 struct bbio bio;
19};
20
21static bool moving_pred(struct keybuf *buf, struct bkey *k)
22{
23 struct cache_set *c = container_of(buf, struct cache_set,
24 moving_gc_keys);
25 unsigned i;
26
Kent Overstreet10d9dcf2014-02-17 15:48:36 -080027 for (i = 0; i < KEY_PTRS(k); i++)
28 if (ptr_available(c, k, i) &&
29 GC_MOVE(PTR_BUCKET(c, k, i)))
Kent Overstreetcafe5632013-03-23 16:11:31 -070030 return true;
Kent Overstreetcafe5632013-03-23 16:11:31 -070031
32 return false;
33}
34
35/* Moving GC - IO loop */
36
37static void moving_io_destructor(struct closure *cl)
38{
Kent Overstreet220bb382013-09-10 19:02:45 -070039 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070040 kfree(io);
41}
42
43static void write_moving_finish(struct closure *cl)
44{
Kent Overstreet220bb382013-09-10 19:02:45 -070045 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070046 struct bio *bio = &io->bio.bio;
Kent Overstreet8e51e412013-06-06 18:15:57 -070047 struct bio_vec *bv;
48 int i;
Kent Overstreetcafe5632013-03-23 16:11:31 -070049
Kent Overstreet8e51e412013-06-06 18:15:57 -070050 bio_for_each_segment_all(bv, bio, i)
Kent Overstreetcafe5632013-03-23 16:11:31 -070051 __free_page(bv->bv_page);
52
Kent Overstreet220bb382013-09-10 19:02:45 -070053 if (io->op.replace_collision)
Kent Overstreetc37511b2013-04-26 15:39:55 -070054 trace_bcache_gc_copy_collision(&io->w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -070055
Kent Overstreet220bb382013-09-10 19:02:45 -070056 bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);
Kent Overstreetcafe5632013-03-23 16:11:31 -070057
Kent Overstreet220bb382013-09-10 19:02:45 -070058 up(&io->op.c->moving_in_flight);
Kent Overstreetcafe5632013-03-23 16:11:31 -070059
60 closure_return_with_destructor(cl, moving_io_destructor);
61}
62
63static void read_moving_endio(struct bio *bio, int error)
64{
Kent Overstreet6d3d1a92013-12-16 14:12:09 -080065 struct bbio *b = container_of(bio, struct bbio, bio);
Kent Overstreetcafe5632013-03-23 16:11:31 -070066 struct moving_io *io = container_of(bio->bi_private,
Kent Overstreet220bb382013-09-10 19:02:45 -070067 struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070068
69 if (error)
Kent Overstreet220bb382013-09-10 19:02:45 -070070 io->op.error = error;
Kent Overstreet6d3d1a92013-12-16 14:12:09 -080071 else if (!KEY_DIRTY(&b->key) &&
72 ptr_stale(io->op.c, &b->key, 0)) {
73 io->op.error = -EINTR;
74 }
Kent Overstreetcafe5632013-03-23 16:11:31 -070075
Kent Overstreet220bb382013-09-10 19:02:45 -070076 bch_bbio_endio(io->op.c, bio, error, "reading data to move");
Kent Overstreetcafe5632013-03-23 16:11:31 -070077}
78
79static void moving_init(struct moving_io *io)
80{
81 struct bio *bio = &io->bio.bio;
82
83 bio_init(bio);
84 bio_get(bio);
85 bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
86
Kent Overstreet4f024f32013-10-11 15:44:27 -070087 bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9;
Kent Overstreetcafe5632013-03-23 16:11:31 -070088 bio->bi_max_vecs = DIV_ROUND_UP(KEY_SIZE(&io->w->key),
89 PAGE_SECTORS);
Kent Overstreet220bb382013-09-10 19:02:45 -070090 bio->bi_private = &io->cl;
Kent Overstreetcafe5632013-03-23 16:11:31 -070091 bio->bi_io_vec = bio->bi_inline_vecs;
Kent Overstreet169ef1c2013-03-28 12:50:55 -060092 bch_bio_map(bio, NULL);
Kent Overstreetcafe5632013-03-23 16:11:31 -070093}
94
95static void write_moving(struct closure *cl)
96{
Kent Overstreet220bb382013-09-10 19:02:45 -070097 struct moving_io *io = container_of(cl, struct moving_io, cl);
98 struct data_insert_op *op = &io->op;
Kent Overstreetcafe5632013-03-23 16:11:31 -070099
Kent Overstreet220bb382013-09-10 19:02:45 -0700100 if (!op->error) {
Kent Overstreetcafe5632013-03-23 16:11:31 -0700101 moving_init(io);
102
Kent Overstreet4f024f32013-10-11 15:44:27 -0700103 io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key);
Kent Overstreet220bb382013-09-10 19:02:45 -0700104 op->write_prio = 1;
105 op->bio = &io->bio.bio;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700106
Kent Overstreet220bb382013-09-10 19:02:45 -0700107 op->writeback = KEY_DIRTY(&io->w->key);
108 op->csum = KEY_CSUM(&io->w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700109
Kent Overstreet220bb382013-09-10 19:02:45 -0700110 bkey_copy(&op->replace_key, &io->w->key);
111 op->replace = true;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700112
Kent Overstreet220bb382013-09-10 19:02:45 -0700113 closure_call(&op->cl, bch_data_insert, NULL, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700114 }
115
Nicholas Swensonda415a02014-01-09 16:03:04 -0800116 continue_at(cl, write_moving_finish, op->wq);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700117}
118
119static void read_moving_submit(struct closure *cl)
120{
Kent Overstreet220bb382013-09-10 19:02:45 -0700121 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700122 struct bio *bio = &io->bio.bio;
123
Kent Overstreet220bb382013-09-10 19:02:45 -0700124 bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700125
Nicholas Swensonda415a02014-01-09 16:03:04 -0800126 continue_at(cl, write_moving, io->op.wq);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700127}
128
Kent Overstreet72a44512013-10-24 17:19:26 -0700129static void read_moving(struct cache_set *c)
Kent Overstreetcafe5632013-03-23 16:11:31 -0700130{
Kent Overstreetcafe5632013-03-23 16:11:31 -0700131 struct keybuf_key *w;
132 struct moving_io *io;
133 struct bio *bio;
Kent Overstreet72a44512013-10-24 17:19:26 -0700134 struct closure cl;
135
136 closure_init_stack(&cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700137
138 /* XXX: if we error, background writeback could stall indefinitely */
139
140 while (!test_bit(CACHE_SET_STOPPING, &c->flags)) {
Kent Overstreet72c27062013-06-05 06:24:39 -0700141 w = bch_keybuf_next_rescan(c, &c->moving_gc_keys,
142 &MAX_KEY, moving_pred);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700143 if (!w)
144 break;
145
Kent Overstreet6d3d1a92013-12-16 14:12:09 -0800146 if (ptr_stale(c, &w->key, 0)) {
147 bch_keybuf_del(&c->moving_gc_keys, w);
148 continue;
149 }
150
Kent Overstreetcafe5632013-03-23 16:11:31 -0700151 io = kzalloc(sizeof(struct moving_io) + sizeof(struct bio_vec)
152 * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
153 GFP_KERNEL);
154 if (!io)
155 goto err;
156
157 w->private = io;
158 io->w = w;
Kent Overstreet220bb382013-09-10 19:02:45 -0700159 io->op.inode = KEY_INODE(&w->key);
160 io->op.c = c;
Nicholas Swensonda415a02014-01-09 16:03:04 -0800161 io->op.wq = c->moving_gc_wq;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700162
163 moving_init(io);
164 bio = &io->bio.bio;
165
166 bio->bi_rw = READ;
167 bio->bi_end_io = read_moving_endio;
168
Kent Overstreet8e51e412013-06-06 18:15:57 -0700169 if (bio_alloc_pages(bio, GFP_KERNEL))
Kent Overstreetcafe5632013-03-23 16:11:31 -0700170 goto err;
171
Kent Overstreetc37511b2013-04-26 15:39:55 -0700172 trace_bcache_gc_copy(&w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700173
Kent Overstreet72a44512013-10-24 17:19:26 -0700174 down(&c->moving_in_flight);
Kent Overstreet220bb382013-09-10 19:02:45 -0700175 closure_call(&io->cl, read_moving_submit, NULL, &cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700176 }
177
178 if (0) {
179err: if (!IS_ERR_OR_NULL(w->private))
180 kfree(w->private);
181
182 bch_keybuf_del(&c->moving_gc_keys, w);
183 }
184
Kent Overstreet72a44512013-10-24 17:19:26 -0700185 closure_sync(&cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700186}
187
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700188static bool bucket_cmp(struct bucket *l, struct bucket *r)
189{
190 return GC_SECTORS_USED(l) < GC_SECTORS_USED(r);
191}
192
193static unsigned bucket_heap_top(struct cache *ca)
194{
Nicholas Swensonbee63f42013-10-31 19:25:18 -0700195 struct bucket *b;
196 return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0;
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700197}
198
Kent Overstreet72a44512013-10-24 17:19:26 -0700199void bch_moving_gc(struct cache_set *c)
Kent Overstreetcafe5632013-03-23 16:11:31 -0700200{
Kent Overstreetcafe5632013-03-23 16:11:31 -0700201 struct cache *ca;
202 struct bucket *b;
203 unsigned i;
204
Kent Overstreetcafe5632013-03-23 16:11:31 -0700205 if (!c->copy_gc_enabled)
Kent Overstreet72a44512013-10-24 17:19:26 -0700206 return;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700207
208 mutex_lock(&c->bucket_lock);
209
210 for_each_cache(ca, c, i) {
211 unsigned sectors_to_move = 0;
212 unsigned reserve_sectors = ca->sb.bucket_size *
Kent Overstreet78365412013-12-17 01:29:34 -0800213 fifo_used(&ca->free[RESERVE_MOVINGGC]);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700214
215 ca->heap.used = 0;
216
217 for_each_bucket(b, ca) {
Nicholas Swenson3f6ef382014-01-23 15:21:02 -0800218 if (GC_MARK(b) == GC_MARK_METADATA ||
219 !GC_SECTORS_USED(b) ||
220 GC_SECTORS_USED(b) == ca->sb.bucket_size ||
221 atomic_read(&b->pin))
Kent Overstreetcafe5632013-03-23 16:11:31 -0700222 continue;
223
224 if (!heap_full(&ca->heap)) {
225 sectors_to_move += GC_SECTORS_USED(b);
226 heap_add(&ca->heap, b, bucket_cmp);
227 } else if (bucket_cmp(b, heap_peek(&ca->heap))) {
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700228 sectors_to_move -= bucket_heap_top(ca);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700229 sectors_to_move += GC_SECTORS_USED(b);
230
231 ca->heap.data[0] = b;
232 heap_sift(&ca->heap, 0, bucket_cmp);
233 }
234 }
235
236 while (sectors_to_move > reserve_sectors) {
237 heap_pop(&ca->heap, b, bucket_cmp);
238 sectors_to_move -= GC_SECTORS_USED(b);
239 }
240
Nicholas Swenson981aa8c2013-11-07 17:53:19 -0800241 while (heap_pop(&ca->heap, b, bucket_cmp))
242 SET_GC_MOVE(b, 1);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700243 }
244
245 mutex_unlock(&c->bucket_lock);
246
247 c->moving_gc_keys.last_scanned = ZERO_KEY;
248
Kent Overstreet72a44512013-10-24 17:19:26 -0700249 read_moving(c);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700250}
251
252void bch_moving_init_cache_set(struct cache_set *c)
253{
Kent Overstreet72c27062013-06-05 06:24:39 -0700254 bch_keybuf_init(&c->moving_gc_keys);
Kent Overstreet72a44512013-10-24 17:19:26 -0700255 sema_init(&c->moving_in_flight, 64);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700256}