blob: 9eb60d102de84532e3a662390b1ba2934b744673 [file] [log] [blame]
Kent Overstreetcafe5632013-03-23 16:11:31 -07001/*
2 * Moving/copying garbage collector
3 *
4 * Copyright 2012 Google, Inc.
5 */
6
7#include "bcache.h"
8#include "btree.h"
9#include "debug.h"
10#include "request.h"
11
Kent Overstreetc37511b2013-04-26 15:39:55 -070012#include <trace/events/bcache.h>
13
Kent Overstreetcafe5632013-03-23 16:11:31 -070014struct moving_io {
Kent Overstreet220bb382013-09-10 19:02:45 -070015 struct closure cl;
Kent Overstreetcafe5632013-03-23 16:11:31 -070016 struct keybuf_key *w;
Kent Overstreet220bb382013-09-10 19:02:45 -070017 struct data_insert_op op;
Kent Overstreetcafe5632013-03-23 16:11:31 -070018 struct bbio bio;
19};
20
21static bool moving_pred(struct keybuf *buf, struct bkey *k)
22{
23 struct cache_set *c = container_of(buf, struct cache_set,
24 moving_gc_keys);
25 unsigned i;
26
27 for (i = 0; i < KEY_PTRS(k); i++) {
Kent Overstreetcafe5632013-03-23 16:11:31 -070028 struct bucket *g = PTR_BUCKET(c, k, i);
29
Nicholas Swenson981aa8c2013-11-07 17:53:19 -080030 if (GC_MOVE(g))
Kent Overstreetcafe5632013-03-23 16:11:31 -070031 return true;
32 }
33
34 return false;
35}
36
37/* Moving GC - IO loop */
38
39static void moving_io_destructor(struct closure *cl)
40{
Kent Overstreet220bb382013-09-10 19:02:45 -070041 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070042 kfree(io);
43}
44
45static void write_moving_finish(struct closure *cl)
46{
Kent Overstreet220bb382013-09-10 19:02:45 -070047 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070048 struct bio *bio = &io->bio.bio;
Kent Overstreet8e51e412013-06-06 18:15:57 -070049 struct bio_vec *bv;
50 int i;
Kent Overstreetcafe5632013-03-23 16:11:31 -070051
Kent Overstreet8e51e412013-06-06 18:15:57 -070052 bio_for_each_segment_all(bv, bio, i)
Kent Overstreetcafe5632013-03-23 16:11:31 -070053 __free_page(bv->bv_page);
54
Kent Overstreet220bb382013-09-10 19:02:45 -070055 if (io->op.replace_collision)
Kent Overstreetc37511b2013-04-26 15:39:55 -070056 trace_bcache_gc_copy_collision(&io->w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -070057
Kent Overstreet220bb382013-09-10 19:02:45 -070058 bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);
Kent Overstreetcafe5632013-03-23 16:11:31 -070059
Kent Overstreet220bb382013-09-10 19:02:45 -070060 up(&io->op.c->moving_in_flight);
Kent Overstreetcafe5632013-03-23 16:11:31 -070061
62 closure_return_with_destructor(cl, moving_io_destructor);
63}
64
65static void read_moving_endio(struct bio *bio, int error)
66{
Kent Overstreet6d3d1a92013-12-16 14:12:09 -080067 struct bbio *b = container_of(bio, struct bbio, bio);
Kent Overstreetcafe5632013-03-23 16:11:31 -070068 struct moving_io *io = container_of(bio->bi_private,
Kent Overstreet220bb382013-09-10 19:02:45 -070069 struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -070070
71 if (error)
Kent Overstreet220bb382013-09-10 19:02:45 -070072 io->op.error = error;
Kent Overstreet6d3d1a92013-12-16 14:12:09 -080073 else if (!KEY_DIRTY(&b->key) &&
74 ptr_stale(io->op.c, &b->key, 0)) {
75 io->op.error = -EINTR;
76 }
Kent Overstreetcafe5632013-03-23 16:11:31 -070077
Kent Overstreet220bb382013-09-10 19:02:45 -070078 bch_bbio_endio(io->op.c, bio, error, "reading data to move");
Kent Overstreetcafe5632013-03-23 16:11:31 -070079}
80
81static void moving_init(struct moving_io *io)
82{
83 struct bio *bio = &io->bio.bio;
84
85 bio_init(bio);
86 bio_get(bio);
87 bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
88
Kent Overstreet4f024f32013-10-11 15:44:27 -070089 bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9;
Kent Overstreetcafe5632013-03-23 16:11:31 -070090 bio->bi_max_vecs = DIV_ROUND_UP(KEY_SIZE(&io->w->key),
91 PAGE_SECTORS);
Kent Overstreet220bb382013-09-10 19:02:45 -070092 bio->bi_private = &io->cl;
Kent Overstreetcafe5632013-03-23 16:11:31 -070093 bio->bi_io_vec = bio->bi_inline_vecs;
Kent Overstreet169ef1c2013-03-28 12:50:55 -060094 bch_bio_map(bio, NULL);
Kent Overstreetcafe5632013-03-23 16:11:31 -070095}
96
97static void write_moving(struct closure *cl)
98{
Kent Overstreet220bb382013-09-10 19:02:45 -070099 struct moving_io *io = container_of(cl, struct moving_io, cl);
100 struct data_insert_op *op = &io->op;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700101
Kent Overstreet220bb382013-09-10 19:02:45 -0700102 if (!op->error) {
Kent Overstreetcafe5632013-03-23 16:11:31 -0700103 moving_init(io);
104
Kent Overstreet4f024f32013-10-11 15:44:27 -0700105 io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key);
Kent Overstreet220bb382013-09-10 19:02:45 -0700106 op->write_prio = 1;
107 op->bio = &io->bio.bio;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700108
Kent Overstreet220bb382013-09-10 19:02:45 -0700109 op->writeback = KEY_DIRTY(&io->w->key);
110 op->csum = KEY_CSUM(&io->w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700111
Kent Overstreet220bb382013-09-10 19:02:45 -0700112 bkey_copy(&op->replace_key, &io->w->key);
113 op->replace = true;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700114
Kent Overstreet220bb382013-09-10 19:02:45 -0700115 closure_call(&op->cl, bch_data_insert, NULL, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700116 }
117
Kent Overstreet72a44512013-10-24 17:19:26 -0700118 continue_at(cl, write_moving_finish, system_wq);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700119}
120
121static void read_moving_submit(struct closure *cl)
122{
Kent Overstreet220bb382013-09-10 19:02:45 -0700123 struct moving_io *io = container_of(cl, struct moving_io, cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700124 struct bio *bio = &io->bio.bio;
125
Kent Overstreet220bb382013-09-10 19:02:45 -0700126 bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700127
Kent Overstreet72a44512013-10-24 17:19:26 -0700128 continue_at(cl, write_moving, system_wq);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700129}
130
Kent Overstreet72a44512013-10-24 17:19:26 -0700131static void read_moving(struct cache_set *c)
Kent Overstreetcafe5632013-03-23 16:11:31 -0700132{
Kent Overstreetcafe5632013-03-23 16:11:31 -0700133 struct keybuf_key *w;
134 struct moving_io *io;
135 struct bio *bio;
Kent Overstreet72a44512013-10-24 17:19:26 -0700136 struct closure cl;
137
138 closure_init_stack(&cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700139
140 /* XXX: if we error, background writeback could stall indefinitely */
141
142 while (!test_bit(CACHE_SET_STOPPING, &c->flags)) {
Kent Overstreet72c27062013-06-05 06:24:39 -0700143 w = bch_keybuf_next_rescan(c, &c->moving_gc_keys,
144 &MAX_KEY, moving_pred);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700145 if (!w)
146 break;
147
Kent Overstreet6d3d1a92013-12-16 14:12:09 -0800148 if (ptr_stale(c, &w->key, 0)) {
149 bch_keybuf_del(&c->moving_gc_keys, w);
150 continue;
151 }
152
Kent Overstreetcafe5632013-03-23 16:11:31 -0700153 io = kzalloc(sizeof(struct moving_io) + sizeof(struct bio_vec)
154 * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
155 GFP_KERNEL);
156 if (!io)
157 goto err;
158
159 w->private = io;
160 io->w = w;
Kent Overstreet220bb382013-09-10 19:02:45 -0700161 io->op.inode = KEY_INODE(&w->key);
162 io->op.c = c;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700163
164 moving_init(io);
165 bio = &io->bio.bio;
166
167 bio->bi_rw = READ;
168 bio->bi_end_io = read_moving_endio;
169
Kent Overstreet8e51e412013-06-06 18:15:57 -0700170 if (bio_alloc_pages(bio, GFP_KERNEL))
Kent Overstreetcafe5632013-03-23 16:11:31 -0700171 goto err;
172
Kent Overstreetc37511b2013-04-26 15:39:55 -0700173 trace_bcache_gc_copy(&w->key);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700174
Kent Overstreet72a44512013-10-24 17:19:26 -0700175 down(&c->moving_in_flight);
Kent Overstreet220bb382013-09-10 19:02:45 -0700176 closure_call(&io->cl, read_moving_submit, NULL, &cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700177 }
178
179 if (0) {
180err: if (!IS_ERR_OR_NULL(w->private))
181 kfree(w->private);
182
183 bch_keybuf_del(&c->moving_gc_keys, w);
184 }
185
Kent Overstreet72a44512013-10-24 17:19:26 -0700186 closure_sync(&cl);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700187}
188
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700189static bool bucket_cmp(struct bucket *l, struct bucket *r)
190{
191 return GC_SECTORS_USED(l) < GC_SECTORS_USED(r);
192}
193
194static unsigned bucket_heap_top(struct cache *ca)
195{
Nicholas Swensonbee63f42013-10-31 19:25:18 -0700196 struct bucket *b;
197 return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0;
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700198}
199
Kent Overstreet72a44512013-10-24 17:19:26 -0700200void bch_moving_gc(struct cache_set *c)
Kent Overstreetcafe5632013-03-23 16:11:31 -0700201{
Kent Overstreetcafe5632013-03-23 16:11:31 -0700202 struct cache *ca;
203 struct bucket *b;
204 unsigned i;
205
Kent Overstreetcafe5632013-03-23 16:11:31 -0700206 if (!c->copy_gc_enabled)
Kent Overstreet72a44512013-10-24 17:19:26 -0700207 return;
Kent Overstreetcafe5632013-03-23 16:11:31 -0700208
209 mutex_lock(&c->bucket_lock);
210
211 for_each_cache(ca, c, i) {
212 unsigned sectors_to_move = 0;
213 unsigned reserve_sectors = ca->sb.bucket_size *
Kent Overstreet78365412013-12-17 01:29:34 -0800214 fifo_used(&ca->free[RESERVE_MOVINGGC]);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700215
216 ca->heap.used = 0;
217
218 for_each_bucket(b, ca) {
219 if (!GC_SECTORS_USED(b))
220 continue;
221
222 if (!heap_full(&ca->heap)) {
223 sectors_to_move += GC_SECTORS_USED(b);
224 heap_add(&ca->heap, b, bucket_cmp);
225 } else if (bucket_cmp(b, heap_peek(&ca->heap))) {
Kent Overstreetb1a67b02013-03-25 11:46:44 -0700226 sectors_to_move -= bucket_heap_top(ca);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700227 sectors_to_move += GC_SECTORS_USED(b);
228
229 ca->heap.data[0] = b;
230 heap_sift(&ca->heap, 0, bucket_cmp);
231 }
232 }
233
234 while (sectors_to_move > reserve_sectors) {
235 heap_pop(&ca->heap, b, bucket_cmp);
236 sectors_to_move -= GC_SECTORS_USED(b);
237 }
238
Nicholas Swenson981aa8c2013-11-07 17:53:19 -0800239 while (heap_pop(&ca->heap, b, bucket_cmp))
240 SET_GC_MOVE(b, 1);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700241 }
242
243 mutex_unlock(&c->bucket_lock);
244
245 c->moving_gc_keys.last_scanned = ZERO_KEY;
246
Kent Overstreet72a44512013-10-24 17:19:26 -0700247 read_moving(c);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700248}
249
250void bch_moving_init_cache_set(struct cache_set *c)
251{
Kent Overstreet72c27062013-06-05 06:24:39 -0700252 bch_keybuf_init(&c->moving_gc_keys);
Kent Overstreet72a44512013-10-24 17:19:26 -0700253 sema_init(&c->moving_in_flight, 64);
Kent Overstreetcafe5632013-03-23 16:11:31 -0700254}