Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Main bcache entry point - handle a read or a write request and decide what to |
| 3 | * do with it; the make_request functions are called by the block layer. |
| 4 | * |
| 5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> |
| 6 | * Copyright 2012 Google, Inc. |
| 7 | */ |
| 8 | |
| 9 | #include "bcache.h" |
| 10 | #include "btree.h" |
| 11 | #include "debug.h" |
| 12 | #include "request.h" |
Kent Overstreet | 279afba | 2013-06-05 06:21:07 -0700 | [diff] [blame] | 13 | #include "writeback.h" |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 14 | |
| 15 | #include <linux/cgroup.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/hash.h> |
| 18 | #include <linux/random.h> |
| 19 | #include "blk-cgroup.h" |
| 20 | |
| 21 | #include <trace/events/bcache.h> |
| 22 | |
| 23 | #define CUTOFF_CACHE_ADD 95 |
| 24 | #define CUTOFF_CACHE_READA 90 |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 25 | |
| 26 | struct kmem_cache *bch_search_cache; |
| 27 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 28 | static void bch_data_insert_start(struct closure *); |
| 29 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 30 | /* Cgroup interface */ |
| 31 | |
| 32 | #ifdef CONFIG_CGROUP_BCACHE |
| 33 | static struct bch_cgroup bcache_default_cgroup = { .cache_mode = -1 }; |
| 34 | |
| 35 | static struct bch_cgroup *cgroup_to_bcache(struct cgroup *cgroup) |
| 36 | { |
| 37 | struct cgroup_subsys_state *css; |
| 38 | return cgroup && |
| 39 | (css = cgroup_subsys_state(cgroup, bcache_subsys_id)) |
| 40 | ? container_of(css, struct bch_cgroup, css) |
| 41 | : &bcache_default_cgroup; |
| 42 | } |
| 43 | |
| 44 | struct bch_cgroup *bch_bio_to_cgroup(struct bio *bio) |
| 45 | { |
| 46 | struct cgroup_subsys_state *css = bio->bi_css |
| 47 | ? cgroup_subsys_state(bio->bi_css->cgroup, bcache_subsys_id) |
| 48 | : task_subsys_state(current, bcache_subsys_id); |
| 49 | |
| 50 | return css |
| 51 | ? container_of(css, struct bch_cgroup, css) |
| 52 | : &bcache_default_cgroup; |
| 53 | } |
| 54 | |
| 55 | static ssize_t cache_mode_read(struct cgroup *cgrp, struct cftype *cft, |
| 56 | struct file *file, |
| 57 | char __user *buf, size_t nbytes, loff_t *ppos) |
| 58 | { |
| 59 | char tmp[1024]; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 60 | int len = bch_snprint_string_list(tmp, PAGE_SIZE, bch_cache_modes, |
| 61 | cgroup_to_bcache(cgrp)->cache_mode + 1); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 62 | |
| 63 | if (len < 0) |
| 64 | return len; |
| 65 | |
| 66 | return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); |
| 67 | } |
| 68 | |
| 69 | static int cache_mode_write(struct cgroup *cgrp, struct cftype *cft, |
| 70 | const char *buf) |
| 71 | { |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 72 | int v = bch_read_string_list(buf, bch_cache_modes); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 73 | if (v < 0) |
| 74 | return v; |
| 75 | |
| 76 | cgroup_to_bcache(cgrp)->cache_mode = v - 1; |
| 77 | return 0; |
| 78 | } |
| 79 | |
| 80 | static u64 bch_verify_read(struct cgroup *cgrp, struct cftype *cft) |
| 81 | { |
| 82 | return cgroup_to_bcache(cgrp)->verify; |
| 83 | } |
| 84 | |
| 85 | static int bch_verify_write(struct cgroup *cgrp, struct cftype *cft, u64 val) |
| 86 | { |
| 87 | cgroup_to_bcache(cgrp)->verify = val; |
| 88 | return 0; |
| 89 | } |
| 90 | |
| 91 | static u64 bch_cache_hits_read(struct cgroup *cgrp, struct cftype *cft) |
| 92 | { |
| 93 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 94 | return atomic_read(&bcachecg->stats.cache_hits); |
| 95 | } |
| 96 | |
| 97 | static u64 bch_cache_misses_read(struct cgroup *cgrp, struct cftype *cft) |
| 98 | { |
| 99 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 100 | return atomic_read(&bcachecg->stats.cache_misses); |
| 101 | } |
| 102 | |
| 103 | static u64 bch_cache_bypass_hits_read(struct cgroup *cgrp, |
| 104 | struct cftype *cft) |
| 105 | { |
| 106 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 107 | return atomic_read(&bcachecg->stats.cache_bypass_hits); |
| 108 | } |
| 109 | |
| 110 | static u64 bch_cache_bypass_misses_read(struct cgroup *cgrp, |
| 111 | struct cftype *cft) |
| 112 | { |
| 113 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 114 | return atomic_read(&bcachecg->stats.cache_bypass_misses); |
| 115 | } |
| 116 | |
| 117 | static struct cftype bch_files[] = { |
| 118 | { |
| 119 | .name = "cache_mode", |
| 120 | .read = cache_mode_read, |
| 121 | .write_string = cache_mode_write, |
| 122 | }, |
| 123 | { |
| 124 | .name = "verify", |
| 125 | .read_u64 = bch_verify_read, |
| 126 | .write_u64 = bch_verify_write, |
| 127 | }, |
| 128 | { |
| 129 | .name = "cache_hits", |
| 130 | .read_u64 = bch_cache_hits_read, |
| 131 | }, |
| 132 | { |
| 133 | .name = "cache_misses", |
| 134 | .read_u64 = bch_cache_misses_read, |
| 135 | }, |
| 136 | { |
| 137 | .name = "cache_bypass_hits", |
| 138 | .read_u64 = bch_cache_bypass_hits_read, |
| 139 | }, |
| 140 | { |
| 141 | .name = "cache_bypass_misses", |
| 142 | .read_u64 = bch_cache_bypass_misses_read, |
| 143 | }, |
| 144 | { } /* terminate */ |
| 145 | }; |
| 146 | |
| 147 | static void init_bch_cgroup(struct bch_cgroup *cg) |
| 148 | { |
| 149 | cg->cache_mode = -1; |
| 150 | } |
| 151 | |
| 152 | static struct cgroup_subsys_state *bcachecg_create(struct cgroup *cgroup) |
| 153 | { |
| 154 | struct bch_cgroup *cg; |
| 155 | |
| 156 | cg = kzalloc(sizeof(*cg), GFP_KERNEL); |
| 157 | if (!cg) |
| 158 | return ERR_PTR(-ENOMEM); |
| 159 | init_bch_cgroup(cg); |
| 160 | return &cg->css; |
| 161 | } |
| 162 | |
| 163 | static void bcachecg_destroy(struct cgroup *cgroup) |
| 164 | { |
| 165 | struct bch_cgroup *cg = cgroup_to_bcache(cgroup); |
| 166 | free_css_id(&bcache_subsys, &cg->css); |
| 167 | kfree(cg); |
| 168 | } |
| 169 | |
| 170 | struct cgroup_subsys bcache_subsys = { |
| 171 | .create = bcachecg_create, |
| 172 | .destroy = bcachecg_destroy, |
| 173 | .subsys_id = bcache_subsys_id, |
| 174 | .name = "bcache", |
| 175 | .module = THIS_MODULE, |
| 176 | }; |
| 177 | EXPORT_SYMBOL_GPL(bcache_subsys); |
| 178 | #endif |
| 179 | |
| 180 | static unsigned cache_mode(struct cached_dev *dc, struct bio *bio) |
| 181 | { |
| 182 | #ifdef CONFIG_CGROUP_BCACHE |
| 183 | int r = bch_bio_to_cgroup(bio)->cache_mode; |
| 184 | if (r >= 0) |
| 185 | return r; |
| 186 | #endif |
| 187 | return BDEV_CACHE_MODE(&dc->sb); |
| 188 | } |
| 189 | |
| 190 | static bool verify(struct cached_dev *dc, struct bio *bio) |
| 191 | { |
| 192 | #ifdef CONFIG_CGROUP_BCACHE |
| 193 | if (bch_bio_to_cgroup(bio)->verify) |
| 194 | return true; |
| 195 | #endif |
| 196 | return dc->verify; |
| 197 | } |
| 198 | |
| 199 | static void bio_csum(struct bio *bio, struct bkey *k) |
| 200 | { |
| 201 | struct bio_vec *bv; |
| 202 | uint64_t csum = 0; |
| 203 | int i; |
| 204 | |
| 205 | bio_for_each_segment(bv, bio, i) { |
| 206 | void *d = kmap(bv->bv_page) + bv->bv_offset; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 207 | csum = bch_crc64_update(csum, d, bv->bv_len); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 208 | kunmap(bv->bv_page); |
| 209 | } |
| 210 | |
| 211 | k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1); |
| 212 | } |
| 213 | |
| 214 | /* Insert data into cache */ |
| 215 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 216 | static void bch_data_insert_keys(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 217 | { |
| 218 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 219 | struct search *s = container_of(op, struct search, op); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 220 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 221 | /* |
| 222 | * If we're looping, might already be waiting on |
| 223 | * another journal write - can't wait on more than one journal write at |
| 224 | * a time |
| 225 | * |
| 226 | * XXX: this looks wrong |
| 227 | */ |
| 228 | #if 0 |
| 229 | while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING) |
| 230 | closure_sync(&s->cl); |
| 231 | #endif |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 232 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 233 | if (s->write) |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 234 | op->journal = bch_journal(op->c, &s->insert_keys, |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 235 | op->flush_journal |
| 236 | ? &s->cl : NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 237 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 238 | if (bch_btree_insert(op, op->c, &s->insert_keys)) { |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 239 | s->error = -ENOMEM; |
| 240 | op->insert_data_done = true; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 241 | } |
| 242 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 243 | if (op->journal) |
| 244 | atomic_dec_bug(op->journal); |
| 245 | op->journal = NULL; |
| 246 | |
| 247 | if (!op->insert_data_done) |
| 248 | continue_at(cl, bch_data_insert_start, bcache_wq); |
| 249 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 250 | bch_keylist_free(&s->insert_keys); |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 251 | closure_return(cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 252 | } |
| 253 | |
| 254 | struct open_bucket { |
| 255 | struct list_head list; |
| 256 | struct task_struct *last; |
| 257 | unsigned sectors_free; |
| 258 | BKEY_PADDED(key); |
| 259 | }; |
| 260 | |
| 261 | void bch_open_buckets_free(struct cache_set *c) |
| 262 | { |
| 263 | struct open_bucket *b; |
| 264 | |
| 265 | while (!list_empty(&c->data_buckets)) { |
| 266 | b = list_first_entry(&c->data_buckets, |
| 267 | struct open_bucket, list); |
| 268 | list_del(&b->list); |
| 269 | kfree(b); |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | int bch_open_buckets_alloc(struct cache_set *c) |
| 274 | { |
| 275 | int i; |
| 276 | |
| 277 | spin_lock_init(&c->data_bucket_lock); |
| 278 | |
| 279 | for (i = 0; i < 6; i++) { |
| 280 | struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL); |
| 281 | if (!b) |
| 282 | return -ENOMEM; |
| 283 | |
| 284 | list_add(&b->list, &c->data_buckets); |
| 285 | } |
| 286 | |
| 287 | return 0; |
| 288 | } |
| 289 | |
| 290 | /* |
| 291 | * We keep multiple buckets open for writes, and try to segregate different |
| 292 | * write streams for better cache utilization: first we look for a bucket where |
| 293 | * the last write to it was sequential with the current write, and failing that |
| 294 | * we look for a bucket that was last used by the same task. |
| 295 | * |
| 296 | * The ideas is if you've got multiple tasks pulling data into the cache at the |
| 297 | * same time, you'll get better cache utilization if you try to segregate their |
| 298 | * data and preserve locality. |
| 299 | * |
| 300 | * For example, say you've starting Firefox at the same time you're copying a |
| 301 | * bunch of files. Firefox will likely end up being fairly hot and stay in the |
| 302 | * cache awhile, but the data you copied might not be; if you wrote all that |
| 303 | * data to the same buckets it'd get invalidated at the same time. |
| 304 | * |
| 305 | * Both of those tasks will be doing fairly random IO so we can't rely on |
| 306 | * detecting sequential IO to segregate their data, but going off of the task |
| 307 | * should be a sane heuristic. |
| 308 | */ |
| 309 | static struct open_bucket *pick_data_bucket(struct cache_set *c, |
| 310 | const struct bkey *search, |
| 311 | struct task_struct *task, |
| 312 | struct bkey *alloc) |
| 313 | { |
| 314 | struct open_bucket *ret, *ret_task = NULL; |
| 315 | |
| 316 | list_for_each_entry_reverse(ret, &c->data_buckets, list) |
| 317 | if (!bkey_cmp(&ret->key, search)) |
| 318 | goto found; |
| 319 | else if (ret->last == task) |
| 320 | ret_task = ret; |
| 321 | |
| 322 | ret = ret_task ?: list_first_entry(&c->data_buckets, |
| 323 | struct open_bucket, list); |
| 324 | found: |
| 325 | if (!ret->sectors_free && KEY_PTRS(alloc)) { |
| 326 | ret->sectors_free = c->sb.bucket_size; |
| 327 | bkey_copy(&ret->key, alloc); |
| 328 | bkey_init(alloc); |
| 329 | } |
| 330 | |
| 331 | if (!ret->sectors_free) |
| 332 | ret = NULL; |
| 333 | |
| 334 | return ret; |
| 335 | } |
| 336 | |
| 337 | /* |
| 338 | * Allocates some space in the cache to write to, and k to point to the newly |
| 339 | * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the |
| 340 | * end of the newly allocated space). |
| 341 | * |
| 342 | * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many |
| 343 | * sectors were actually allocated. |
| 344 | * |
| 345 | * If s->writeback is true, will not fail. |
| 346 | */ |
| 347 | static bool bch_alloc_sectors(struct bkey *k, unsigned sectors, |
| 348 | struct search *s) |
| 349 | { |
| 350 | struct cache_set *c = s->op.c; |
| 351 | struct open_bucket *b; |
| 352 | BKEY_PADDED(key) alloc; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 353 | unsigned i; |
| 354 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 355 | /* |
| 356 | * We might have to allocate a new bucket, which we can't do with a |
| 357 | * spinlock held. So if we have to allocate, we drop the lock, allocate |
| 358 | * and then retry. KEY_PTRS() indicates whether alloc points to |
| 359 | * allocated bucket(s). |
| 360 | */ |
| 361 | |
| 362 | bkey_init(&alloc.key); |
| 363 | spin_lock(&c->data_bucket_lock); |
| 364 | |
| 365 | while (!(b = pick_data_bucket(c, k, s->task, &alloc.key))) { |
| 366 | unsigned watermark = s->op.write_prio |
| 367 | ? WATERMARK_MOVINGGC |
| 368 | : WATERMARK_NONE; |
| 369 | |
| 370 | spin_unlock(&c->data_bucket_lock); |
| 371 | |
Kent Overstreet | 35fcd84 | 2013-07-24 17:29:09 -0700 | [diff] [blame] | 372 | if (bch_bucket_alloc_set(c, watermark, &alloc.key, |
| 373 | 1, s->writeback)) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 374 | return false; |
| 375 | |
| 376 | spin_lock(&c->data_bucket_lock); |
| 377 | } |
| 378 | |
| 379 | /* |
| 380 | * If we had to allocate, we might race and not need to allocate the |
| 381 | * second time we call find_data_bucket(). If we allocated a bucket but |
| 382 | * didn't use it, drop the refcount bch_bucket_alloc_set() took: |
| 383 | */ |
| 384 | if (KEY_PTRS(&alloc.key)) |
| 385 | __bkey_put(c, &alloc.key); |
| 386 | |
| 387 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 388 | EBUG_ON(ptr_stale(c, &b->key, i)); |
| 389 | |
| 390 | /* Set up the pointer to the space we're allocating: */ |
| 391 | |
| 392 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 393 | k->ptr[i] = b->key.ptr[i]; |
| 394 | |
| 395 | sectors = min(sectors, b->sectors_free); |
| 396 | |
| 397 | SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors); |
| 398 | SET_KEY_SIZE(k, sectors); |
| 399 | SET_KEY_PTRS(k, KEY_PTRS(&b->key)); |
| 400 | |
| 401 | /* |
| 402 | * Move b to the end of the lru, and keep track of what this bucket was |
| 403 | * last used for: |
| 404 | */ |
| 405 | list_move_tail(&b->list, &c->data_buckets); |
| 406 | bkey_copy_key(&b->key, k); |
| 407 | b->last = s->task; |
| 408 | |
| 409 | b->sectors_free -= sectors; |
| 410 | |
| 411 | for (i = 0; i < KEY_PTRS(&b->key); i++) { |
| 412 | SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors); |
| 413 | |
| 414 | atomic_long_add(sectors, |
| 415 | &PTR_CACHE(c, &b->key, i)->sectors_written); |
| 416 | } |
| 417 | |
| 418 | if (b->sectors_free < c->sb.block_size) |
| 419 | b->sectors_free = 0; |
| 420 | |
| 421 | /* |
| 422 | * k takes refcounts on the buckets it points to until it's inserted |
| 423 | * into the btree, but if we're done with this bucket we just transfer |
| 424 | * get_data_bucket()'s refcount. |
| 425 | */ |
| 426 | if (b->sectors_free) |
| 427 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 428 | atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin); |
| 429 | |
| 430 | spin_unlock(&c->data_bucket_lock); |
| 431 | return true; |
| 432 | } |
| 433 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 434 | static void bch_data_invalidate(struct closure *cl) |
| 435 | { |
| 436 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 437 | struct search *s = container_of(op, struct search, op); |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 438 | struct bio *bio = op->cache_bio; |
| 439 | |
| 440 | pr_debug("invalidating %i sectors from %llu", |
| 441 | bio_sectors(bio), (uint64_t) bio->bi_sector); |
| 442 | |
| 443 | while (bio_sectors(bio)) { |
| 444 | unsigned len = min(bio_sectors(bio), 1U << 14); |
| 445 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 446 | if (bch_keylist_realloc(&s->insert_keys, 0, op->c)) |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 447 | goto out; |
| 448 | |
| 449 | bio->bi_sector += len; |
| 450 | bio->bi_size -= len << 9; |
| 451 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 452 | bch_keylist_add(&s->insert_keys, |
| 453 | &KEY(op->inode, bio->bi_sector, len)); |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 454 | } |
| 455 | |
| 456 | op->insert_data_done = true; |
| 457 | bio_put(bio); |
| 458 | out: |
| 459 | continue_at(cl, bch_data_insert_keys, bcache_wq); |
| 460 | } |
| 461 | |
| 462 | static void bch_data_insert_error(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 463 | { |
| 464 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 465 | struct search *s = container_of(op, struct search, op); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 466 | |
| 467 | /* |
| 468 | * Our data write just errored, which means we've got a bunch of keys to |
| 469 | * insert that point to data that wasn't succesfully written. |
| 470 | * |
| 471 | * We don't have to insert those keys but we still have to invalidate |
| 472 | * that region of the cache - so, if we just strip off all the pointers |
| 473 | * from the keys we'll accomplish just that. |
| 474 | */ |
| 475 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 476 | struct bkey *src = s->insert_keys.keys, *dst = s->insert_keys.keys; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 477 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 478 | while (src != s->insert_keys.top) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 479 | struct bkey *n = bkey_next(src); |
| 480 | |
| 481 | SET_KEY_PTRS(src, 0); |
Kent Overstreet | c2f95ae | 2013-07-24 17:24:25 -0700 | [diff] [blame] | 482 | memmove(dst, src, bkey_bytes(src)); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 483 | |
| 484 | dst = bkey_next(dst); |
| 485 | src = n; |
| 486 | } |
| 487 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 488 | s->insert_keys.top = dst; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 489 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 490 | bch_data_insert_keys(cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 491 | } |
| 492 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 493 | static void bch_data_insert_endio(struct bio *bio, int error) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 494 | { |
| 495 | struct closure *cl = bio->bi_private; |
| 496 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 497 | struct search *s = container_of(op, struct search, op); |
| 498 | |
| 499 | if (error) { |
| 500 | /* TODO: We could try to recover from this. */ |
| 501 | if (s->writeback) |
| 502 | s->error = error; |
| 503 | else if (s->write) |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 504 | set_closure_fn(cl, bch_data_insert_error, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 505 | else |
| 506 | set_closure_fn(cl, NULL, NULL); |
| 507 | } |
| 508 | |
| 509 | bch_bbio_endio(op->c, bio, error, "writing data to cache"); |
| 510 | } |
| 511 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 512 | static void bch_data_insert_start(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 513 | { |
| 514 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 515 | struct search *s = container_of(op, struct search, op); |
| 516 | struct bio *bio = op->cache_bio, *n; |
| 517 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 518 | if (op->bypass) |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 519 | return bch_data_invalidate(cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 520 | |
| 521 | if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) { |
| 522 | set_gc_sectors(op->c); |
Kent Overstreet | 72a4451 | 2013-10-24 17:19:26 -0700 | [diff] [blame] | 523 | wake_up_gc(op->c); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 524 | } |
| 525 | |
Kent Overstreet | 54d12f2 | 2013-07-10 18:44:40 -0700 | [diff] [blame] | 526 | /* |
| 527 | * Journal writes are marked REQ_FLUSH; if the original write was a |
| 528 | * flush, it'll wait on the journal write. |
| 529 | */ |
| 530 | bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA); |
| 531 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 532 | do { |
| 533 | unsigned i; |
| 534 | struct bkey *k; |
| 535 | struct bio_set *split = s->d |
| 536 | ? s->d->bio_split : op->c->bio_split; |
| 537 | |
| 538 | /* 1 for the device pointer and 1 for the chksum */ |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 539 | if (bch_keylist_realloc(&s->insert_keys, |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 540 | 1 + (op->csum ? 1 : 0), |
| 541 | op->c)) |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 542 | continue_at(cl, bch_data_insert_keys, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 543 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 544 | k = s->insert_keys.top; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 545 | bkey_init(k); |
| 546 | SET_KEY_INODE(k, op->inode); |
| 547 | SET_KEY_OFFSET(k, bio->bi_sector); |
| 548 | |
| 549 | if (!bch_alloc_sectors(k, bio_sectors(bio), s)) |
| 550 | goto err; |
| 551 | |
| 552 | n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 553 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 554 | n->bi_end_io = bch_data_insert_endio; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 555 | n->bi_private = cl; |
| 556 | |
| 557 | if (s->writeback) { |
| 558 | SET_KEY_DIRTY(k, true); |
| 559 | |
| 560 | for (i = 0; i < KEY_PTRS(k); i++) |
| 561 | SET_GC_MARK(PTR_BUCKET(op->c, k, i), |
| 562 | GC_MARK_DIRTY); |
| 563 | } |
| 564 | |
| 565 | SET_KEY_CSUM(k, op->csum); |
| 566 | if (KEY_CSUM(k)) |
| 567 | bio_csum(n, k); |
| 568 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 569 | trace_bcache_cache_insert(k); |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 570 | bch_keylist_push(&s->insert_keys); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 571 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 572 | n->bi_rw |= REQ_WRITE; |
| 573 | bch_submit_bbio(n, op->c, k, 0); |
| 574 | } while (n != bio); |
| 575 | |
| 576 | op->insert_data_done = true; |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 577 | continue_at(cl, bch_data_insert_keys, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 578 | err: |
| 579 | /* bch_alloc_sectors() blocks if s->writeback = true */ |
| 580 | BUG_ON(s->writeback); |
| 581 | |
| 582 | /* |
| 583 | * But if it's not a writeback write we'd rather just bail out if |
| 584 | * there aren't any buckets ready to write to - it might take awhile and |
| 585 | * we might be starving btree writes for gc or something. |
| 586 | */ |
| 587 | |
| 588 | if (s->write) { |
| 589 | /* |
| 590 | * Writethrough write: We can't complete the write until we've |
| 591 | * updated the index. But we don't want to delay the write while |
| 592 | * we wait for buckets to be freed up, so just invalidate the |
| 593 | * rest of the write. |
| 594 | */ |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 595 | op->bypass = true; |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 596 | return bch_data_invalidate(cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 597 | } else { |
| 598 | /* |
| 599 | * From a cache miss, we can just insert the keys for the data |
| 600 | * we have written or bail out if we didn't do anything. |
| 601 | */ |
| 602 | op->insert_data_done = true; |
| 603 | bio_put(bio); |
| 604 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 605 | if (!bch_keylist_empty(&s->insert_keys)) |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 606 | continue_at(cl, bch_data_insert_keys, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 607 | else |
| 608 | closure_return(cl); |
| 609 | } |
| 610 | } |
| 611 | |
| 612 | /** |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 613 | * bch_data_insert - stick some data in the cache |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 614 | * |
| 615 | * This is the starting point for any data to end up in a cache device; it could |
| 616 | * be from a normal write, or a writeback write, or a write to a flash only |
| 617 | * volume - it's also used by the moving garbage collector to compact data in |
| 618 | * mostly empty buckets. |
| 619 | * |
| 620 | * It first writes the data to the cache, creating a list of keys to be inserted |
| 621 | * (if the data had to be fragmented there will be multiple keys); after the |
| 622 | * data is written it calls bch_journal, and after the keys have been added to |
| 623 | * the next journal write they're inserted into the btree. |
| 624 | * |
| 625 | * It inserts the data in op->cache_bio; bi_sector is used for the key offset, |
| 626 | * and op->inode is used for the key inode. |
| 627 | * |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 628 | * If op->bypass is true, instead of inserting the data it invalidates the |
| 629 | * region of the cache represented by op->cache_bio and op->inode. |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 630 | */ |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 631 | void bch_data_insert(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 632 | { |
| 633 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 634 | struct search *s = container_of(op, struct search, op); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 635 | |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 636 | bch_keylist_init(&s->insert_keys); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 637 | bio_get(op->cache_bio); |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 638 | bch_data_insert_start(cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 639 | } |
| 640 | |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 641 | /* Cache lookup */ |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 642 | |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 643 | static void bch_cache_read_endio(struct bio *bio, int error) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 644 | { |
| 645 | struct bbio *b = container_of(bio, struct bbio, bio); |
| 646 | struct closure *cl = bio->bi_private; |
| 647 | struct search *s = container_of(cl, struct search, cl); |
| 648 | |
| 649 | /* |
| 650 | * If the bucket was reused while our bio was in flight, we might have |
| 651 | * read the wrong data. Set s->error but not error so it doesn't get |
| 652 | * counted against the cache device, but we'll still reread the data |
| 653 | * from the backing device. |
| 654 | */ |
| 655 | |
| 656 | if (error) |
| 657 | s->error = error; |
| 658 | else if (ptr_stale(s->op.c, &b->key, 0)) { |
| 659 | atomic_long_inc(&s->op.c->cache_read_races); |
| 660 | s->error = -EINTR; |
| 661 | } |
| 662 | |
| 663 | bch_bbio_endio(s->op.c, bio, error, "reading from cache"); |
| 664 | } |
| 665 | |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 666 | static int submit_partial_cache_miss(struct btree *b, struct search *s, |
| 667 | struct bkey *k) |
| 668 | { |
| 669 | struct bio *bio = &s->bio.bio; |
| 670 | int ret = MAP_CONTINUE; |
| 671 | |
| 672 | do { |
| 673 | unsigned sectors = INT_MAX; |
| 674 | |
| 675 | if (KEY_INODE(k) == s->op.inode) { |
| 676 | if (KEY_START(k) <= bio->bi_sector) |
| 677 | break; |
| 678 | |
| 679 | sectors = min_t(uint64_t, sectors, |
| 680 | KEY_START(k) - bio->bi_sector); |
| 681 | } |
| 682 | |
| 683 | ret = s->d->cache_miss(b, s, bio, sectors); |
| 684 | } while (ret == MAP_CONTINUE); |
| 685 | |
| 686 | return ret; |
| 687 | } |
| 688 | |
| 689 | /* |
| 690 | * Read from a single key, handling the initial cache miss if the key starts in |
| 691 | * the middle of the bio |
| 692 | */ |
| 693 | static int submit_partial_cache_hit(struct btree_op *op, struct btree *b, |
| 694 | struct bkey *k) |
| 695 | { |
| 696 | struct search *s = container_of(op, struct search, op); |
| 697 | struct bio *bio = &s->bio.bio; |
| 698 | unsigned ptr; |
| 699 | struct bio *n; |
| 700 | |
| 701 | int ret = submit_partial_cache_miss(b, s, k); |
| 702 | if (ret != MAP_CONTINUE || !KEY_SIZE(k)) |
| 703 | return ret; |
| 704 | |
| 705 | /* XXX: figure out best pointer - for multiple cache devices */ |
| 706 | ptr = 0; |
| 707 | |
| 708 | PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO; |
| 709 | |
| 710 | while (ret == MAP_CONTINUE && |
| 711 | KEY_INODE(k) == op->inode && |
| 712 | bio->bi_sector < KEY_OFFSET(k)) { |
| 713 | struct bkey *bio_key; |
| 714 | sector_t sector = PTR_OFFSET(k, ptr) + |
| 715 | (bio->bi_sector - KEY_START(k)); |
| 716 | unsigned sectors = min_t(uint64_t, INT_MAX, |
| 717 | KEY_OFFSET(k) - bio->bi_sector); |
| 718 | |
| 719 | n = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split); |
| 720 | if (n == bio) |
| 721 | ret = MAP_DONE; |
| 722 | |
| 723 | bio_key = &container_of(n, struct bbio, bio)->key; |
| 724 | |
| 725 | /* |
| 726 | * The bucket we're reading from might be reused while our bio |
| 727 | * is in flight, and we could then end up reading the wrong |
| 728 | * data. |
| 729 | * |
| 730 | * We guard against this by checking (in cache_read_endio()) if |
| 731 | * the pointer is stale again; if so, we treat it as an error |
| 732 | * and reread from the backing device (but we don't pass that |
| 733 | * error up anywhere). |
| 734 | */ |
| 735 | |
| 736 | bch_bkey_copy_single_ptr(bio_key, k, ptr); |
| 737 | SET_PTR_OFFSET(bio_key, 0, sector); |
| 738 | |
| 739 | n->bi_end_io = bch_cache_read_endio; |
| 740 | n->bi_private = &s->cl; |
| 741 | |
| 742 | __bch_submit_bbio(n, b->c); |
| 743 | } |
| 744 | |
| 745 | return ret; |
| 746 | } |
| 747 | |
| 748 | static void cache_lookup(struct closure *cl) |
| 749 | { |
| 750 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 751 | struct search *s = container_of(op, struct search, op); |
| 752 | struct bio *bio = &s->bio.bio; |
| 753 | |
| 754 | int ret = bch_btree_map_keys(op, op->c, |
| 755 | &KEY(op->inode, bio->bi_sector, 0), |
| 756 | submit_partial_cache_hit, 1); |
| 757 | if (ret == -EAGAIN) |
| 758 | continue_at(cl, cache_lookup, bcache_wq); |
| 759 | |
| 760 | closure_return(cl); |
| 761 | } |
| 762 | |
| 763 | /* Common code for the make_request functions */ |
| 764 | |
| 765 | static void request_endio(struct bio *bio, int error) |
| 766 | { |
| 767 | struct closure *cl = bio->bi_private; |
| 768 | |
| 769 | if (error) { |
| 770 | struct search *s = container_of(cl, struct search, cl); |
| 771 | s->error = error; |
| 772 | /* Only cache read errors are recoverable */ |
| 773 | s->recoverable = false; |
| 774 | } |
| 775 | |
| 776 | bio_put(bio); |
| 777 | closure_put(cl); |
| 778 | } |
| 779 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 780 | static void bio_complete(struct search *s) |
| 781 | { |
| 782 | if (s->orig_bio) { |
| 783 | int cpu, rw = bio_data_dir(s->orig_bio); |
| 784 | unsigned long duration = jiffies - s->start_time; |
| 785 | |
| 786 | cpu = part_stat_lock(); |
| 787 | part_round_stats(cpu, &s->d->disk->part0); |
| 788 | part_stat_add(cpu, &s->d->disk->part0, ticks[rw], duration); |
| 789 | part_stat_unlock(); |
| 790 | |
| 791 | trace_bcache_request_end(s, s->orig_bio); |
| 792 | bio_endio(s->orig_bio, s->error); |
| 793 | s->orig_bio = NULL; |
| 794 | } |
| 795 | } |
| 796 | |
| 797 | static void do_bio_hook(struct search *s) |
| 798 | { |
| 799 | struct bio *bio = &s->bio.bio; |
| 800 | memcpy(bio, s->orig_bio, sizeof(struct bio)); |
| 801 | |
| 802 | bio->bi_end_io = request_endio; |
| 803 | bio->bi_private = &s->cl; |
| 804 | atomic_set(&bio->bi_cnt, 3); |
| 805 | } |
| 806 | |
| 807 | static void search_free(struct closure *cl) |
| 808 | { |
| 809 | struct search *s = container_of(cl, struct search, cl); |
| 810 | bio_complete(s); |
| 811 | |
| 812 | if (s->op.cache_bio) |
| 813 | bio_put(s->op.cache_bio); |
| 814 | |
| 815 | if (s->unaligned_bvec) |
| 816 | mempool_free(s->bio.bio.bi_io_vec, s->d->unaligned_bvec); |
| 817 | |
| 818 | closure_debug_destroy(cl); |
| 819 | mempool_free(s, s->d->c->search); |
| 820 | } |
| 821 | |
| 822 | static struct search *search_alloc(struct bio *bio, struct bcache_device *d) |
| 823 | { |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 824 | struct search *s; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 825 | struct bio_vec *bv; |
Kent Overstreet | 0b93207 | 2013-07-24 17:26:51 -0700 | [diff] [blame] | 826 | |
| 827 | s = mempool_alloc(d->c->search, GFP_NOIO); |
| 828 | memset(s, 0, offsetof(struct search, insert_keys)); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 829 | |
| 830 | __closure_init(&s->cl, NULL); |
| 831 | |
| 832 | s->op.inode = d->id; |
| 833 | s->op.c = d->c; |
| 834 | s->d = d; |
| 835 | s->op.lock = -1; |
| 836 | s->task = current; |
| 837 | s->orig_bio = bio; |
| 838 | s->write = (bio->bi_rw & REQ_WRITE) != 0; |
Kent Overstreet | 54d12f2 | 2013-07-10 18:44:40 -0700 | [diff] [blame] | 839 | s->op.flush_journal = (bio->bi_rw & (REQ_FLUSH|REQ_FUA)) != 0; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 840 | s->recoverable = 1; |
| 841 | s->start_time = jiffies; |
| 842 | do_bio_hook(s); |
| 843 | |
| 844 | if (bio->bi_size != bio_segments(bio) * PAGE_SIZE) { |
| 845 | bv = mempool_alloc(d->unaligned_bvec, GFP_NOIO); |
| 846 | memcpy(bv, bio_iovec(bio), |
| 847 | sizeof(struct bio_vec) * bio_segments(bio)); |
| 848 | |
| 849 | s->bio.bio.bi_io_vec = bv; |
| 850 | s->unaligned_bvec = 1; |
| 851 | } |
| 852 | |
| 853 | return s; |
| 854 | } |
| 855 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 856 | /* Cached devices */ |
| 857 | |
| 858 | static void cached_dev_bio_complete(struct closure *cl) |
| 859 | { |
| 860 | struct search *s = container_of(cl, struct search, cl); |
| 861 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 862 | |
| 863 | search_free(cl); |
| 864 | cached_dev_put(dc); |
| 865 | } |
| 866 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 867 | unsigned bch_get_congested(struct cache_set *c) |
| 868 | { |
| 869 | int i; |
| 870 | long rand; |
| 871 | |
| 872 | if (!c->congested_read_threshold_us && |
| 873 | !c->congested_write_threshold_us) |
| 874 | return 0; |
| 875 | |
| 876 | i = (local_clock_us() - c->congested_last_us) / 1024; |
| 877 | if (i < 0) |
| 878 | return 0; |
| 879 | |
| 880 | i += atomic_read(&c->congested); |
| 881 | if (i >= 0) |
| 882 | return 0; |
| 883 | |
| 884 | i += CONGESTED_MAX; |
| 885 | |
| 886 | if (i > 0) |
| 887 | i = fract_exp_two(i, 6); |
| 888 | |
| 889 | rand = get_random_int(); |
| 890 | i -= bitmap_weight(&rand, BITS_PER_LONG); |
| 891 | |
| 892 | return i > 0 ? i : 1; |
| 893 | } |
| 894 | |
| 895 | static void add_sequential(struct task_struct *t) |
| 896 | { |
| 897 | ewma_add(t->sequential_io_avg, |
| 898 | t->sequential_io, 8, 0); |
| 899 | |
| 900 | t->sequential_io = 0; |
| 901 | } |
| 902 | |
| 903 | static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k) |
| 904 | { |
| 905 | return &dc->io_hash[hash_64(k, RECENT_IO_BITS)]; |
| 906 | } |
| 907 | |
| 908 | static bool check_should_bypass(struct cached_dev *dc, struct search *s) |
| 909 | { |
| 910 | struct cache_set *c = s->op.c; |
| 911 | struct bio *bio = &s->bio.bio; |
| 912 | unsigned mode = cache_mode(dc, bio); |
| 913 | unsigned sectors, congested = bch_get_congested(c); |
| 914 | |
| 915 | if (atomic_read(&dc->disk.detaching) || |
| 916 | c->gc_stats.in_use > CUTOFF_CACHE_ADD || |
| 917 | (bio->bi_rw & REQ_DISCARD)) |
| 918 | goto skip; |
| 919 | |
| 920 | if (mode == CACHE_MODE_NONE || |
| 921 | (mode == CACHE_MODE_WRITEAROUND && |
| 922 | (bio->bi_rw & REQ_WRITE))) |
| 923 | goto skip; |
| 924 | |
| 925 | if (bio->bi_sector & (c->sb.block_size - 1) || |
| 926 | bio_sectors(bio) & (c->sb.block_size - 1)) { |
| 927 | pr_debug("skipping unaligned io"); |
| 928 | goto skip; |
| 929 | } |
| 930 | |
| 931 | if (!congested && !dc->sequential_cutoff) |
| 932 | goto rescale; |
| 933 | |
| 934 | if (!congested && |
| 935 | mode == CACHE_MODE_WRITEBACK && |
| 936 | (bio->bi_rw & REQ_WRITE) && |
| 937 | (bio->bi_rw & REQ_SYNC)) |
| 938 | goto rescale; |
| 939 | |
| 940 | if (dc->sequential_merge) { |
| 941 | struct io *i; |
| 942 | |
| 943 | spin_lock(&dc->io_lock); |
| 944 | |
| 945 | hlist_for_each_entry(i, iohash(dc, bio->bi_sector), hash) |
| 946 | if (i->last == bio->bi_sector && |
| 947 | time_before(jiffies, i->jiffies)) |
| 948 | goto found; |
| 949 | |
| 950 | i = list_first_entry(&dc->io_lru, struct io, lru); |
| 951 | |
| 952 | add_sequential(s->task); |
| 953 | i->sequential = 0; |
| 954 | found: |
| 955 | if (i->sequential + bio->bi_size > i->sequential) |
| 956 | i->sequential += bio->bi_size; |
| 957 | |
| 958 | i->last = bio_end_sector(bio); |
| 959 | i->jiffies = jiffies + msecs_to_jiffies(5000); |
| 960 | s->task->sequential_io = i->sequential; |
| 961 | |
| 962 | hlist_del(&i->hash); |
| 963 | hlist_add_head(&i->hash, iohash(dc, i->last)); |
| 964 | list_move_tail(&i->lru, &dc->io_lru); |
| 965 | |
| 966 | spin_unlock(&dc->io_lock); |
| 967 | } else { |
| 968 | s->task->sequential_io = bio->bi_size; |
| 969 | |
| 970 | add_sequential(s->task); |
| 971 | } |
| 972 | |
| 973 | sectors = max(s->task->sequential_io, |
| 974 | s->task->sequential_io_avg) >> 9; |
| 975 | |
| 976 | if (dc->sequential_cutoff && |
| 977 | sectors >= dc->sequential_cutoff >> 9) { |
| 978 | trace_bcache_bypass_sequential(s->orig_bio); |
| 979 | goto skip; |
| 980 | } |
| 981 | |
| 982 | if (congested && sectors >= congested) { |
| 983 | trace_bcache_bypass_congested(s->orig_bio); |
| 984 | goto skip; |
| 985 | } |
| 986 | |
| 987 | rescale: |
| 988 | bch_rescale_priorities(c, bio_sectors(bio)); |
| 989 | return false; |
| 990 | skip: |
| 991 | bch_mark_sectors_bypassed(s, bio_sectors(bio)); |
| 992 | return true; |
| 993 | } |
| 994 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 995 | /* Process reads */ |
| 996 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 997 | static void cached_dev_cache_miss_done(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 998 | { |
| 999 | struct search *s = container_of(cl, struct search, cl); |
| 1000 | |
| 1001 | if (s->op.insert_collision) |
| 1002 | bch_mark_cache_miss_collision(s); |
| 1003 | |
| 1004 | if (s->op.cache_bio) { |
| 1005 | int i; |
| 1006 | struct bio_vec *bv; |
| 1007 | |
| 1008 | __bio_for_each_segment(bv, s->op.cache_bio, i, 0) |
| 1009 | __free_page(bv->bv_page); |
| 1010 | } |
| 1011 | |
| 1012 | cached_dev_bio_complete(cl); |
| 1013 | } |
| 1014 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1015 | static void cached_dev_read_error(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1016 | { |
| 1017 | struct search *s = container_of(cl, struct search, cl); |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1018 | struct bio *bio = &s->bio.bio; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1019 | struct bio_vec *bv; |
| 1020 | int i; |
| 1021 | |
| 1022 | if (s->recoverable) { |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1023 | /* Retry from the backing device: */ |
| 1024 | trace_bcache_read_retry(s->orig_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1025 | |
| 1026 | s->error = 0; |
| 1027 | bv = s->bio.bio.bi_io_vec; |
| 1028 | do_bio_hook(s); |
| 1029 | s->bio.bio.bi_io_vec = bv; |
| 1030 | |
| 1031 | if (!s->unaligned_bvec) |
| 1032 | bio_for_each_segment(bv, s->orig_bio, i) |
| 1033 | bv->bv_offset = 0, bv->bv_len = PAGE_SIZE; |
| 1034 | else |
| 1035 | memcpy(s->bio.bio.bi_io_vec, |
| 1036 | bio_iovec(s->orig_bio), |
| 1037 | sizeof(struct bio_vec) * |
| 1038 | bio_segments(s->orig_bio)); |
| 1039 | |
| 1040 | /* XXX: invalidate cache */ |
| 1041 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1042 | closure_bio_submit(bio, cl, s->d); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1043 | } |
| 1044 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1045 | continue_at(cl, cached_dev_cache_miss_done, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1046 | } |
| 1047 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1048 | static void cached_dev_read_done(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1049 | { |
| 1050 | struct search *s = container_of(cl, struct search, cl); |
| 1051 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 1052 | |
| 1053 | /* |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1054 | * We had a cache miss; cache_bio now contains data ready to be inserted |
| 1055 | * into the cache. |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1056 | * |
| 1057 | * First, we copy the data we just read from cache_bio's bounce buffers |
| 1058 | * to the buffers the original bio pointed to: |
| 1059 | */ |
| 1060 | |
| 1061 | if (s->op.cache_bio) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1062 | bio_reset(s->op.cache_bio); |
| 1063 | s->op.cache_bio->bi_sector = s->cache_miss->bi_sector; |
| 1064 | s->op.cache_bio->bi_bdev = s->cache_miss->bi_bdev; |
| 1065 | s->op.cache_bio->bi_size = s->cache_bio_sectors << 9; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 1066 | bch_bio_map(s->op.cache_bio, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1067 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1068 | bio_copy_data(s->cache_miss, s->op.cache_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1069 | |
| 1070 | bio_put(s->cache_miss); |
| 1071 | s->cache_miss = NULL; |
| 1072 | } |
| 1073 | |
| 1074 | if (verify(dc, &s->bio.bio) && s->recoverable) |
| 1075 | bch_data_verify(s); |
| 1076 | |
| 1077 | bio_complete(s); |
| 1078 | |
| 1079 | if (s->op.cache_bio && |
| 1080 | !test_bit(CACHE_SET_STOPPING, &s->op.c->flags)) { |
| 1081 | s->op.type = BTREE_REPLACE; |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1082 | closure_call(&s->op.cl, bch_data_insert, NULL, cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1083 | } |
| 1084 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1085 | continue_at(cl, cached_dev_cache_miss_done, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1086 | } |
| 1087 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1088 | static void cached_dev_read_done_bh(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1089 | { |
| 1090 | struct search *s = container_of(cl, struct search, cl); |
| 1091 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 1092 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1093 | bch_mark_cache_accounting(s, !s->cache_miss, s->op.bypass); |
| 1094 | trace_bcache_read(s->orig_bio, !s->cache_miss, s->op.bypass); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1095 | |
| 1096 | if (s->error) |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1097 | continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1098 | else if (s->op.cache_bio || verify(dc, &s->bio.bio)) |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1099 | continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1100 | else |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1101 | continue_at_nobarrier(cl, cached_dev_bio_complete, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1102 | } |
| 1103 | |
| 1104 | static int cached_dev_cache_miss(struct btree *b, struct search *s, |
| 1105 | struct bio *bio, unsigned sectors) |
| 1106 | { |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1107 | int ret = MAP_CONTINUE; |
Kent Overstreet | e7c590e | 2013-09-10 18:39:16 -0700 | [diff] [blame] | 1108 | unsigned reada = 0; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1109 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1110 | struct bio *miss, *cache_bio; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1111 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1112 | if (s->cache_miss || s->op.bypass) { |
Kent Overstreet | e7c590e | 2013-09-10 18:39:16 -0700 | [diff] [blame] | 1113 | miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split); |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1114 | ret = miss == bio ? MAP_DONE : MAP_CONTINUE; |
Kent Overstreet | e7c590e | 2013-09-10 18:39:16 -0700 | [diff] [blame] | 1115 | goto out_submit; |
| 1116 | } |
| 1117 | |
| 1118 | if (!(bio->bi_rw & REQ_RAHEAD) && |
| 1119 | !(bio->bi_rw & REQ_META) && |
| 1120 | s->op.c->gc_stats.in_use < CUTOFF_CACHE_READA) |
| 1121 | reada = min_t(sector_t, dc->readahead >> 9, |
| 1122 | bdev_sectors(bio->bi_bdev) - bio_end_sector(bio)); |
| 1123 | |
| 1124 | s->cache_bio_sectors = min(sectors, bio_sectors(bio) + reada); |
| 1125 | |
| 1126 | s->op.replace = KEY(s->op.inode, bio->bi_sector + |
| 1127 | s->cache_bio_sectors, s->cache_bio_sectors); |
| 1128 | |
| 1129 | ret = bch_btree_insert_check_key(b, &s->op, &s->op.replace); |
| 1130 | if (ret) |
| 1131 | return ret; |
| 1132 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1133 | miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split); |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1134 | |
| 1135 | /* btree_search_recurse()'s btree iterator is no good anymore */ |
| 1136 | ret = miss == bio ? MAP_DONE : -EINTR; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1137 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1138 | cache_bio = bio_alloc_bioset(GFP_NOWAIT, |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1139 | DIV_ROUND_UP(s->cache_bio_sectors, PAGE_SECTORS), |
| 1140 | dc->disk.bio_split); |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1141 | if (!cache_bio) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1142 | goto out_submit; |
| 1143 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1144 | cache_bio->bi_sector = miss->bi_sector; |
| 1145 | cache_bio->bi_bdev = miss->bi_bdev; |
| 1146 | cache_bio->bi_size = s->cache_bio_sectors << 9; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1147 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1148 | cache_bio->bi_end_io = request_endio; |
| 1149 | cache_bio->bi_private = &s->cl; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1150 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1151 | bch_bio_map(cache_bio, NULL); |
| 1152 | if (bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO)) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1153 | goto out_put; |
| 1154 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1155 | s->cache_miss = miss; |
| 1156 | s->op.cache_bio = cache_bio; |
| 1157 | bio_get(cache_bio); |
| 1158 | closure_bio_submit(cache_bio, &s->cl, s->d); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1159 | |
| 1160 | return ret; |
| 1161 | out_put: |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1162 | bio_put(cache_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1163 | out_submit: |
Kent Overstreet | e7c590e | 2013-09-10 18:39:16 -0700 | [diff] [blame] | 1164 | miss->bi_end_io = request_endio; |
| 1165 | miss->bi_private = &s->cl; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1166 | closure_bio_submit(miss, &s->cl, s->d); |
| 1167 | return ret; |
| 1168 | } |
| 1169 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1170 | static void cached_dev_read(struct cached_dev *dc, struct search *s) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1171 | { |
| 1172 | struct closure *cl = &s->cl; |
| 1173 | |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1174 | closure_call(&s->op.cl, cache_lookup, NULL, cl); |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1175 | continue_at(cl, cached_dev_read_done_bh, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1176 | } |
| 1177 | |
| 1178 | /* Process writes */ |
| 1179 | |
| 1180 | static void cached_dev_write_complete(struct closure *cl) |
| 1181 | { |
| 1182 | struct search *s = container_of(cl, struct search, cl); |
| 1183 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 1184 | |
| 1185 | up_read_non_owner(&dc->writeback_lock); |
| 1186 | cached_dev_bio_complete(cl); |
| 1187 | } |
| 1188 | |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1189 | static void cached_dev_write(struct cached_dev *dc, struct search *s) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1190 | { |
| 1191 | struct closure *cl = &s->cl; |
| 1192 | struct bio *bio = &s->bio.bio; |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1193 | struct bkey start = KEY(dc->disk.id, bio->bi_sector, 0); |
| 1194 | struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1195 | |
| 1196 | bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, &start, &end); |
| 1197 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1198 | down_read_non_owner(&dc->writeback_lock); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1199 | if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) { |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1200 | /* |
| 1201 | * We overlap with some dirty data undergoing background |
| 1202 | * writeback, force this write to writeback |
| 1203 | */ |
| 1204 | s->op.bypass = false; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1205 | s->writeback = true; |
| 1206 | } |
| 1207 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1208 | /* |
| 1209 | * Discards aren't _required_ to do anything, so skipping if |
| 1210 | * check_overlapping returned true is ok |
| 1211 | * |
| 1212 | * But check_overlapping drops dirty keys for which io hasn't started, |
| 1213 | * so we still want to call it. |
| 1214 | */ |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1215 | if (bio->bi_rw & REQ_DISCARD) |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1216 | s->op.bypass = true; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1217 | |
Kent Overstreet | 72c2706 | 2013-06-05 06:24:39 -0700 | [diff] [blame] | 1218 | if (should_writeback(dc, s->orig_bio, |
| 1219 | cache_mode(dc, bio), |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1220 | s->op.bypass)) { |
| 1221 | s->op.bypass = false; |
Kent Overstreet | 72c2706 | 2013-06-05 06:24:39 -0700 | [diff] [blame] | 1222 | s->writeback = true; |
| 1223 | } |
| 1224 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1225 | trace_bcache_write(s->orig_bio, s->writeback, s->op.bypass); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1226 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1227 | if (s->op.bypass) { |
| 1228 | s->op.cache_bio = s->orig_bio; |
| 1229 | bio_get(s->op.cache_bio); |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1230 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1231 | if (!(bio->bi_rw & REQ_DISCARD) || |
| 1232 | blk_queue_discard(bdev_get_queue(dc->bdev))) |
| 1233 | closure_bio_submit(bio, cl, s->d); |
| 1234 | } else if (s->writeback) { |
Kent Overstreet | 279afba | 2013-06-05 06:21:07 -0700 | [diff] [blame] | 1235 | bch_writeback_add(dc); |
Kent Overstreet | 2fe80d3 | 2013-10-10 17:31:15 -0700 | [diff] [blame] | 1236 | s->op.cache_bio = bio; |
Kent Overstreet | e49c7c3 | 2013-06-26 17:25:38 -0700 | [diff] [blame] | 1237 | |
Kent Overstreet | c0f04d8 | 2013-09-23 23:17:36 -0700 | [diff] [blame] | 1238 | if (bio->bi_rw & REQ_FLUSH) { |
Kent Overstreet | e49c7c3 | 2013-06-26 17:25:38 -0700 | [diff] [blame] | 1239 | /* Also need to send a flush to the backing device */ |
Kent Overstreet | d4eddd4 | 2013-10-22 15:35:50 -0700 | [diff] [blame] | 1240 | struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0, |
Kent Overstreet | c0f04d8 | 2013-09-23 23:17:36 -0700 | [diff] [blame] | 1241 | dc->disk.bio_split); |
Kent Overstreet | e49c7c3 | 2013-06-26 17:25:38 -0700 | [diff] [blame] | 1242 | |
Kent Overstreet | c0f04d8 | 2013-09-23 23:17:36 -0700 | [diff] [blame] | 1243 | flush->bi_rw = WRITE_FLUSH; |
| 1244 | flush->bi_bdev = bio->bi_bdev; |
| 1245 | flush->bi_end_io = request_endio; |
| 1246 | flush->bi_private = cl; |
| 1247 | |
| 1248 | closure_bio_submit(flush, cl, s->d); |
Kent Overstreet | e49c7c3 | 2013-06-26 17:25:38 -0700 | [diff] [blame] | 1249 | } |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1250 | } else { |
| 1251 | s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO, |
| 1252 | dc->disk.bio_split); |
| 1253 | |
| 1254 | closure_bio_submit(bio, cl, s->d); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1255 | } |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1256 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1257 | closure_call(&s->op.cl, bch_data_insert, NULL, cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1258 | continue_at(cl, cached_dev_write_complete, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1259 | } |
| 1260 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1261 | static void cached_dev_nodata(struct closure *cl) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1262 | { |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1263 | struct search *s = container_of(cl, struct search, cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1264 | struct bio *bio = &s->bio.bio; |
| 1265 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1266 | if (s->op.flush_journal) |
| 1267 | bch_journal_meta(s->op.c, cl); |
| 1268 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1269 | /* If it's a flush, we send the flush to the backing device too */ |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1270 | closure_bio_submit(bio, cl, s->d); |
| 1271 | |
| 1272 | continue_at(cl, cached_dev_bio_complete, NULL); |
| 1273 | } |
| 1274 | |
| 1275 | /* Cached devices - read & write stuff */ |
| 1276 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1277 | static void cached_dev_make_request(struct request_queue *q, struct bio *bio) |
| 1278 | { |
| 1279 | struct search *s; |
| 1280 | struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; |
| 1281 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1282 | int cpu, rw = bio_data_dir(bio); |
| 1283 | |
| 1284 | cpu = part_stat_lock(); |
| 1285 | part_stat_inc(cpu, &d->disk->part0, ios[rw]); |
| 1286 | part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio)); |
| 1287 | part_stat_unlock(); |
| 1288 | |
| 1289 | bio->bi_bdev = dc->bdev; |
Kent Overstreet | 2903381 | 2013-04-11 15:14:35 -0700 | [diff] [blame] | 1290 | bio->bi_sector += dc->sb.data_offset; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1291 | |
| 1292 | if (cached_dev_get(dc)) { |
| 1293 | s = search_alloc(bio, d); |
| 1294 | trace_bcache_request_start(s, bio); |
| 1295 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1296 | if (!bio->bi_size) { |
| 1297 | /* |
| 1298 | * can't call bch_journal_meta from under |
| 1299 | * generic_make_request |
| 1300 | */ |
| 1301 | continue_at_nobarrier(&s->cl, |
| 1302 | cached_dev_nodata, |
| 1303 | bcache_wq); |
| 1304 | } else { |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1305 | s->op.bypass = check_should_bypass(dc, s); |
| 1306 | |
| 1307 | if (rw) |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1308 | cached_dev_write(dc, s); |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1309 | else |
Kent Overstreet | cdd972b | 2013-09-10 17:06:17 -0700 | [diff] [blame] | 1310 | cached_dev_read(dc, s); |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1311 | } |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1312 | } else { |
| 1313 | if ((bio->bi_rw & REQ_DISCARD) && |
| 1314 | !blk_queue_discard(bdev_get_queue(dc->bdev))) |
| 1315 | bio_endio(bio, 0); |
| 1316 | else |
| 1317 | bch_generic_make_request(bio, &d->bio_split_hook); |
| 1318 | } |
| 1319 | } |
| 1320 | |
| 1321 | static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode, |
| 1322 | unsigned int cmd, unsigned long arg) |
| 1323 | { |
| 1324 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1325 | return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg); |
| 1326 | } |
| 1327 | |
| 1328 | static int cached_dev_congested(void *data, int bits) |
| 1329 | { |
| 1330 | struct bcache_device *d = data; |
| 1331 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1332 | struct request_queue *q = bdev_get_queue(dc->bdev); |
| 1333 | int ret = 0; |
| 1334 | |
| 1335 | if (bdi_congested(&q->backing_dev_info, bits)) |
| 1336 | return 1; |
| 1337 | |
| 1338 | if (cached_dev_get(dc)) { |
| 1339 | unsigned i; |
| 1340 | struct cache *ca; |
| 1341 | |
| 1342 | for_each_cache(ca, d->c, i) { |
| 1343 | q = bdev_get_queue(ca->bdev); |
| 1344 | ret |= bdi_congested(&q->backing_dev_info, bits); |
| 1345 | } |
| 1346 | |
| 1347 | cached_dev_put(dc); |
| 1348 | } |
| 1349 | |
| 1350 | return ret; |
| 1351 | } |
| 1352 | |
| 1353 | void bch_cached_dev_request_init(struct cached_dev *dc) |
| 1354 | { |
| 1355 | struct gendisk *g = dc->disk.disk; |
| 1356 | |
| 1357 | g->queue->make_request_fn = cached_dev_make_request; |
| 1358 | g->queue->backing_dev_info.congested_fn = cached_dev_congested; |
| 1359 | dc->disk.cache_miss = cached_dev_cache_miss; |
| 1360 | dc->disk.ioctl = cached_dev_ioctl; |
| 1361 | } |
| 1362 | |
| 1363 | /* Flash backed devices */ |
| 1364 | |
| 1365 | static int flash_dev_cache_miss(struct btree *b, struct search *s, |
| 1366 | struct bio *bio, unsigned sectors) |
| 1367 | { |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1368 | struct bio_vec *bv; |
| 1369 | int i; |
| 1370 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1371 | /* Zero fill bio */ |
| 1372 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1373 | bio_for_each_segment(bv, bio, i) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1374 | unsigned j = min(bv->bv_len >> 9, sectors); |
| 1375 | |
| 1376 | void *p = kmap(bv->bv_page); |
| 1377 | memset(p + bv->bv_offset, 0, j << 9); |
| 1378 | kunmap(bv->bv_page); |
| 1379 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1380 | sectors -= j; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1381 | } |
| 1382 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1383 | bio_advance(bio, min(sectors << 9, bio->bi_size)); |
| 1384 | |
| 1385 | if (!bio->bi_size) |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1386 | return MAP_DONE; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1387 | |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1388 | return MAP_CONTINUE; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1389 | } |
| 1390 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1391 | static void flash_dev_nodata(struct closure *cl) |
| 1392 | { |
| 1393 | struct search *s = container_of(cl, struct search, cl); |
| 1394 | |
| 1395 | if (s->op.flush_journal) |
| 1396 | bch_journal_meta(s->op.c, cl); |
| 1397 | |
| 1398 | continue_at(cl, search_free, NULL); |
| 1399 | } |
| 1400 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1401 | static void flash_dev_make_request(struct request_queue *q, struct bio *bio) |
| 1402 | { |
| 1403 | struct search *s; |
| 1404 | struct closure *cl; |
| 1405 | struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; |
| 1406 | int cpu, rw = bio_data_dir(bio); |
| 1407 | |
| 1408 | cpu = part_stat_lock(); |
| 1409 | part_stat_inc(cpu, &d->disk->part0, ios[rw]); |
| 1410 | part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio)); |
| 1411 | part_stat_unlock(); |
| 1412 | |
| 1413 | s = search_alloc(bio, d); |
| 1414 | cl = &s->cl; |
| 1415 | bio = &s->bio.bio; |
| 1416 | |
| 1417 | trace_bcache_request_start(s, bio); |
| 1418 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1419 | if (!bio->bi_size) { |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1420 | /* |
| 1421 | * can't call bch_journal_meta from under |
| 1422 | * generic_make_request |
| 1423 | */ |
| 1424 | continue_at_nobarrier(&s->cl, |
| 1425 | flash_dev_nodata, |
| 1426 | bcache_wq); |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1427 | } else if (rw) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1428 | bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame] | 1429 | &KEY(d->id, bio->bi_sector, 0), |
| 1430 | &KEY(d->id, bio_end_sector(bio), 0)); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1431 | |
Kent Overstreet | 84f0db0 | 2013-07-24 17:24:52 -0700 | [diff] [blame] | 1432 | s->op.bypass = (bio->bi_rw & REQ_DISCARD) != 0; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1433 | s->writeback = true; |
| 1434 | s->op.cache_bio = bio; |
| 1435 | |
Kent Overstreet | a34a8bf | 2013-10-24 17:07:04 -0700 | [diff] [blame] | 1436 | closure_call(&s->op.cl, bch_data_insert, NULL, cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1437 | } else { |
Kent Overstreet | 2c1953e | 2013-07-24 17:41:08 -0700 | [diff] [blame^] | 1438 | closure_call(&s->op.cl, cache_lookup, NULL, cl); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1439 | } |
| 1440 | |
| 1441 | continue_at(cl, search_free, NULL); |
| 1442 | } |
| 1443 | |
| 1444 | static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode, |
| 1445 | unsigned int cmd, unsigned long arg) |
| 1446 | { |
| 1447 | return -ENOTTY; |
| 1448 | } |
| 1449 | |
| 1450 | static int flash_dev_congested(void *data, int bits) |
| 1451 | { |
| 1452 | struct bcache_device *d = data; |
| 1453 | struct request_queue *q; |
| 1454 | struct cache *ca; |
| 1455 | unsigned i; |
| 1456 | int ret = 0; |
| 1457 | |
| 1458 | for_each_cache(ca, d->c, i) { |
| 1459 | q = bdev_get_queue(ca->bdev); |
| 1460 | ret |= bdi_congested(&q->backing_dev_info, bits); |
| 1461 | } |
| 1462 | |
| 1463 | return ret; |
| 1464 | } |
| 1465 | |
| 1466 | void bch_flash_dev_request_init(struct bcache_device *d) |
| 1467 | { |
| 1468 | struct gendisk *g = d->disk; |
| 1469 | |
| 1470 | g->queue->make_request_fn = flash_dev_make_request; |
| 1471 | g->queue->backing_dev_info.congested_fn = flash_dev_congested; |
| 1472 | d->cache_miss = flash_dev_cache_miss; |
| 1473 | d->ioctl = flash_dev_ioctl; |
| 1474 | } |
| 1475 | |
| 1476 | void bch_request_exit(void) |
| 1477 | { |
| 1478 | #ifdef CONFIG_CGROUP_BCACHE |
| 1479 | cgroup_unload_subsys(&bcache_subsys); |
| 1480 | #endif |
| 1481 | if (bch_search_cache) |
| 1482 | kmem_cache_destroy(bch_search_cache); |
| 1483 | } |
| 1484 | |
| 1485 | int __init bch_request_init(void) |
| 1486 | { |
| 1487 | bch_search_cache = KMEM_CACHE(search, 0); |
| 1488 | if (!bch_search_cache) |
| 1489 | return -ENOMEM; |
| 1490 | |
| 1491 | #ifdef CONFIG_CGROUP_BCACHE |
| 1492 | cgroup_load_subsys(&bcache_subsys); |
| 1493 | init_bch_cgroup(&bcache_default_cgroup); |
| 1494 | |
| 1495 | cgroup_add_cftypes(&bcache_subsys, bch_files); |
| 1496 | #endif |
| 1497 | return 0; |
| 1498 | } |