Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Main bcache entry point - handle a read or a write request and decide what to |
| 3 | * do with it; the make_request functions are called by the block layer. |
| 4 | * |
| 5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> |
| 6 | * Copyright 2012 Google, Inc. |
| 7 | */ |
| 8 | |
| 9 | #include "bcache.h" |
| 10 | #include "btree.h" |
| 11 | #include "debug.h" |
| 12 | #include "request.h" |
Kent Overstreet | 279afba | 2013-06-05 06:21:07 -0700 | [diff] [blame] | 13 | #include "writeback.h" |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 14 | |
| 15 | #include <linux/cgroup.h> |
| 16 | #include <linux/module.h> |
| 17 | #include <linux/hash.h> |
| 18 | #include <linux/random.h> |
| 19 | #include "blk-cgroup.h" |
| 20 | |
| 21 | #include <trace/events/bcache.h> |
| 22 | |
| 23 | #define CUTOFF_CACHE_ADD 95 |
| 24 | #define CUTOFF_CACHE_READA 90 |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 25 | |
| 26 | struct kmem_cache *bch_search_cache; |
| 27 | |
| 28 | static void check_should_skip(struct cached_dev *, struct search *); |
| 29 | |
| 30 | /* Cgroup interface */ |
| 31 | |
| 32 | #ifdef CONFIG_CGROUP_BCACHE |
| 33 | static struct bch_cgroup bcache_default_cgroup = { .cache_mode = -1 }; |
| 34 | |
| 35 | static struct bch_cgroup *cgroup_to_bcache(struct cgroup *cgroup) |
| 36 | { |
| 37 | struct cgroup_subsys_state *css; |
| 38 | return cgroup && |
| 39 | (css = cgroup_subsys_state(cgroup, bcache_subsys_id)) |
| 40 | ? container_of(css, struct bch_cgroup, css) |
| 41 | : &bcache_default_cgroup; |
| 42 | } |
| 43 | |
| 44 | struct bch_cgroup *bch_bio_to_cgroup(struct bio *bio) |
| 45 | { |
| 46 | struct cgroup_subsys_state *css = bio->bi_css |
| 47 | ? cgroup_subsys_state(bio->bi_css->cgroup, bcache_subsys_id) |
| 48 | : task_subsys_state(current, bcache_subsys_id); |
| 49 | |
| 50 | return css |
| 51 | ? container_of(css, struct bch_cgroup, css) |
| 52 | : &bcache_default_cgroup; |
| 53 | } |
| 54 | |
| 55 | static ssize_t cache_mode_read(struct cgroup *cgrp, struct cftype *cft, |
| 56 | struct file *file, |
| 57 | char __user *buf, size_t nbytes, loff_t *ppos) |
| 58 | { |
| 59 | char tmp[1024]; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 60 | int len = bch_snprint_string_list(tmp, PAGE_SIZE, bch_cache_modes, |
| 61 | cgroup_to_bcache(cgrp)->cache_mode + 1); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 62 | |
| 63 | if (len < 0) |
| 64 | return len; |
| 65 | |
| 66 | return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); |
| 67 | } |
| 68 | |
| 69 | static int cache_mode_write(struct cgroup *cgrp, struct cftype *cft, |
| 70 | const char *buf) |
| 71 | { |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 72 | int v = bch_read_string_list(buf, bch_cache_modes); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 73 | if (v < 0) |
| 74 | return v; |
| 75 | |
| 76 | cgroup_to_bcache(cgrp)->cache_mode = v - 1; |
| 77 | return 0; |
| 78 | } |
| 79 | |
| 80 | static u64 bch_verify_read(struct cgroup *cgrp, struct cftype *cft) |
| 81 | { |
| 82 | return cgroup_to_bcache(cgrp)->verify; |
| 83 | } |
| 84 | |
| 85 | static int bch_verify_write(struct cgroup *cgrp, struct cftype *cft, u64 val) |
| 86 | { |
| 87 | cgroup_to_bcache(cgrp)->verify = val; |
| 88 | return 0; |
| 89 | } |
| 90 | |
| 91 | static u64 bch_cache_hits_read(struct cgroup *cgrp, struct cftype *cft) |
| 92 | { |
| 93 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 94 | return atomic_read(&bcachecg->stats.cache_hits); |
| 95 | } |
| 96 | |
| 97 | static u64 bch_cache_misses_read(struct cgroup *cgrp, struct cftype *cft) |
| 98 | { |
| 99 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 100 | return atomic_read(&bcachecg->stats.cache_misses); |
| 101 | } |
| 102 | |
| 103 | static u64 bch_cache_bypass_hits_read(struct cgroup *cgrp, |
| 104 | struct cftype *cft) |
| 105 | { |
| 106 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 107 | return atomic_read(&bcachecg->stats.cache_bypass_hits); |
| 108 | } |
| 109 | |
| 110 | static u64 bch_cache_bypass_misses_read(struct cgroup *cgrp, |
| 111 | struct cftype *cft) |
| 112 | { |
| 113 | struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp); |
| 114 | return atomic_read(&bcachecg->stats.cache_bypass_misses); |
| 115 | } |
| 116 | |
| 117 | static struct cftype bch_files[] = { |
| 118 | { |
| 119 | .name = "cache_mode", |
| 120 | .read = cache_mode_read, |
| 121 | .write_string = cache_mode_write, |
| 122 | }, |
| 123 | { |
| 124 | .name = "verify", |
| 125 | .read_u64 = bch_verify_read, |
| 126 | .write_u64 = bch_verify_write, |
| 127 | }, |
| 128 | { |
| 129 | .name = "cache_hits", |
| 130 | .read_u64 = bch_cache_hits_read, |
| 131 | }, |
| 132 | { |
| 133 | .name = "cache_misses", |
| 134 | .read_u64 = bch_cache_misses_read, |
| 135 | }, |
| 136 | { |
| 137 | .name = "cache_bypass_hits", |
| 138 | .read_u64 = bch_cache_bypass_hits_read, |
| 139 | }, |
| 140 | { |
| 141 | .name = "cache_bypass_misses", |
| 142 | .read_u64 = bch_cache_bypass_misses_read, |
| 143 | }, |
| 144 | { } /* terminate */ |
| 145 | }; |
| 146 | |
| 147 | static void init_bch_cgroup(struct bch_cgroup *cg) |
| 148 | { |
| 149 | cg->cache_mode = -1; |
| 150 | } |
| 151 | |
| 152 | static struct cgroup_subsys_state *bcachecg_create(struct cgroup *cgroup) |
| 153 | { |
| 154 | struct bch_cgroup *cg; |
| 155 | |
| 156 | cg = kzalloc(sizeof(*cg), GFP_KERNEL); |
| 157 | if (!cg) |
| 158 | return ERR_PTR(-ENOMEM); |
| 159 | init_bch_cgroup(cg); |
| 160 | return &cg->css; |
| 161 | } |
| 162 | |
| 163 | static void bcachecg_destroy(struct cgroup *cgroup) |
| 164 | { |
| 165 | struct bch_cgroup *cg = cgroup_to_bcache(cgroup); |
| 166 | free_css_id(&bcache_subsys, &cg->css); |
| 167 | kfree(cg); |
| 168 | } |
| 169 | |
| 170 | struct cgroup_subsys bcache_subsys = { |
| 171 | .create = bcachecg_create, |
| 172 | .destroy = bcachecg_destroy, |
| 173 | .subsys_id = bcache_subsys_id, |
| 174 | .name = "bcache", |
| 175 | .module = THIS_MODULE, |
| 176 | }; |
| 177 | EXPORT_SYMBOL_GPL(bcache_subsys); |
| 178 | #endif |
| 179 | |
| 180 | static unsigned cache_mode(struct cached_dev *dc, struct bio *bio) |
| 181 | { |
| 182 | #ifdef CONFIG_CGROUP_BCACHE |
| 183 | int r = bch_bio_to_cgroup(bio)->cache_mode; |
| 184 | if (r >= 0) |
| 185 | return r; |
| 186 | #endif |
| 187 | return BDEV_CACHE_MODE(&dc->sb); |
| 188 | } |
| 189 | |
| 190 | static bool verify(struct cached_dev *dc, struct bio *bio) |
| 191 | { |
| 192 | #ifdef CONFIG_CGROUP_BCACHE |
| 193 | if (bch_bio_to_cgroup(bio)->verify) |
| 194 | return true; |
| 195 | #endif |
| 196 | return dc->verify; |
| 197 | } |
| 198 | |
| 199 | static void bio_csum(struct bio *bio, struct bkey *k) |
| 200 | { |
| 201 | struct bio_vec *bv; |
| 202 | uint64_t csum = 0; |
| 203 | int i; |
| 204 | |
| 205 | bio_for_each_segment(bv, bio, i) { |
| 206 | void *d = kmap(bv->bv_page) + bv->bv_offset; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 207 | csum = bch_crc64_update(csum, d, bv->bv_len); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 208 | kunmap(bv->bv_page); |
| 209 | } |
| 210 | |
| 211 | k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1); |
| 212 | } |
| 213 | |
| 214 | /* Insert data into cache */ |
| 215 | |
| 216 | static void bio_invalidate(struct closure *cl) |
| 217 | { |
| 218 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 219 | struct bio *bio = op->cache_bio; |
| 220 | |
| 221 | pr_debug("invalidating %i sectors from %llu", |
| 222 | bio_sectors(bio), (uint64_t) bio->bi_sector); |
| 223 | |
| 224 | while (bio_sectors(bio)) { |
| 225 | unsigned len = min(bio_sectors(bio), 1U << 14); |
| 226 | |
| 227 | if (bch_keylist_realloc(&op->keys, 0, op->c)) |
| 228 | goto out; |
| 229 | |
| 230 | bio->bi_sector += len; |
| 231 | bio->bi_size -= len << 9; |
| 232 | |
| 233 | bch_keylist_add(&op->keys, |
| 234 | &KEY(op->inode, bio->bi_sector, len)); |
| 235 | } |
| 236 | |
| 237 | op->insert_data_done = true; |
| 238 | bio_put(bio); |
| 239 | out: |
| 240 | continue_at(cl, bch_journal, bcache_wq); |
| 241 | } |
| 242 | |
| 243 | struct open_bucket { |
| 244 | struct list_head list; |
| 245 | struct task_struct *last; |
| 246 | unsigned sectors_free; |
| 247 | BKEY_PADDED(key); |
| 248 | }; |
| 249 | |
| 250 | void bch_open_buckets_free(struct cache_set *c) |
| 251 | { |
| 252 | struct open_bucket *b; |
| 253 | |
| 254 | while (!list_empty(&c->data_buckets)) { |
| 255 | b = list_first_entry(&c->data_buckets, |
| 256 | struct open_bucket, list); |
| 257 | list_del(&b->list); |
| 258 | kfree(b); |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | int bch_open_buckets_alloc(struct cache_set *c) |
| 263 | { |
| 264 | int i; |
| 265 | |
| 266 | spin_lock_init(&c->data_bucket_lock); |
| 267 | |
| 268 | for (i = 0; i < 6; i++) { |
| 269 | struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL); |
| 270 | if (!b) |
| 271 | return -ENOMEM; |
| 272 | |
| 273 | list_add(&b->list, &c->data_buckets); |
| 274 | } |
| 275 | |
| 276 | return 0; |
| 277 | } |
| 278 | |
| 279 | /* |
| 280 | * We keep multiple buckets open for writes, and try to segregate different |
| 281 | * write streams for better cache utilization: first we look for a bucket where |
| 282 | * the last write to it was sequential with the current write, and failing that |
| 283 | * we look for a bucket that was last used by the same task. |
| 284 | * |
| 285 | * The ideas is if you've got multiple tasks pulling data into the cache at the |
| 286 | * same time, you'll get better cache utilization if you try to segregate their |
| 287 | * data and preserve locality. |
| 288 | * |
| 289 | * For example, say you've starting Firefox at the same time you're copying a |
| 290 | * bunch of files. Firefox will likely end up being fairly hot and stay in the |
| 291 | * cache awhile, but the data you copied might not be; if you wrote all that |
| 292 | * data to the same buckets it'd get invalidated at the same time. |
| 293 | * |
| 294 | * Both of those tasks will be doing fairly random IO so we can't rely on |
| 295 | * detecting sequential IO to segregate their data, but going off of the task |
| 296 | * should be a sane heuristic. |
| 297 | */ |
| 298 | static struct open_bucket *pick_data_bucket(struct cache_set *c, |
| 299 | const struct bkey *search, |
| 300 | struct task_struct *task, |
| 301 | struct bkey *alloc) |
| 302 | { |
| 303 | struct open_bucket *ret, *ret_task = NULL; |
| 304 | |
| 305 | list_for_each_entry_reverse(ret, &c->data_buckets, list) |
| 306 | if (!bkey_cmp(&ret->key, search)) |
| 307 | goto found; |
| 308 | else if (ret->last == task) |
| 309 | ret_task = ret; |
| 310 | |
| 311 | ret = ret_task ?: list_first_entry(&c->data_buckets, |
| 312 | struct open_bucket, list); |
| 313 | found: |
| 314 | if (!ret->sectors_free && KEY_PTRS(alloc)) { |
| 315 | ret->sectors_free = c->sb.bucket_size; |
| 316 | bkey_copy(&ret->key, alloc); |
| 317 | bkey_init(alloc); |
| 318 | } |
| 319 | |
| 320 | if (!ret->sectors_free) |
| 321 | ret = NULL; |
| 322 | |
| 323 | return ret; |
| 324 | } |
| 325 | |
| 326 | /* |
| 327 | * Allocates some space in the cache to write to, and k to point to the newly |
| 328 | * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the |
| 329 | * end of the newly allocated space). |
| 330 | * |
| 331 | * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many |
| 332 | * sectors were actually allocated. |
| 333 | * |
| 334 | * If s->writeback is true, will not fail. |
| 335 | */ |
| 336 | static bool bch_alloc_sectors(struct bkey *k, unsigned sectors, |
| 337 | struct search *s) |
| 338 | { |
| 339 | struct cache_set *c = s->op.c; |
| 340 | struct open_bucket *b; |
| 341 | BKEY_PADDED(key) alloc; |
| 342 | struct closure cl, *w = NULL; |
| 343 | unsigned i; |
| 344 | |
| 345 | if (s->writeback) { |
| 346 | closure_init_stack(&cl); |
| 347 | w = &cl; |
| 348 | } |
| 349 | |
| 350 | /* |
| 351 | * We might have to allocate a new bucket, which we can't do with a |
| 352 | * spinlock held. So if we have to allocate, we drop the lock, allocate |
| 353 | * and then retry. KEY_PTRS() indicates whether alloc points to |
| 354 | * allocated bucket(s). |
| 355 | */ |
| 356 | |
| 357 | bkey_init(&alloc.key); |
| 358 | spin_lock(&c->data_bucket_lock); |
| 359 | |
| 360 | while (!(b = pick_data_bucket(c, k, s->task, &alloc.key))) { |
| 361 | unsigned watermark = s->op.write_prio |
| 362 | ? WATERMARK_MOVINGGC |
| 363 | : WATERMARK_NONE; |
| 364 | |
| 365 | spin_unlock(&c->data_bucket_lock); |
| 366 | |
| 367 | if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, w)) |
| 368 | return false; |
| 369 | |
| 370 | spin_lock(&c->data_bucket_lock); |
| 371 | } |
| 372 | |
| 373 | /* |
| 374 | * If we had to allocate, we might race and not need to allocate the |
| 375 | * second time we call find_data_bucket(). If we allocated a bucket but |
| 376 | * didn't use it, drop the refcount bch_bucket_alloc_set() took: |
| 377 | */ |
| 378 | if (KEY_PTRS(&alloc.key)) |
| 379 | __bkey_put(c, &alloc.key); |
| 380 | |
| 381 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 382 | EBUG_ON(ptr_stale(c, &b->key, i)); |
| 383 | |
| 384 | /* Set up the pointer to the space we're allocating: */ |
| 385 | |
| 386 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 387 | k->ptr[i] = b->key.ptr[i]; |
| 388 | |
| 389 | sectors = min(sectors, b->sectors_free); |
| 390 | |
| 391 | SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors); |
| 392 | SET_KEY_SIZE(k, sectors); |
| 393 | SET_KEY_PTRS(k, KEY_PTRS(&b->key)); |
| 394 | |
| 395 | /* |
| 396 | * Move b to the end of the lru, and keep track of what this bucket was |
| 397 | * last used for: |
| 398 | */ |
| 399 | list_move_tail(&b->list, &c->data_buckets); |
| 400 | bkey_copy_key(&b->key, k); |
| 401 | b->last = s->task; |
| 402 | |
| 403 | b->sectors_free -= sectors; |
| 404 | |
| 405 | for (i = 0; i < KEY_PTRS(&b->key); i++) { |
| 406 | SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors); |
| 407 | |
| 408 | atomic_long_add(sectors, |
| 409 | &PTR_CACHE(c, &b->key, i)->sectors_written); |
| 410 | } |
| 411 | |
| 412 | if (b->sectors_free < c->sb.block_size) |
| 413 | b->sectors_free = 0; |
| 414 | |
| 415 | /* |
| 416 | * k takes refcounts on the buckets it points to until it's inserted |
| 417 | * into the btree, but if we're done with this bucket we just transfer |
| 418 | * get_data_bucket()'s refcount. |
| 419 | */ |
| 420 | if (b->sectors_free) |
| 421 | for (i = 0; i < KEY_PTRS(&b->key); i++) |
| 422 | atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin); |
| 423 | |
| 424 | spin_unlock(&c->data_bucket_lock); |
| 425 | return true; |
| 426 | } |
| 427 | |
| 428 | static void bch_insert_data_error(struct closure *cl) |
| 429 | { |
| 430 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 431 | |
| 432 | /* |
| 433 | * Our data write just errored, which means we've got a bunch of keys to |
| 434 | * insert that point to data that wasn't succesfully written. |
| 435 | * |
| 436 | * We don't have to insert those keys but we still have to invalidate |
| 437 | * that region of the cache - so, if we just strip off all the pointers |
| 438 | * from the keys we'll accomplish just that. |
| 439 | */ |
| 440 | |
| 441 | struct bkey *src = op->keys.bottom, *dst = op->keys.bottom; |
| 442 | |
| 443 | while (src != op->keys.top) { |
| 444 | struct bkey *n = bkey_next(src); |
| 445 | |
| 446 | SET_KEY_PTRS(src, 0); |
| 447 | bkey_copy(dst, src); |
| 448 | |
| 449 | dst = bkey_next(dst); |
| 450 | src = n; |
| 451 | } |
| 452 | |
| 453 | op->keys.top = dst; |
| 454 | |
| 455 | bch_journal(cl); |
| 456 | } |
| 457 | |
| 458 | static void bch_insert_data_endio(struct bio *bio, int error) |
| 459 | { |
| 460 | struct closure *cl = bio->bi_private; |
| 461 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 462 | struct search *s = container_of(op, struct search, op); |
| 463 | |
| 464 | if (error) { |
| 465 | /* TODO: We could try to recover from this. */ |
| 466 | if (s->writeback) |
| 467 | s->error = error; |
| 468 | else if (s->write) |
| 469 | set_closure_fn(cl, bch_insert_data_error, bcache_wq); |
| 470 | else |
| 471 | set_closure_fn(cl, NULL, NULL); |
| 472 | } |
| 473 | |
| 474 | bch_bbio_endio(op->c, bio, error, "writing data to cache"); |
| 475 | } |
| 476 | |
| 477 | static void bch_insert_data_loop(struct closure *cl) |
| 478 | { |
| 479 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 480 | struct search *s = container_of(op, struct search, op); |
| 481 | struct bio *bio = op->cache_bio, *n; |
| 482 | |
| 483 | if (op->skip) |
| 484 | return bio_invalidate(cl); |
| 485 | |
| 486 | if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) { |
| 487 | set_gc_sectors(op->c); |
| 488 | bch_queue_gc(op->c); |
| 489 | } |
| 490 | |
| 491 | do { |
| 492 | unsigned i; |
| 493 | struct bkey *k; |
| 494 | struct bio_set *split = s->d |
| 495 | ? s->d->bio_split : op->c->bio_split; |
| 496 | |
| 497 | /* 1 for the device pointer and 1 for the chksum */ |
| 498 | if (bch_keylist_realloc(&op->keys, |
| 499 | 1 + (op->csum ? 1 : 0), |
| 500 | op->c)) |
| 501 | continue_at(cl, bch_journal, bcache_wq); |
| 502 | |
| 503 | k = op->keys.top; |
| 504 | bkey_init(k); |
| 505 | SET_KEY_INODE(k, op->inode); |
| 506 | SET_KEY_OFFSET(k, bio->bi_sector); |
| 507 | |
| 508 | if (!bch_alloc_sectors(k, bio_sectors(bio), s)) |
| 509 | goto err; |
| 510 | |
| 511 | n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 512 | |
| 513 | n->bi_end_io = bch_insert_data_endio; |
| 514 | n->bi_private = cl; |
| 515 | |
| 516 | if (s->writeback) { |
| 517 | SET_KEY_DIRTY(k, true); |
| 518 | |
| 519 | for (i = 0; i < KEY_PTRS(k); i++) |
| 520 | SET_GC_MARK(PTR_BUCKET(op->c, k, i), |
| 521 | GC_MARK_DIRTY); |
| 522 | } |
| 523 | |
| 524 | SET_KEY_CSUM(k, op->csum); |
| 525 | if (KEY_CSUM(k)) |
| 526 | bio_csum(n, k); |
| 527 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 528 | trace_bcache_cache_insert(k); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 529 | bch_keylist_push(&op->keys); |
| 530 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 531 | n->bi_rw |= REQ_WRITE; |
| 532 | bch_submit_bbio(n, op->c, k, 0); |
| 533 | } while (n != bio); |
| 534 | |
| 535 | op->insert_data_done = true; |
| 536 | continue_at(cl, bch_journal, bcache_wq); |
| 537 | err: |
| 538 | /* bch_alloc_sectors() blocks if s->writeback = true */ |
| 539 | BUG_ON(s->writeback); |
| 540 | |
| 541 | /* |
| 542 | * But if it's not a writeback write we'd rather just bail out if |
| 543 | * there aren't any buckets ready to write to - it might take awhile and |
| 544 | * we might be starving btree writes for gc or something. |
| 545 | */ |
| 546 | |
| 547 | if (s->write) { |
| 548 | /* |
| 549 | * Writethrough write: We can't complete the write until we've |
| 550 | * updated the index. But we don't want to delay the write while |
| 551 | * we wait for buckets to be freed up, so just invalidate the |
| 552 | * rest of the write. |
| 553 | */ |
| 554 | op->skip = true; |
| 555 | return bio_invalidate(cl); |
| 556 | } else { |
| 557 | /* |
| 558 | * From a cache miss, we can just insert the keys for the data |
| 559 | * we have written or bail out if we didn't do anything. |
| 560 | */ |
| 561 | op->insert_data_done = true; |
| 562 | bio_put(bio); |
| 563 | |
| 564 | if (!bch_keylist_empty(&op->keys)) |
| 565 | continue_at(cl, bch_journal, bcache_wq); |
| 566 | else |
| 567 | closure_return(cl); |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | /** |
| 572 | * bch_insert_data - stick some data in the cache |
| 573 | * |
| 574 | * This is the starting point for any data to end up in a cache device; it could |
| 575 | * be from a normal write, or a writeback write, or a write to a flash only |
| 576 | * volume - it's also used by the moving garbage collector to compact data in |
| 577 | * mostly empty buckets. |
| 578 | * |
| 579 | * It first writes the data to the cache, creating a list of keys to be inserted |
| 580 | * (if the data had to be fragmented there will be multiple keys); after the |
| 581 | * data is written it calls bch_journal, and after the keys have been added to |
| 582 | * the next journal write they're inserted into the btree. |
| 583 | * |
| 584 | * It inserts the data in op->cache_bio; bi_sector is used for the key offset, |
| 585 | * and op->inode is used for the key inode. |
| 586 | * |
| 587 | * If op->skip is true, instead of inserting the data it invalidates the region |
| 588 | * of the cache represented by op->cache_bio and op->inode. |
| 589 | */ |
| 590 | void bch_insert_data(struct closure *cl) |
| 591 | { |
| 592 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 593 | |
| 594 | bch_keylist_init(&op->keys); |
| 595 | bio_get(op->cache_bio); |
| 596 | bch_insert_data_loop(cl); |
| 597 | } |
| 598 | |
| 599 | void bch_btree_insert_async(struct closure *cl) |
| 600 | { |
| 601 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 602 | struct search *s = container_of(op, struct search, op); |
| 603 | |
| 604 | if (bch_btree_insert(op, op->c)) { |
| 605 | s->error = -ENOMEM; |
| 606 | op->insert_data_done = true; |
| 607 | } |
| 608 | |
| 609 | if (op->insert_data_done) { |
| 610 | bch_keylist_free(&op->keys); |
| 611 | closure_return(cl); |
| 612 | } else |
| 613 | continue_at(cl, bch_insert_data_loop, bcache_wq); |
| 614 | } |
| 615 | |
| 616 | /* Common code for the make_request functions */ |
| 617 | |
| 618 | static void request_endio(struct bio *bio, int error) |
| 619 | { |
| 620 | struct closure *cl = bio->bi_private; |
| 621 | |
| 622 | if (error) { |
| 623 | struct search *s = container_of(cl, struct search, cl); |
| 624 | s->error = error; |
| 625 | /* Only cache read errors are recoverable */ |
| 626 | s->recoverable = false; |
| 627 | } |
| 628 | |
| 629 | bio_put(bio); |
| 630 | closure_put(cl); |
| 631 | } |
| 632 | |
| 633 | void bch_cache_read_endio(struct bio *bio, int error) |
| 634 | { |
| 635 | struct bbio *b = container_of(bio, struct bbio, bio); |
| 636 | struct closure *cl = bio->bi_private; |
| 637 | struct search *s = container_of(cl, struct search, cl); |
| 638 | |
| 639 | /* |
| 640 | * If the bucket was reused while our bio was in flight, we might have |
| 641 | * read the wrong data. Set s->error but not error so it doesn't get |
| 642 | * counted against the cache device, but we'll still reread the data |
| 643 | * from the backing device. |
| 644 | */ |
| 645 | |
| 646 | if (error) |
| 647 | s->error = error; |
| 648 | else if (ptr_stale(s->op.c, &b->key, 0)) { |
| 649 | atomic_long_inc(&s->op.c->cache_read_races); |
| 650 | s->error = -EINTR; |
| 651 | } |
| 652 | |
| 653 | bch_bbio_endio(s->op.c, bio, error, "reading from cache"); |
| 654 | } |
| 655 | |
| 656 | static void bio_complete(struct search *s) |
| 657 | { |
| 658 | if (s->orig_bio) { |
| 659 | int cpu, rw = bio_data_dir(s->orig_bio); |
| 660 | unsigned long duration = jiffies - s->start_time; |
| 661 | |
| 662 | cpu = part_stat_lock(); |
| 663 | part_round_stats(cpu, &s->d->disk->part0); |
| 664 | part_stat_add(cpu, &s->d->disk->part0, ticks[rw], duration); |
| 665 | part_stat_unlock(); |
| 666 | |
| 667 | trace_bcache_request_end(s, s->orig_bio); |
| 668 | bio_endio(s->orig_bio, s->error); |
| 669 | s->orig_bio = NULL; |
| 670 | } |
| 671 | } |
| 672 | |
| 673 | static void do_bio_hook(struct search *s) |
| 674 | { |
| 675 | struct bio *bio = &s->bio.bio; |
| 676 | memcpy(bio, s->orig_bio, sizeof(struct bio)); |
| 677 | |
| 678 | bio->bi_end_io = request_endio; |
| 679 | bio->bi_private = &s->cl; |
| 680 | atomic_set(&bio->bi_cnt, 3); |
| 681 | } |
| 682 | |
| 683 | static void search_free(struct closure *cl) |
| 684 | { |
| 685 | struct search *s = container_of(cl, struct search, cl); |
| 686 | bio_complete(s); |
| 687 | |
| 688 | if (s->op.cache_bio) |
| 689 | bio_put(s->op.cache_bio); |
| 690 | |
| 691 | if (s->unaligned_bvec) |
| 692 | mempool_free(s->bio.bio.bi_io_vec, s->d->unaligned_bvec); |
| 693 | |
| 694 | closure_debug_destroy(cl); |
| 695 | mempool_free(s, s->d->c->search); |
| 696 | } |
| 697 | |
| 698 | static struct search *search_alloc(struct bio *bio, struct bcache_device *d) |
| 699 | { |
| 700 | struct bio_vec *bv; |
| 701 | struct search *s = mempool_alloc(d->c->search, GFP_NOIO); |
| 702 | memset(s, 0, offsetof(struct search, op.keys)); |
| 703 | |
| 704 | __closure_init(&s->cl, NULL); |
| 705 | |
| 706 | s->op.inode = d->id; |
| 707 | s->op.c = d->c; |
| 708 | s->d = d; |
| 709 | s->op.lock = -1; |
| 710 | s->task = current; |
| 711 | s->orig_bio = bio; |
| 712 | s->write = (bio->bi_rw & REQ_WRITE) != 0; |
| 713 | s->op.flush_journal = (bio->bi_rw & REQ_FLUSH) != 0; |
| 714 | s->op.skip = (bio->bi_rw & REQ_DISCARD) != 0; |
| 715 | s->recoverable = 1; |
| 716 | s->start_time = jiffies; |
| 717 | do_bio_hook(s); |
| 718 | |
| 719 | if (bio->bi_size != bio_segments(bio) * PAGE_SIZE) { |
| 720 | bv = mempool_alloc(d->unaligned_bvec, GFP_NOIO); |
| 721 | memcpy(bv, bio_iovec(bio), |
| 722 | sizeof(struct bio_vec) * bio_segments(bio)); |
| 723 | |
| 724 | s->bio.bio.bi_io_vec = bv; |
| 725 | s->unaligned_bvec = 1; |
| 726 | } |
| 727 | |
| 728 | return s; |
| 729 | } |
| 730 | |
| 731 | static void btree_read_async(struct closure *cl) |
| 732 | { |
| 733 | struct btree_op *op = container_of(cl, struct btree_op, cl); |
| 734 | |
| 735 | int ret = btree_root(search_recurse, op->c, op); |
| 736 | |
| 737 | if (ret == -EAGAIN) |
| 738 | continue_at(cl, btree_read_async, bcache_wq); |
| 739 | |
| 740 | closure_return(cl); |
| 741 | } |
| 742 | |
| 743 | /* Cached devices */ |
| 744 | |
| 745 | static void cached_dev_bio_complete(struct closure *cl) |
| 746 | { |
| 747 | struct search *s = container_of(cl, struct search, cl); |
| 748 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 749 | |
| 750 | search_free(cl); |
| 751 | cached_dev_put(dc); |
| 752 | } |
| 753 | |
| 754 | /* Process reads */ |
| 755 | |
| 756 | static void cached_dev_read_complete(struct closure *cl) |
| 757 | { |
| 758 | struct search *s = container_of(cl, struct search, cl); |
| 759 | |
| 760 | if (s->op.insert_collision) |
| 761 | bch_mark_cache_miss_collision(s); |
| 762 | |
| 763 | if (s->op.cache_bio) { |
| 764 | int i; |
| 765 | struct bio_vec *bv; |
| 766 | |
| 767 | __bio_for_each_segment(bv, s->op.cache_bio, i, 0) |
| 768 | __free_page(bv->bv_page); |
| 769 | } |
| 770 | |
| 771 | cached_dev_bio_complete(cl); |
| 772 | } |
| 773 | |
| 774 | static void request_read_error(struct closure *cl) |
| 775 | { |
| 776 | struct search *s = container_of(cl, struct search, cl); |
| 777 | struct bio_vec *bv; |
| 778 | int i; |
| 779 | |
| 780 | if (s->recoverable) { |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 781 | /* Retry from the backing device: */ |
| 782 | trace_bcache_read_retry(s->orig_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 783 | |
| 784 | s->error = 0; |
| 785 | bv = s->bio.bio.bi_io_vec; |
| 786 | do_bio_hook(s); |
| 787 | s->bio.bio.bi_io_vec = bv; |
| 788 | |
| 789 | if (!s->unaligned_bvec) |
| 790 | bio_for_each_segment(bv, s->orig_bio, i) |
| 791 | bv->bv_offset = 0, bv->bv_len = PAGE_SIZE; |
| 792 | else |
| 793 | memcpy(s->bio.bio.bi_io_vec, |
| 794 | bio_iovec(s->orig_bio), |
| 795 | sizeof(struct bio_vec) * |
| 796 | bio_segments(s->orig_bio)); |
| 797 | |
| 798 | /* XXX: invalidate cache */ |
| 799 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 800 | closure_bio_submit(&s->bio.bio, &s->cl, s->d); |
| 801 | } |
| 802 | |
| 803 | continue_at(cl, cached_dev_read_complete, NULL); |
| 804 | } |
| 805 | |
| 806 | static void request_read_done(struct closure *cl) |
| 807 | { |
| 808 | struct search *s = container_of(cl, struct search, cl); |
| 809 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 810 | |
| 811 | /* |
| 812 | * s->cache_bio != NULL implies that we had a cache miss; cache_bio now |
| 813 | * contains data ready to be inserted into the cache. |
| 814 | * |
| 815 | * First, we copy the data we just read from cache_bio's bounce buffers |
| 816 | * to the buffers the original bio pointed to: |
| 817 | */ |
| 818 | |
| 819 | if (s->op.cache_bio) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 820 | bio_reset(s->op.cache_bio); |
| 821 | s->op.cache_bio->bi_sector = s->cache_miss->bi_sector; |
| 822 | s->op.cache_bio->bi_bdev = s->cache_miss->bi_bdev; |
| 823 | s->op.cache_bio->bi_size = s->cache_bio_sectors << 9; |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 824 | bch_bio_map(s->op.cache_bio, NULL); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 825 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 826 | bio_copy_data(s->cache_miss, s->op.cache_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 827 | |
| 828 | bio_put(s->cache_miss); |
| 829 | s->cache_miss = NULL; |
| 830 | } |
| 831 | |
| 832 | if (verify(dc, &s->bio.bio) && s->recoverable) |
| 833 | bch_data_verify(s); |
| 834 | |
| 835 | bio_complete(s); |
| 836 | |
| 837 | if (s->op.cache_bio && |
| 838 | !test_bit(CACHE_SET_STOPPING, &s->op.c->flags)) { |
| 839 | s->op.type = BTREE_REPLACE; |
| 840 | closure_call(&s->op.cl, bch_insert_data, NULL, cl); |
| 841 | } |
| 842 | |
| 843 | continue_at(cl, cached_dev_read_complete, NULL); |
| 844 | } |
| 845 | |
| 846 | static void request_read_done_bh(struct closure *cl) |
| 847 | { |
| 848 | struct search *s = container_of(cl, struct search, cl); |
| 849 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 850 | |
| 851 | bch_mark_cache_accounting(s, !s->cache_miss, s->op.skip); |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 852 | trace_bcache_read(s->orig_bio, !s->cache_miss, s->op.skip); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 853 | |
| 854 | if (s->error) |
| 855 | continue_at_nobarrier(cl, request_read_error, bcache_wq); |
| 856 | else if (s->op.cache_bio || verify(dc, &s->bio.bio)) |
| 857 | continue_at_nobarrier(cl, request_read_done, bcache_wq); |
| 858 | else |
| 859 | continue_at_nobarrier(cl, cached_dev_read_complete, NULL); |
| 860 | } |
| 861 | |
| 862 | static int cached_dev_cache_miss(struct btree *b, struct search *s, |
| 863 | struct bio *bio, unsigned sectors) |
| 864 | { |
| 865 | int ret = 0; |
| 866 | unsigned reada; |
| 867 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 868 | struct bio *miss; |
| 869 | |
| 870 | miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 871 | if (miss == bio) |
| 872 | s->op.lookup_done = true; |
| 873 | |
| 874 | miss->bi_end_io = request_endio; |
| 875 | miss->bi_private = &s->cl; |
| 876 | |
| 877 | if (s->cache_miss || s->op.skip) |
| 878 | goto out_submit; |
| 879 | |
| 880 | if (miss != bio || |
| 881 | (bio->bi_rw & REQ_RAHEAD) || |
| 882 | (bio->bi_rw & REQ_META) || |
| 883 | s->op.c->gc_stats.in_use >= CUTOFF_CACHE_READA) |
| 884 | reada = 0; |
| 885 | else { |
| 886 | reada = min(dc->readahead >> 9, |
| 887 | sectors - bio_sectors(miss)); |
| 888 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 889 | if (bio_end_sector(miss) + reada > bdev_sectors(miss->bi_bdev)) |
| 890 | reada = bdev_sectors(miss->bi_bdev) - |
| 891 | bio_end_sector(miss); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 892 | } |
| 893 | |
| 894 | s->cache_bio_sectors = bio_sectors(miss) + reada; |
| 895 | s->op.cache_bio = bio_alloc_bioset(GFP_NOWAIT, |
| 896 | DIV_ROUND_UP(s->cache_bio_sectors, PAGE_SECTORS), |
| 897 | dc->disk.bio_split); |
| 898 | |
| 899 | if (!s->op.cache_bio) |
| 900 | goto out_submit; |
| 901 | |
| 902 | s->op.cache_bio->bi_sector = miss->bi_sector; |
| 903 | s->op.cache_bio->bi_bdev = miss->bi_bdev; |
| 904 | s->op.cache_bio->bi_size = s->cache_bio_sectors << 9; |
| 905 | |
| 906 | s->op.cache_bio->bi_end_io = request_endio; |
| 907 | s->op.cache_bio->bi_private = &s->cl; |
| 908 | |
| 909 | /* btree_search_recurse()'s btree iterator is no good anymore */ |
| 910 | ret = -EINTR; |
| 911 | if (!bch_btree_insert_check_key(b, &s->op, s->op.cache_bio)) |
| 912 | goto out_put; |
| 913 | |
Kent Overstreet | 169ef1c | 2013-03-28 12:50:55 -0600 | [diff] [blame] | 914 | bch_bio_map(s->op.cache_bio, NULL); |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 915 | if (bio_alloc_pages(s->op.cache_bio, __GFP_NOWARN|GFP_NOIO)) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 916 | goto out_put; |
| 917 | |
| 918 | s->cache_miss = miss; |
| 919 | bio_get(s->op.cache_bio); |
| 920 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 921 | closure_bio_submit(s->op.cache_bio, &s->cl, s->d); |
| 922 | |
| 923 | return ret; |
| 924 | out_put: |
| 925 | bio_put(s->op.cache_bio); |
| 926 | s->op.cache_bio = NULL; |
| 927 | out_submit: |
| 928 | closure_bio_submit(miss, &s->cl, s->d); |
| 929 | return ret; |
| 930 | } |
| 931 | |
| 932 | static void request_read(struct cached_dev *dc, struct search *s) |
| 933 | { |
| 934 | struct closure *cl = &s->cl; |
| 935 | |
| 936 | check_should_skip(dc, s); |
| 937 | closure_call(&s->op.cl, btree_read_async, NULL, cl); |
| 938 | |
| 939 | continue_at(cl, request_read_done_bh, NULL); |
| 940 | } |
| 941 | |
| 942 | /* Process writes */ |
| 943 | |
| 944 | static void cached_dev_write_complete(struct closure *cl) |
| 945 | { |
| 946 | struct search *s = container_of(cl, struct search, cl); |
| 947 | struct cached_dev *dc = container_of(s->d, struct cached_dev, disk); |
| 948 | |
| 949 | up_read_non_owner(&dc->writeback_lock); |
| 950 | cached_dev_bio_complete(cl); |
| 951 | } |
| 952 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 953 | static void request_write(struct cached_dev *dc, struct search *s) |
| 954 | { |
| 955 | struct closure *cl = &s->cl; |
| 956 | struct bio *bio = &s->bio.bio; |
| 957 | struct bkey start, end; |
| 958 | start = KEY(dc->disk.id, bio->bi_sector, 0); |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 959 | end = KEY(dc->disk.id, bio_end_sector(bio), 0); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 960 | |
| 961 | bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, &start, &end); |
| 962 | |
| 963 | check_should_skip(dc, s); |
| 964 | down_read_non_owner(&dc->writeback_lock); |
| 965 | |
| 966 | if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) { |
| 967 | s->op.skip = false; |
| 968 | s->writeback = true; |
| 969 | } |
| 970 | |
| 971 | if (bio->bi_rw & REQ_DISCARD) |
| 972 | goto skip; |
| 973 | |
Kent Overstreet | 72c2706 | 2013-06-05 06:24:39 -0700 | [diff] [blame] | 974 | if (should_writeback(dc, s->orig_bio, |
| 975 | cache_mode(dc, bio), |
| 976 | s->op.skip)) { |
| 977 | s->op.skip = false; |
| 978 | s->writeback = true; |
| 979 | } |
| 980 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 981 | if (s->op.skip) |
| 982 | goto skip; |
| 983 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 984 | trace_bcache_write(s->orig_bio, s->writeback, s->op.skip); |
| 985 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 986 | if (!s->writeback) { |
| 987 | s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO, |
| 988 | dc->disk.bio_split); |
| 989 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 990 | closure_bio_submit(bio, cl, s->d); |
| 991 | } else { |
Kent Overstreet | 279afba | 2013-06-05 06:21:07 -0700 | [diff] [blame] | 992 | bch_writeback_add(dc); |
Kent Overstreet | e49c7c3 | 2013-06-26 17:25:38 -0700 | [diff] [blame] | 993 | |
| 994 | if (s->op.flush_journal) { |
| 995 | /* Also need to send a flush to the backing device */ |
| 996 | s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO, |
| 997 | dc->disk.bio_split); |
| 998 | |
| 999 | bio->bi_size = 0; |
| 1000 | bio->bi_vcnt = 0; |
| 1001 | closure_bio_submit(bio, cl, s->d); |
| 1002 | } else { |
| 1003 | s->op.cache_bio = bio; |
| 1004 | } |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1005 | } |
| 1006 | out: |
| 1007 | closure_call(&s->op.cl, bch_insert_data, NULL, cl); |
| 1008 | continue_at(cl, cached_dev_write_complete, NULL); |
| 1009 | skip: |
| 1010 | s->op.skip = true; |
| 1011 | s->op.cache_bio = s->orig_bio; |
| 1012 | bio_get(s->op.cache_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1013 | |
| 1014 | if ((bio->bi_rw & REQ_DISCARD) && |
| 1015 | !blk_queue_discard(bdev_get_queue(dc->bdev))) |
| 1016 | goto out; |
| 1017 | |
| 1018 | closure_bio_submit(bio, cl, s->d); |
| 1019 | goto out; |
| 1020 | } |
| 1021 | |
| 1022 | static void request_nodata(struct cached_dev *dc, struct search *s) |
| 1023 | { |
| 1024 | struct closure *cl = &s->cl; |
| 1025 | struct bio *bio = &s->bio.bio; |
| 1026 | |
| 1027 | if (bio->bi_rw & REQ_DISCARD) { |
| 1028 | request_write(dc, s); |
| 1029 | return; |
| 1030 | } |
| 1031 | |
| 1032 | if (s->op.flush_journal) |
| 1033 | bch_journal_meta(s->op.c, cl); |
| 1034 | |
| 1035 | closure_bio_submit(bio, cl, s->d); |
| 1036 | |
| 1037 | continue_at(cl, cached_dev_bio_complete, NULL); |
| 1038 | } |
| 1039 | |
| 1040 | /* Cached devices - read & write stuff */ |
| 1041 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1042 | unsigned bch_get_congested(struct cache_set *c) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1043 | { |
| 1044 | int i; |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1045 | long rand; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1046 | |
| 1047 | if (!c->congested_read_threshold_us && |
| 1048 | !c->congested_write_threshold_us) |
| 1049 | return 0; |
| 1050 | |
| 1051 | i = (local_clock_us() - c->congested_last_us) / 1024; |
| 1052 | if (i < 0) |
| 1053 | return 0; |
| 1054 | |
| 1055 | i += atomic_read(&c->congested); |
| 1056 | if (i >= 0) |
| 1057 | return 0; |
| 1058 | |
| 1059 | i += CONGESTED_MAX; |
| 1060 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1061 | if (i > 0) |
| 1062 | i = fract_exp_two(i, 6); |
| 1063 | |
| 1064 | rand = get_random_int(); |
| 1065 | i -= bitmap_weight(&rand, BITS_PER_LONG); |
| 1066 | |
| 1067 | return i > 0 ? i : 1; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1068 | } |
| 1069 | |
| 1070 | static void add_sequential(struct task_struct *t) |
| 1071 | { |
| 1072 | ewma_add(t->sequential_io_avg, |
| 1073 | t->sequential_io, 8, 0); |
| 1074 | |
| 1075 | t->sequential_io = 0; |
| 1076 | } |
| 1077 | |
Kent Overstreet | b1a67b0 | 2013-03-25 11:46:44 -0700 | [diff] [blame] | 1078 | static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k) |
| 1079 | { |
| 1080 | return &dc->io_hash[hash_64(k, RECENT_IO_BITS)]; |
| 1081 | } |
| 1082 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1083 | static void check_should_skip(struct cached_dev *dc, struct search *s) |
| 1084 | { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1085 | struct cache_set *c = s->op.c; |
| 1086 | struct bio *bio = &s->bio.bio; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1087 | unsigned mode = cache_mode(dc, bio); |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1088 | unsigned sectors, congested = bch_get_congested(c); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1089 | |
| 1090 | if (atomic_read(&dc->disk.detaching) || |
| 1091 | c->gc_stats.in_use > CUTOFF_CACHE_ADD || |
| 1092 | (bio->bi_rw & REQ_DISCARD)) |
| 1093 | goto skip; |
| 1094 | |
| 1095 | if (mode == CACHE_MODE_NONE || |
| 1096 | (mode == CACHE_MODE_WRITEAROUND && |
| 1097 | (bio->bi_rw & REQ_WRITE))) |
| 1098 | goto skip; |
| 1099 | |
| 1100 | if (bio->bi_sector & (c->sb.block_size - 1) || |
| 1101 | bio_sectors(bio) & (c->sb.block_size - 1)) { |
| 1102 | pr_debug("skipping unaligned io"); |
| 1103 | goto skip; |
| 1104 | } |
| 1105 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1106 | if (!congested && !dc->sequential_cutoff) |
| 1107 | goto rescale; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1108 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1109 | if (!congested && |
| 1110 | mode == CACHE_MODE_WRITEBACK && |
| 1111 | (bio->bi_rw & REQ_WRITE) && |
| 1112 | (bio->bi_rw & REQ_SYNC)) |
| 1113 | goto rescale; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1114 | |
| 1115 | if (dc->sequential_merge) { |
| 1116 | struct io *i; |
| 1117 | |
| 1118 | spin_lock(&dc->io_lock); |
| 1119 | |
Kent Overstreet | b1a67b0 | 2013-03-25 11:46:44 -0700 | [diff] [blame] | 1120 | hlist_for_each_entry(i, iohash(dc, bio->bi_sector), hash) |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1121 | if (i->last == bio->bi_sector && |
| 1122 | time_before(jiffies, i->jiffies)) |
| 1123 | goto found; |
| 1124 | |
| 1125 | i = list_first_entry(&dc->io_lru, struct io, lru); |
| 1126 | |
| 1127 | add_sequential(s->task); |
| 1128 | i->sequential = 0; |
| 1129 | found: |
| 1130 | if (i->sequential + bio->bi_size > i->sequential) |
| 1131 | i->sequential += bio->bi_size; |
| 1132 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1133 | i->last = bio_end_sector(bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1134 | i->jiffies = jiffies + msecs_to_jiffies(5000); |
| 1135 | s->task->sequential_io = i->sequential; |
| 1136 | |
| 1137 | hlist_del(&i->hash); |
Kent Overstreet | b1a67b0 | 2013-03-25 11:46:44 -0700 | [diff] [blame] | 1138 | hlist_add_head(&i->hash, iohash(dc, i->last)); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1139 | list_move_tail(&i->lru, &dc->io_lru); |
| 1140 | |
| 1141 | spin_unlock(&dc->io_lock); |
| 1142 | } else { |
| 1143 | s->task->sequential_io = bio->bi_size; |
| 1144 | |
| 1145 | add_sequential(s->task); |
| 1146 | } |
| 1147 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1148 | sectors = max(s->task->sequential_io, |
| 1149 | s->task->sequential_io_avg) >> 9; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1150 | |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1151 | if (dc->sequential_cutoff && |
| 1152 | sectors >= dc->sequential_cutoff >> 9) { |
| 1153 | trace_bcache_bypass_sequential(s->orig_bio); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1154 | goto skip; |
Kent Overstreet | c37511b | 2013-04-26 15:39:55 -0700 | [diff] [blame] | 1155 | } |
| 1156 | |
| 1157 | if (congested && sectors >= congested) { |
| 1158 | trace_bcache_bypass_congested(s->orig_bio); |
| 1159 | goto skip; |
| 1160 | } |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1161 | |
| 1162 | rescale: |
| 1163 | bch_rescale_priorities(c, bio_sectors(bio)); |
| 1164 | return; |
| 1165 | skip: |
| 1166 | bch_mark_sectors_bypassed(s, bio_sectors(bio)); |
| 1167 | s->op.skip = true; |
| 1168 | } |
| 1169 | |
| 1170 | static void cached_dev_make_request(struct request_queue *q, struct bio *bio) |
| 1171 | { |
| 1172 | struct search *s; |
| 1173 | struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; |
| 1174 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1175 | int cpu, rw = bio_data_dir(bio); |
| 1176 | |
| 1177 | cpu = part_stat_lock(); |
| 1178 | part_stat_inc(cpu, &d->disk->part0, ios[rw]); |
| 1179 | part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio)); |
| 1180 | part_stat_unlock(); |
| 1181 | |
| 1182 | bio->bi_bdev = dc->bdev; |
Kent Overstreet | 2903381 | 2013-04-11 15:14:35 -0700 | [diff] [blame] | 1183 | bio->bi_sector += dc->sb.data_offset; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1184 | |
| 1185 | if (cached_dev_get(dc)) { |
| 1186 | s = search_alloc(bio, d); |
| 1187 | trace_bcache_request_start(s, bio); |
| 1188 | |
| 1189 | if (!bio_has_data(bio)) |
| 1190 | request_nodata(dc, s); |
| 1191 | else if (rw) |
| 1192 | request_write(dc, s); |
| 1193 | else |
| 1194 | request_read(dc, s); |
| 1195 | } else { |
| 1196 | if ((bio->bi_rw & REQ_DISCARD) && |
| 1197 | !blk_queue_discard(bdev_get_queue(dc->bdev))) |
| 1198 | bio_endio(bio, 0); |
| 1199 | else |
| 1200 | bch_generic_make_request(bio, &d->bio_split_hook); |
| 1201 | } |
| 1202 | } |
| 1203 | |
| 1204 | static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode, |
| 1205 | unsigned int cmd, unsigned long arg) |
| 1206 | { |
| 1207 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1208 | return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg); |
| 1209 | } |
| 1210 | |
| 1211 | static int cached_dev_congested(void *data, int bits) |
| 1212 | { |
| 1213 | struct bcache_device *d = data; |
| 1214 | struct cached_dev *dc = container_of(d, struct cached_dev, disk); |
| 1215 | struct request_queue *q = bdev_get_queue(dc->bdev); |
| 1216 | int ret = 0; |
| 1217 | |
| 1218 | if (bdi_congested(&q->backing_dev_info, bits)) |
| 1219 | return 1; |
| 1220 | |
| 1221 | if (cached_dev_get(dc)) { |
| 1222 | unsigned i; |
| 1223 | struct cache *ca; |
| 1224 | |
| 1225 | for_each_cache(ca, d->c, i) { |
| 1226 | q = bdev_get_queue(ca->bdev); |
| 1227 | ret |= bdi_congested(&q->backing_dev_info, bits); |
| 1228 | } |
| 1229 | |
| 1230 | cached_dev_put(dc); |
| 1231 | } |
| 1232 | |
| 1233 | return ret; |
| 1234 | } |
| 1235 | |
| 1236 | void bch_cached_dev_request_init(struct cached_dev *dc) |
| 1237 | { |
| 1238 | struct gendisk *g = dc->disk.disk; |
| 1239 | |
| 1240 | g->queue->make_request_fn = cached_dev_make_request; |
| 1241 | g->queue->backing_dev_info.congested_fn = cached_dev_congested; |
| 1242 | dc->disk.cache_miss = cached_dev_cache_miss; |
| 1243 | dc->disk.ioctl = cached_dev_ioctl; |
| 1244 | } |
| 1245 | |
| 1246 | /* Flash backed devices */ |
| 1247 | |
| 1248 | static int flash_dev_cache_miss(struct btree *b, struct search *s, |
| 1249 | struct bio *bio, unsigned sectors) |
| 1250 | { |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1251 | struct bio_vec *bv; |
| 1252 | int i; |
| 1253 | |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1254 | /* Zero fill bio */ |
| 1255 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1256 | bio_for_each_segment(bv, bio, i) { |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1257 | unsigned j = min(bv->bv_len >> 9, sectors); |
| 1258 | |
| 1259 | void *p = kmap(bv->bv_page); |
| 1260 | memset(p + bv->bv_offset, 0, j << 9); |
| 1261 | kunmap(bv->bv_page); |
| 1262 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1263 | sectors -= j; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1264 | } |
| 1265 | |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1266 | bio_advance(bio, min(sectors << 9, bio->bi_size)); |
| 1267 | |
| 1268 | if (!bio->bi_size) |
| 1269 | s->op.lookup_done = true; |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1270 | |
| 1271 | return 0; |
| 1272 | } |
| 1273 | |
| 1274 | static void flash_dev_make_request(struct request_queue *q, struct bio *bio) |
| 1275 | { |
| 1276 | struct search *s; |
| 1277 | struct closure *cl; |
| 1278 | struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; |
| 1279 | int cpu, rw = bio_data_dir(bio); |
| 1280 | |
| 1281 | cpu = part_stat_lock(); |
| 1282 | part_stat_inc(cpu, &d->disk->part0, ios[rw]); |
| 1283 | part_stat_add(cpu, &d->disk->part0, sectors[rw], bio_sectors(bio)); |
| 1284 | part_stat_unlock(); |
| 1285 | |
| 1286 | s = search_alloc(bio, d); |
| 1287 | cl = &s->cl; |
| 1288 | bio = &s->bio.bio; |
| 1289 | |
| 1290 | trace_bcache_request_start(s, bio); |
| 1291 | |
| 1292 | if (bio_has_data(bio) && !rw) { |
| 1293 | closure_call(&s->op.cl, btree_read_async, NULL, cl); |
| 1294 | } else if (bio_has_data(bio) || s->op.skip) { |
| 1295 | bch_keybuf_check_overlapping(&s->op.c->moving_gc_keys, |
Kent Overstreet | 8e51e41 | 2013-06-06 18:15:57 -0700 | [diff] [blame^] | 1296 | &KEY(d->id, bio->bi_sector, 0), |
| 1297 | &KEY(d->id, bio_end_sector(bio), 0)); |
Kent Overstreet | cafe563 | 2013-03-23 16:11:31 -0700 | [diff] [blame] | 1298 | |
| 1299 | s->writeback = true; |
| 1300 | s->op.cache_bio = bio; |
| 1301 | |
| 1302 | closure_call(&s->op.cl, bch_insert_data, NULL, cl); |
| 1303 | } else { |
| 1304 | /* No data - probably a cache flush */ |
| 1305 | if (s->op.flush_journal) |
| 1306 | bch_journal_meta(s->op.c, cl); |
| 1307 | } |
| 1308 | |
| 1309 | continue_at(cl, search_free, NULL); |
| 1310 | } |
| 1311 | |
| 1312 | static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode, |
| 1313 | unsigned int cmd, unsigned long arg) |
| 1314 | { |
| 1315 | return -ENOTTY; |
| 1316 | } |
| 1317 | |
| 1318 | static int flash_dev_congested(void *data, int bits) |
| 1319 | { |
| 1320 | struct bcache_device *d = data; |
| 1321 | struct request_queue *q; |
| 1322 | struct cache *ca; |
| 1323 | unsigned i; |
| 1324 | int ret = 0; |
| 1325 | |
| 1326 | for_each_cache(ca, d->c, i) { |
| 1327 | q = bdev_get_queue(ca->bdev); |
| 1328 | ret |= bdi_congested(&q->backing_dev_info, bits); |
| 1329 | } |
| 1330 | |
| 1331 | return ret; |
| 1332 | } |
| 1333 | |
| 1334 | void bch_flash_dev_request_init(struct bcache_device *d) |
| 1335 | { |
| 1336 | struct gendisk *g = d->disk; |
| 1337 | |
| 1338 | g->queue->make_request_fn = flash_dev_make_request; |
| 1339 | g->queue->backing_dev_info.congested_fn = flash_dev_congested; |
| 1340 | d->cache_miss = flash_dev_cache_miss; |
| 1341 | d->ioctl = flash_dev_ioctl; |
| 1342 | } |
| 1343 | |
| 1344 | void bch_request_exit(void) |
| 1345 | { |
| 1346 | #ifdef CONFIG_CGROUP_BCACHE |
| 1347 | cgroup_unload_subsys(&bcache_subsys); |
| 1348 | #endif |
| 1349 | if (bch_search_cache) |
| 1350 | kmem_cache_destroy(bch_search_cache); |
| 1351 | } |
| 1352 | |
| 1353 | int __init bch_request_init(void) |
| 1354 | { |
| 1355 | bch_search_cache = KMEM_CACHE(search, 0); |
| 1356 | if (!bch_search_cache) |
| 1357 | return -ENOMEM; |
| 1358 | |
| 1359 | #ifdef CONFIG_CGROUP_BCACHE |
| 1360 | cgroup_load_subsys(&bcache_subsys); |
| 1361 | init_bch_cgroup(&bcache_default_cgroup); |
| 1362 | |
| 1363 | cgroup_add_cftypes(&bcache_subsys, bch_files); |
| 1364 | #endif |
| 1365 | return 0; |
| 1366 | } |